mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

udp: paged allocation with gso

Browse source 
When sending large datagrams that are later segmented, store data in
page frags to avoid copying from linear in skb_segment.

Signed-off-by: Willem de Bruijn <willemb@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Willem de Bruijn 2018-04-26 13:42:19 -04:00 committed by David S. Miller
parent ad405857b1
commit 15e36f5b8e
2 changed files with 25 additions and 9 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

15
net/ipv4/ip_output.c
View file

@ -878,11 +878,13 @@ static int __ip_append_data(struct sock *sk,
struct rtable *rt = (struct rtable *)cork->dst; struct rtable *rt = (struct rtable *)cork->dst;
unsigned int wmem_alloc_delta = 0; unsigned int wmem_alloc_delta = 0;
u32 tskey = 0; u32 tskey = 0;
bool paged;
skb = skb_peek_tail(queue); skb = skb_peek_tail(queue);
exthdrlen = !skb ? rt->dst.header_len : 0; exthdrlen = !skb ? rt->dst.header_len : 0;
mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize; mtu = cork->gso_size ? IP_MAX_MTU : cork->fragsize;
paged = !!cork->gso_size;
if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP && if (cork->tx_flags & SKBTX_ANY_SW_TSTAMP &&
sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID) sk->sk_tsflags & SOF_TIMESTAMPING_OPT_ID)
@ -934,6 +936,7 @@ static int __ip_append_data(struct sock *sk,
unsigned int fraglen; unsigned int fraglen;
unsigned int fraggap; unsigned int fraggap;
unsigned int alloclen; unsigned int alloclen;
unsigned int pagedlen = 0;
struct sk_buff *skb_prev; struct sk_buff *skb_prev;
alloc_new_skb: alloc_new_skb:
skb_prev = skb; skb_prev = skb;
@ -954,8 +957,12 @@ static int __ip_append_data(struct sock *sk,
if ((flags & MSG_MORE) && if ((flags & MSG_MORE) &&
!(rt->dst.dev->features&NETIF_F_SG)) !(rt->dst.dev->features&NETIF_F_SG))
alloclen = mtu; alloclen = mtu;
else else if (!paged)
alloclen = fraglen; alloclen = fraglen;
else {
alloclen = min_t(int, fraglen, MAX_HEADER);
pagedlen = fraglen - alloclen;
}
alloclen += exthdrlen; alloclen += exthdrlen;
@ -999,7 +1006,7 @@ static int __ip_append_data(struct sock *sk,
/* /*
* Find where to start putting bytes. * Find where to start putting bytes.
*/ */
data = skb_put(skb, fraglen + exthdrlen); data = skb_put(skb, fraglen + exthdrlen - pagedlen);
skb_set_network_header(skb, exthdrlen); skb_set_network_header(skb, exthdrlen);
skb->transport_header = (skb->network_header + skb->transport_header = (skb->network_header +
fragheaderlen); fragheaderlen);
@ -1015,7 +1022,7 @@ static int __ip_append_data(struct sock *sk,
pskb_trim_unique(skb_prev, maxfraglen); pskb_trim_unique(skb_prev, maxfraglen);
} }
copy = datalen - transhdrlen - fraggap; copy = datalen - transhdrlen - fraggap - pagedlen;
if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) { if (copy > 0 && getfrag(from, data + transhdrlen, offset, copy, fraggap, skb) < 0) {
err = -EFAULT; err = -EFAULT;
kfree_skb(skb); kfree_skb(skb);
@ -1023,7 +1030,7 @@ static int __ip_append_data(struct sock *sk,
} }
offset += copy; offset += copy;
length -= datalen - fraggap; length -= copy + transhdrlen;
transhdrlen = 0; transhdrlen = 0;
exthdrlen = 0; exthdrlen = 0;
csummode = CHECKSUM_NONE; csummode = CHECKSUM_NONE;

19
net/ipv6/ip6_output.c
View file

@ -1276,6 +1276,7 @@ static int __ip6_append_data(struct sock *sk,
int csummode = CHECKSUM_NONE; int csummode = CHECKSUM_NONE;
unsigned int maxnonfragsize, headersize; unsigned int maxnonfragsize, headersize;
unsigned int wmem_alloc_delta = 0; unsigned int wmem_alloc_delta = 0;
bool paged;
skb = skb_peek_tail(queue); skb = skb_peek_tail(queue);
if (!skb) { if (!skb) {
@ -1283,6 +1284,7 @@ static int __ip6_append_data(struct sock *sk,
dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len; dst_exthdrlen = rt->dst.header_len - rt->rt6i_nfheader_len;
} }
paged = !!cork->gso_size;
mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize; mtu = cork->gso_size ? IP6_MAX_MTU : cork->fragsize;
orig_mtu = mtu; orig_mtu = mtu;
@ -1374,6 +1376,7 @@ static int __ip6_append_data(struct sock *sk,
unsigned int fraglen; unsigned int fraglen;
unsigned int fraggap; unsigned int fraggap;
unsigned int alloclen; unsigned int alloclen;
unsigned int pagedlen = 0;
alloc_new_skb: alloc_new_skb:
/* There's no room in the current skb */ /* There's no room in the current skb */
if (skb) if (skb)
@ -1396,11 +1399,17 @@ static int __ip6_append_data(struct sock *sk,
if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen) if (datalen > (cork->length <= mtu && !(cork->flags & IPCORK_ALLFRAG) ? mtu : maxfraglen) - fragheaderlen)
datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len; datalen = maxfraglen - fragheaderlen - rt->dst.trailer_len;
fraglen = datalen + fragheaderlen;
if ((flags & MSG_MORE) && if ((flags & MSG_MORE) &&
!(rt->dst.dev->features&NETIF_F_SG)) !(rt->dst.dev->features&NETIF_F_SG))
alloclen = mtu; alloclen = mtu;
else else if (!paged)
alloclen = datalen + fragheaderlen; alloclen = fraglen;
else {
alloclen = min_t(int, fraglen, MAX_HEADER);
pagedlen = fraglen - alloclen;
}
alloclen += dst_exthdrlen; alloclen += dst_exthdrlen;
@ -1422,7 +1431,7 @@ static int __ip6_append_data(struct sock *sk,
*/ */
alloclen += sizeof(struct frag_hdr); alloclen += sizeof(struct frag_hdr);
copy = datalen - transhdrlen - fraggap; copy = datalen - transhdrlen - fraggap - pagedlen;
if (copy < 0) { if (copy < 0) {
err = -EINVAL; err = -EINVAL;
goto error; goto error;
@ -1461,7 +1470,7 @@ static int __ip6_append_data(struct sock *sk,
/* /*
* Find where to start putting bytes * Find where to start putting bytes
*/ */
data = skb_put(skb, fraglen); data = skb_put(skb, fraglen - pagedlen);
skb_set_network_header(skb, exthdrlen); skb_set_network_header(skb, exthdrlen);
data += fragheaderlen; data += fragheaderlen;
skb->transport_header = (skb->network_header + skb->transport_header = (skb->network_header +
@ -1484,7 +1493,7 @@ static int __ip6_append_data(struct sock *sk,
} }
offset += copy; offset += copy;
length -= datalen - fraggap; length -= copy + transhdrlen;
transhdrlen = 0; transhdrlen = 0;
exthdrlen = 0; exthdrlen = 0;
dst_exthdrlen = 0; dst_exthdrlen = 0;