nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

tcp: use TCP_SKB_CB(skb)->tcp_flags in input path

Browse source 
Input path of TCP do not currently uses TCP_SKB_CB(skb)->tcp_flags,
which is only used in output path.

tcp_recvmsg(), looks at tcp_hdr(skb)->syn for every skb found in receive queue,
and its unfortunate because this bit is located in a cache line right before
the payload.

We can simplify TCP by copying tcp flags into TCP_SKB_CB(skb)->tcp_flags.

This patch does so, and avoids the cache line miss in tcp_recvmsg()

Following patches will
- allow a segment with FIN being coalesced in tcp_try_coalesce()
- simplify tcp_collapse() by not copying the headers.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Neal Cardwell <ncardwell@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2014-09-15 04:19:51 -07:00 committed by David S. Miller
parent 13bb5180e8
commit e11ecddf51
4 changed files with 17 additions and 13 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

18
net/ipv4/tcp.c
View file

@ -1510,9 +1510,9 @@ static struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) { while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
offset = seq - TCP_SKB_CB(skb)->seq; offset = seq - TCP_SKB_CB(skb)->seq;
if (tcp_hdr(skb)->syn) if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
offset--; offset--;
if (offset < skb->len || tcp_hdr(skb)->fin) { if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
*off = offset; *off = offset;
return skb; return skb;
} }
@ -1585,7 +1585,7 @@ int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
if (offset + 1 != skb->len) if (offset + 1 != skb->len)
continue; continue;
} }
if (tcp_hdr(skb)->fin) { if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
sk_eat_skb(sk, skb, false); sk_eat_skb(sk, skb, false);
++seq; ++seq;
break; break;
@ -1722,11 +1722,11 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
break; break;
offset = *seq - TCP_SKB_CB(skb)->seq; offset = *seq - TCP_SKB_CB(skb)->seq;
if (tcp_hdr(skb)->syn) if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)
offset--; offset--;
if (offset < skb->len) if (offset < skb->len)
goto found_ok_skb; goto found_ok_skb;
if (tcp_hdr(skb)->fin) if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
goto found_fin_ok; goto found_fin_ok;
WARN(!(flags & MSG_PEEK), WARN(!(flags & MSG_PEEK),
"recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n", "recvmsg bug 2: copied %X seq %X rcvnxt %X fl %X\n",
@ -1959,7 +1959,7 @@ int tcp_recvmsg(struct kiocb *iocb, struct sock *sk, struct msghdr *msg,
if (used + offset < skb->len) if (used + offset < skb->len)
continue; continue;
if (tcp_hdr(skb)->fin) if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
goto found_fin_ok; goto found_fin_ok;
if (!(flags & MSG_PEEK)) { if (!(flags & MSG_PEEK)) {
sk_eat_skb(sk, skb, copied_early); sk_eat_skb(sk, skb, copied_early);
@ -2160,8 +2160,10 @@ void tcp_close(struct sock *sk, long timeout)
* reader process may not have drained the data yet! * reader process may not have drained the data yet!
*/ */
while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) { while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq - u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
tcp_hdr(skb)->fin;
if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
len--;
data_was_unread += len; data_was_unread += len;
__kfree_skb(skb); __kfree_skb(skb);
} }

10
net/ipv4/tcp_input.c
View file

@ -4093,7 +4093,7 @@ static void tcp_ofo_queue(struct sock *sk)
__skb_unlink(skb, &tp->out_of_order_queue); __skb_unlink(skb, &tp->out_of_order_queue);
__skb_queue_tail(&sk->sk_receive_queue, skb); __skb_queue_tail(&sk->sk_receive_queue, skb);
tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq; tp->rcv_nxt = TCP_SKB_CB(skb)->end_seq;
if (tcp_hdr(skb)->fin) if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
tcp_fin(sk); tcp_fin(sk);
} }
} }
@ -4513,7 +4513,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list,
* - bloated or contains data before "start" or * - bloated or contains data before "start" or
* overlaps to the next one. * overlaps to the next one.
*/ */
if (!tcp_hdr(skb)->syn && !tcp_hdr(skb)->fin && if (!(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)) &&
(tcp_win_from_space(skb->truesize) > skb->len || (tcp_win_from_space(skb->truesize) > skb->len ||
before(TCP_SKB_CB(skb)->seq, start))) { before(TCP_SKB_CB(skb)->seq, start))) {
end_of_skbs = false; end_of_skbs = false;
@ -4532,7 +4532,8 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list,
/* Decided to skip this, advance start seq. */ /* Decided to skip this, advance start seq. */
start = TCP_SKB_CB(skb)->end_seq; start = TCP_SKB_CB(skb)->end_seq;
} }
if (end_of_skbs || tcp_hdr(skb)->syn || tcp_hdr(skb)->fin) if (end_of_skbs ||
(TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
return; return;
while (before(start, end)) { while (before(start, end)) {
@ -4579,8 +4580,7 @@ tcp_collapse(struct sock *sk, struct sk_buff_head *list,
skb = tcp_collapse_one(sk, skb, list); skb = tcp_collapse_one(sk, skb, list);
if (!skb || if (!skb ||
skb == tail || skb == tail ||
tcp_hdr(skb)->syn || (TCP_SKB_CB(skb)->tcp_flags & (TCPHDR_SYN | TCPHDR_FIN)))
tcp_hdr(skb)->fin)
return; return;
} }
} }

1
net/ipv4/tcp_ipv4.c
View file

@ -1638,6 +1638,7 @@ int tcp_v4_rcv(struct sk_buff *skb)
TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
skb->len - th->doff * 4); skb->len - th->doff * 4);
TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
TCP_SKB_CB(skb)->tcp_tw_isn = 0; TCP_SKB_CB(skb)->tcp_tw_isn = 0;
TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph); TCP_SKB_CB(skb)->ip_dsfield = ipv4_get_dsfield(iph);
TCP_SKB_CB(skb)->sacked = 0; TCP_SKB_CB(skb)->sacked = 0;

1
net/ipv6/tcp_ipv6.c
View file

@ -1415,6 +1415,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin + TCP_SKB_CB(skb)->end_seq = (TCP_SKB_CB(skb)->seq + th->syn + th->fin +
skb->len - th->doff*4); skb->len - th->doff*4);
TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq); TCP_SKB_CB(skb)->ack_seq = ntohl(th->ack_seq);
TCP_SKB_CB(skb)->tcp_flags = tcp_flag_byte(th);
TCP_SKB_CB(skb)->tcp_tw_isn = 0; TCP_SKB_CB(skb)->tcp_tw_isn = 0;
TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr); TCP_SKB_CB(skb)->ip_dsfield = ipv6_get_dsfield(hdr);
TCP_SKB_CB(skb)->sacked = 0; TCP_SKB_CB(skb)->sacked = 0;