nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

tcp: add tcp_inet6_sk() helper

Browse source 
TCP ipv6 fast path dereferences a pointer to get to the inet6
part of a tcp socket, but given the fixed memory placement,
we can do better and avoid a possible cache line miss.

This also reduces register pressure, since we let the compiler
know about this memory placement.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2019-03-19 07:01:08 -07:00 committed by David S. Miller
parent f8d6ae0d27
commit 93a77c11ae
1 changed files with 26 additions and 18 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

44
net/ipv6/tcp_ipv6.c
View file

@ -90,6 +90,17 @@ static struct tcp_md5sig_key *tcp_v6_md5_do_lookup(const struct sock *sk,
} }
#endif #endif
/* Helper returning the inet6 address from a given tcp socket.
* It can be used in TCP stack instead of inet6_sk(sk).
* This avoids a dereference and allow compiler optimizations.
*/
static struct ipv6_pinfo *tcp_inet6_sk(const struct sock *sk)
{
struct tcp6_sock *tcp6 = container_of(tcp_sk(sk), struct tcp6_sock, tcp);
return &tcp6->inet6;
}
static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb) static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
{ {
struct dst_entry *dst = skb_dst(skb); struct dst_entry *dst = skb_dst(skb);
@ -99,7 +110,7 @@ static void inet6_sk_rx_dst_set(struct sock *sk, const struct sk_buff *skb)
sk->sk_rx_dst = dst; sk->sk_rx_dst = dst;
inet_sk(sk)->rx_dst_ifindex = skb->skb_iif; inet_sk(sk)->rx_dst_ifindex = skb->skb_iif;
inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt); tcp_inet6_sk(sk)->rx_dst_cookie = rt6_get_cookie(rt);
} }
} }
@ -138,7 +149,7 @@ static int tcp_v6_connect(struct sock *sk, struct sockaddr *uaddr,
struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr; struct sockaddr_in6 *usin = (struct sockaddr_in6 *) uaddr;
struct inet_sock *inet = inet_sk(sk); struct inet_sock *inet = inet_sk(sk);
struct inet_connection_sock *icsk = inet_csk(sk); struct inet_connection_sock *icsk = inet_csk(sk);
struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_pinfo *np = tcp_inet6_sk(sk);
struct tcp_sock *tp = tcp_sk(sk); struct tcp_sock *tp = tcp_sk(sk);
struct in6_addr *saddr = NULL, *final_p, final; struct in6_addr *saddr = NULL, *final_p, final;
struct ipv6_txoptions *opt; struct ipv6_txoptions *opt;
@ -390,7 +401,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
if (sk->sk_state == TCP_CLOSE) if (sk->sk_state == TCP_CLOSE)
goto out; goto out;
if (ipv6_hdr(skb)->hop_limit < inet6_sk(sk)->min_hopcount) { if (ipv6_hdr(skb)->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
goto out; goto out;
} }
@ -405,7 +416,7 @@ static int tcp_v6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
goto out; goto out;
} }
np = inet6_sk(sk); np = tcp_inet6_sk(sk);
if (type == NDISC_REDIRECT) { if (type == NDISC_REDIRECT) {
if (!sock_owned_by_user(sk)) { if (!sock_owned_by_user(sk)) {
@ -478,7 +489,7 @@ static int tcp_v6_send_synack(const struct sock *sk, struct dst_entry *dst,
enum tcp_synack_type synack_type) enum tcp_synack_type synack_type)
{ {
struct inet_request_sock *ireq = inet_rsk(req); struct inet_request_sock *ireq = inet_rsk(req);
struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_pinfo *np = tcp_inet6_sk(sk);
struct ipv6_txoptions *opt; struct ipv6_txoptions *opt;
struct flowi6 *fl6 = &fl->u.ip6; struct flowi6 *fl6 = &fl->u.ip6;
struct sk_buff *skb; struct sk_buff *skb;
@ -737,7 +748,7 @@ static void tcp_v6_init_req(struct request_sock *req,
{ {
bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags); bool l3_slave = ipv6_l3mdev_skb(TCP_SKB_CB(skb)->header.h6.flags);
struct inet_request_sock *ireq = inet_rsk(req); struct inet_request_sock *ireq = inet_rsk(req);
const struct ipv6_pinfo *np = inet6_sk(sk_listener); const struct ipv6_pinfo *np = tcp_inet6_sk(sk_listener);
ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr; ireq->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr; ireq->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
@ -1066,9 +1077,8 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
{ {
struct inet_request_sock *ireq; struct inet_request_sock *ireq;
struct ipv6_pinfo *newnp; struct ipv6_pinfo *newnp;
const struct ipv6_pinfo *np = inet6_sk(sk); const struct ipv6_pinfo *np = tcp_inet6_sk(sk);
struct ipv6_txoptions *opt; struct ipv6_txoptions *opt;
struct tcp6_sock *newtcp6sk;
struct inet_sock *newinet; struct inet_sock *newinet;
struct tcp_sock *newtp; struct tcp_sock *newtp;
struct sock *newsk; struct sock *newsk;
@ -1088,11 +1098,10 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
if (!newsk) if (!newsk)
return NULL; return NULL;
newtcp6sk = (struct tcp6_sock *)newsk; inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;
newinet = inet_sk(newsk); newinet = inet_sk(newsk);
newnp = inet6_sk(newsk); newnp = tcp_inet6_sk(newsk);
newtp = tcp_sk(newsk); newtp = tcp_sk(newsk);
memcpy(newnp, np, sizeof(struct ipv6_pinfo)); memcpy(newnp, np, sizeof(struct ipv6_pinfo));
@ -1156,12 +1165,11 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
ip6_dst_store(newsk, dst, NULL, NULL); ip6_dst_store(newsk, dst, NULL, NULL);
inet6_sk_rx_dst_set(newsk, skb); inet6_sk_rx_dst_set(newsk, skb);
newtcp6sk = (struct tcp6_sock *)newsk; inet_sk(newsk)->pinet6 = tcp_inet6_sk(newsk);
inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;
newtp = tcp_sk(newsk); newtp = tcp_sk(newsk);
newinet = inet_sk(newsk); newinet = inet_sk(newsk);
newnp = inet6_sk(newsk); newnp = tcp_inet6_sk(newsk);
memcpy(newnp, np, sizeof(struct ipv6_pinfo)); memcpy(newnp, np, sizeof(struct ipv6_pinfo));
@ -1276,9 +1284,9 @@ static struct sock *tcp_v6_syn_recv_sock(const struct sock *sk, struct sk_buff *
*/ */
static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb) static int tcp_v6_do_rcv(struct sock *sk, struct sk_buff *skb)
{ {
struct ipv6_pinfo *np = inet6_sk(sk); struct ipv6_pinfo *np = tcp_inet6_sk(sk);
struct tcp_sock *tp;
struct sk_buff *opt_skb = NULL; struct sk_buff *opt_skb = NULL;
struct tcp_sock *tp;
/* Imagine: socket is IPv6. IPv4 packet arrives, /* Imagine: socket is IPv6. IPv4 packet arrives,
goes to IPv4 receive handler and backlogged. goes to IPv4 receive handler and backlogged.
@ -1524,7 +1532,7 @@ static int tcp_v6_rcv(struct sk_buff *skb)
return 0; return 0;
} }
} }
if (hdr->hop_limit < inet6_sk(sk)->min_hopcount) { if (hdr->hop_limit < tcp_inet6_sk(sk)->min_hopcount) {
__NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP); __NET_INC_STATS(net, LINUX_MIB_TCPMINTTLDROP);
goto discard_and_relse; goto discard_and_relse;
} }
@ -1669,7 +1677,7 @@ static void tcp_v6_early_demux(struct sk_buff *skb)
struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst); struct dst_entry *dst = READ_ONCE(sk->sk_rx_dst);
if (dst) if (dst)
dst = dst_check(dst, inet6_sk(sk)->rx_dst_cookie); dst = dst_check(dst, tcp_inet6_sk(sk)->rx_dst_cookie);
if (dst && if (dst &&
inet_sk(sk)->rx_dst_ifindex == skb->skb_iif) inet_sk(sk)->rx_dst_ifindex == skb->skb_iif)
skb_dst_set_noref(skb, dst); skb_dst_set_noref(skb, dst);