mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

netfilter: merge ctinfo into nfct pointer storage area

Browse source 
After this change conntrack operations (lookup, creation, matching from
ruleset) only access one instead of two sk_buff cache lines.

This works for normal conntracks because those are allocated from a slab
that guarantees hw cacheline or 8byte alignment (whatever is larger)
so the 3 bits needed for ctinfo won't overlap with nf_conn addresses.

Template allocation now does manual address alignment (see previous change)
on arches that don't have sufficent kmalloc min alignment.

Some spots intentionally use skb->_nfct instead of skb_nfct() helpers,
this is to avoid undoing the skb_nfct() use when we remove untracked
conntrack object in the future.

Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: Pablo Neira Ayuso <pablo@netfilter.org>
This commit is contained in:
Florian Westphal 2017-01-23 18:21:59 +01:00 committed by Pablo Neira Ayuso
parent 3032230920
commit a9e419dc7b
7 changed files with 28 additions and 26 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

21
include/linux/skbuff.h
View file

@ -585,7 +585,6 @@ static inline bool skb_mstamp_after(const struct skb_mstamp *t1,
* @cloned: Head may be cloned (check refcnt to be sure) * @cloned: Head may be cloned (check refcnt to be sure)
* @ip_summed: Driver fed us an IP checksum * @ip_summed: Driver fed us an IP checksum
* @nohdr: Payload reference only, must not modify header * @nohdr: Payload reference only, must not modify header
* @nfctinfo: Relationship of this skb to the connection
* @pkt_type: Packet class * @pkt_type: Packet class
* @fclone: skbuff clone status * @fclone: skbuff clone status
* @ipvs_property: skbuff is owned by ipvs * @ipvs_property: skbuff is owned by ipvs
@ -594,7 +593,7 @@ static inline bool skb_mstamp_after(const struct skb_mstamp *t1,
* @nf_trace: netfilter packet trace flag * @nf_trace: netfilter packet trace flag
* @protocol: Packet protocol from driver * @protocol: Packet protocol from driver
* @destructor: Destruct function * @destructor: Destruct function
* @nfct: Associated connection, if any * @_nfct: Associated connection, if any (with nfctinfo bits)
* @nf_bridge: Saved data about a bridged frame - see br_netfilter.c * @nf_bridge: Saved data about a bridged frame - see br_netfilter.c
* @skb_iif: ifindex of device we arrived on * @skb_iif: ifindex of device we arrived on
* @tc_index: Traffic control index * @tc_index: Traffic control index
@ -668,7 +667,7 @@ struct sk_buff {
struct sec_path *sp; struct sec_path *sp;
#endif #endif
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
struct nf_conntrack *nfct; unsigned long _nfct;
#endif #endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
struct nf_bridge_info *nf_bridge; struct nf_bridge_info *nf_bridge;
@ -721,7 +720,6 @@ struct sk_buff {
__u8 pkt_type:3; __u8 pkt_type:3;
__u8 pfmemalloc:1; __u8 pfmemalloc:1;
__u8 ignore_df:1; __u8 ignore_df:1;
__u8 nfctinfo:3;
__u8 nf_trace:1; __u8 nf_trace:1;
__u8 ip_summed:2; __u8 ip_summed:2;
@ -836,6 +834,7 @@ static inline bool skb_pfmemalloc(const struct sk_buff *skb)
#define SKB_DST_NOREF 1UL #define SKB_DST_NOREF 1UL
#define SKB_DST_PTRMASK ~(SKB_DST_NOREF) #define SKB_DST_PTRMASK ~(SKB_DST_NOREF)
#define SKB_NFCT_PTRMASK ~(7UL)
/** /**
* skb_dst - returns skb dst_entry * skb_dst - returns skb dst_entry
* @skb: buffer * @skb: buffer
@ -3556,7 +3555,7 @@ static inline void skb_remcsum_process(struct sk_buff *skb, void *ptr,
static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb) static inline struct nf_conntrack *skb_nfct(const struct sk_buff *skb)
{ {
#if IS_ENABLED(CONFIG_NF_CONNTRACK) #if IS_ENABLED(CONFIG_NF_CONNTRACK)
return skb->nfct; return (void *)(skb->_nfct & SKB_NFCT_PTRMASK);
#else #else
return NULL; return NULL;
#endif #endif
@ -3590,8 +3589,8 @@ static inline void nf_bridge_get(struct nf_bridge_info *nf_bridge)
static inline void nf_reset(struct sk_buff *skb) static inline void nf_reset(struct sk_buff *skb)
{ {
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
nf_conntrack_put(skb->nfct); nf_conntrack_put(skb_nfct(skb));
skb->nfct = NULL; skb->_nfct = 0;
#endif #endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
nf_bridge_put(skb->nf_bridge); nf_bridge_put(skb->nf_bridge);
@ -3611,10 +3610,8 @@ static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
bool copy) bool copy)
{ {
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
dst->nfct = src->nfct; dst->_nfct = src->_nfct;
nf_conntrack_get(src->nfct); nf_conntrack_get(skb_nfct(src));
if (copy)
dst->nfctinfo = src->nfctinfo;
#endif #endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
dst->nf_bridge = src->nf_bridge; dst->nf_bridge = src->nf_bridge;
@ -3629,7 +3626,7 @@ static inline void __nf_copy(struct sk_buff *dst, const struct sk_buff *src,
static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src) static inline void nf_copy(struct sk_buff *dst, const struct sk_buff *src)
{ {
#if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE) #if defined(CONFIG_NF_CONNTRACK) || defined(CONFIG_NF_CONNTRACK_MODULE)
nf_conntrack_put(dst->nfct); nf_conntrack_put(skb_nfct(dst));
#endif #endif
#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER) #if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)
nf_bridge_put(dst->nf_bridge); nf_bridge_put(dst->nf_bridge);

11
include/net/netfilter/nf_conntrack.h
View file

@ -76,7 +76,7 @@ struct nf_conn {
/* Usage count in here is 1 for hash table, 1 per skb, /* Usage count in here is 1 for hash table, 1 per skb,
* plus 1 for any connection(s) we are `master' for * plus 1 for any connection(s) we are `master' for
* *
* Hint, SKB address this struct and refcnt via skb->nfct and * Hint, SKB address this struct and refcnt via skb->_nfct and
* helpers nf_conntrack_get() and nf_conntrack_put(). * helpers nf_conntrack_get() and nf_conntrack_put().
* Helper nf_ct_put() equals nf_conntrack_put() by dec refcnt, * Helper nf_ct_put() equals nf_conntrack_put() by dec refcnt,
* beware nf_ct_get() is different and don't inc refcnt. * beware nf_ct_get() is different and don't inc refcnt.
@ -164,13 +164,15 @@ int nf_conntrack_tuple_taken(const struct nf_conntrack_tuple *tuple,
const struct nf_conn *ignored_conntrack); const struct nf_conn *ignored_conntrack);
#define NFCT_INFOMASK 7UL #define NFCT_INFOMASK 7UL
#define NFCT_PTRMASK ~(NFCT_INFOMASK)
/* Return conntrack_info and tuple hash for given skb. */ /* Return conntrack_info and tuple hash for given skb. */
static inline struct nf_conn * static inline struct nf_conn *
nf_ct_get(const struct sk_buff *skb, enum ip_conntrack_info *ctinfo) nf_ct_get(const struct sk_buff *skb, enum ip_conntrack_info *ctinfo)
{ {
*ctinfo = skb->nfctinfo; *ctinfo = skb->_nfct & NFCT_INFOMASK;
return (struct nf_conn *)skb->nfct;
return (struct nf_conn *)(skb->_nfct & NFCT_PTRMASK);
} }
/* decrement reference count on a conntrack */ /* decrement reference count on a conntrack */
@ -347,8 +349,7 @@ void nf_ct_tmpl_free(struct nf_conn *tmpl);
static inline void static inline void
nf_ct_set(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info info) nf_ct_set(struct sk_buff *skb, struct nf_conn *ct, enum ip_conntrack_info info)
{ {
skb->nfct = &ct->ct_general; skb->_nfct = (unsigned long)ct | info;
skb->nfctinfo = info;
} }
#define NF_CT_STAT_INC(net, count) __this_cpu_inc((net)->ct.stat->count) #define NF_CT_STAT_INC(net, count) __this_cpu_inc((net)->ct.stat->count)

2
net/ipv6/netfilter/nf_dup_ipv6.c
View file

@ -59,7 +59,7 @@ void nf_dup_ipv6(struct net *net, struct sk_buff *skb, unsigned int hooknum,
#if IS_ENABLED(CONFIG_NF_CONNTRACK) #if IS_ENABLED(CONFIG_NF_CONNTRACK)
nf_reset(skb); nf_reset(skb);
nf_ct_set(skb, nf_ct_untracked_get(), IP_CT_NEW); nf_ct_set(skb, nf_ct_untracked_get(), IP_CT_NEW);
nf_conntrack_get(skb->nfct); nf_conntrack_get(skb_nfct(skb));
#endif #endif
if (hooknum == NF_INET_PRE_ROUTING || if (hooknum == NF_INET_PRE_ROUTING ||
hooknum == NF_INET_LOCAL_IN) { hooknum == NF_INET_LOCAL_IN) {

2
net/netfilter/core.c
View file

@ -375,7 +375,7 @@ void nf_ct_attach(struct sk_buff *new, const struct sk_buff *skb)
{ {
void (*attach)(struct sk_buff *, const struct sk_buff *); void (*attach)(struct sk_buff *, const struct sk_buff *);
if (skb->nfct) { if (skb->_nfct) {
rcu_read_lock(); rcu_read_lock();
attach = rcu_dereference(ip_ct_attach); attach = rcu_dereference(ip_ct_attach);
if (attach) if (attach)

11
net/netfilter/nf_conntrack_core.c
View file

@ -1239,7 +1239,7 @@ init_conntrack(struct net *net, struct nf_conn *tmpl,
return &ct->tuplehash[IP_CT_DIR_ORIGINAL]; return &ct->tuplehash[IP_CT_DIR_ORIGINAL];
} }
/* On success, returns conntrack ptr, sets skb->nfct and ctinfo */ /* On success, returns conntrack ptr, sets skb->_nfct | ctinfo */
static inline struct nf_conn * static inline struct nf_conn *
resolve_normal_ct(struct net *net, struct nf_conn *tmpl, resolve_normal_ct(struct net *net, struct nf_conn *tmpl,
struct sk_buff *skb, struct sk_buff *skb,
@ -1323,7 +1323,7 @@ nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
NF_CT_STAT_INC_ATOMIC(net, ignore); NF_CT_STAT_INC_ATOMIC(net, ignore);
return NF_ACCEPT; return NF_ACCEPT;
} }
skb->nfct = NULL; skb->_nfct = 0;
} }
/* rcu_read_lock()ed by nf_hook_thresh */ /* rcu_read_lock()ed by nf_hook_thresh */
@ -1352,7 +1352,7 @@ nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
goto out; goto out;
} }
/* ICMP[v6] protocol trackers may assign one conntrack. */ /* ICMP[v6] protocol trackers may assign one conntrack. */
if (skb->nfct) if (skb->_nfct)
goto out; goto out;
} }
repeat: repeat:
@ -1383,7 +1383,7 @@ nf_conntrack_in(struct net *net, u_int8_t pf, unsigned int hooknum,
* the netfilter core what to do */ * the netfilter core what to do */
pr_debug("nf_conntrack_in: Can't track with proto module\n"); pr_debug("nf_conntrack_in: Can't track with proto module\n");
nf_conntrack_put(&ct->ct_general); nf_conntrack_put(&ct->ct_general);
skb->nfct = NULL; skb->_nfct = 0;
NF_CT_STAT_INC_ATOMIC(net, invalid); NF_CT_STAT_INC_ATOMIC(net, invalid);
if (ret == -NF_DROP) if (ret == -NF_DROP)
NF_CT_STAT_INC_ATOMIC(net, drop); NF_CT_STAT_INC_ATOMIC(net, drop);
@ -1878,7 +1878,8 @@ int nf_conntrack_init_start(void)
nf_conntrack_max = max_factor * nf_conntrack_htable_size; nf_conntrack_max = max_factor * nf_conntrack_htable_size;
nf_conntrack_cachep = kmem_cache_create("nf_conntrack", nf_conntrack_cachep = kmem_cache_create("nf_conntrack",
sizeof(struct nf_conn), 0, sizeof(struct nf_conn),
NFCT_INFOMASK + 1,
SLAB_DESTROY_BY_RCU | SLAB_HWCACHE_ALIGN, NULL); SLAB_DESTROY_BY_RCU | SLAB_HWCACHE_ALIGN, NULL);
if (!nf_conntrack_cachep) if (!nf_conntrack_cachep)
goto err_cachep; goto err_cachep;

3
net/netfilter/nf_conntrack_standalone.c
View file

@ -642,6 +642,9 @@ static int __init nf_conntrack_standalone_init(void)
if (ret < 0) if (ret < 0)
goto out_start; goto out_start;
BUILD_BUG_ON(SKB_NFCT_PTRMASK != NFCT_PTRMASK);
BUILD_BUG_ON(NFCT_INFOMASK <= IP_CT_NUMBER);
#ifdef CONFIG_SYSCTL #ifdef CONFIG_SYSCTL
nf_ct_netfilter_header = nf_ct_netfilter_header =
register_net_sysctl(&init_net, "net", nf_ct_netfilter_table); register_net_sysctl(&init_net, "net", nf_ct_netfilter_table);

4
net/netfilter/xt_CT.c
View file

@ -23,7 +23,7 @@
static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct) static inline int xt_ct_target(struct sk_buff *skb, struct nf_conn *ct)
{ {
/* Previously seen (loopback)? Ignore. */ /* Previously seen (loopback)? Ignore. */
if (skb->nfct != NULL) if (skb->_nfct != 0)
return XT_CONTINUE; return XT_CONTINUE;
/* special case the untracked ct : we want the percpu object */ /* special case the untracked ct : we want the percpu object */
@ -409,7 +409,7 @@ static unsigned int
notrack_tg(struct sk_buff *skb, const struct xt_action_param *par) notrack_tg(struct sk_buff *skb, const struct xt_action_param *par)
{ {
/* Previously seen (loopback)? Ignore. */ /* Previously seen (loopback)? Ignore. */
if (skb->nfct != NULL) if (skb->_nfct != 0)
return XT_CONTINUE; return XT_CONTINUE;
nf_ct_set(skb, nf_ct_untracked_get(), IP_CT_NEW); nf_ct_set(skb, nf_ct_untracked_get(), IP_CT_NEW);