mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

i40e: xsk: Move tmp desc array from driver to pool

Browse source 
Move desc_array from the driver to the pool. The reason behind this is
that we can then reuse this array as a temporary storage for descriptors
in all zero-copy drivers that use the batched interface. This will make
it easier to add batching to more drivers.

i40e is the only driver that has a batched Tx zero-copy
implementation, so no need to touch any other driver.

Signed-off-by: Magnus Karlsson <magnus.karlsson@intel.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Alexander Lobakin <alexandr.lobakin@intel.com>
Link: https://lore.kernel.org/bpf/20220125160446.78976-6-maciej.fijalkowski@intel.com
This commit is contained in:
Magnus Karlsson 2022-01-25 17:04:43 +01:00 committed by Daniel Borkmann
parent 3dd411efe1
commit d1bc532e99
8 changed files with 24 additions and 30 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

11
drivers/net/ethernet/intel/i40e/i40e_txrx.c
View file

@ -830,8 +830,6 @@ void i40e_free_tx_resources(struct i40e_ring *tx_ring)
i40e_clean_tx_ring(tx_ring); i40e_clean_tx_ring(tx_ring);
kfree(tx_ring->tx_bi); kfree(tx_ring->tx_bi);
tx_ring->tx_bi = NULL; tx_ring->tx_bi = NULL;
kfree(tx_ring->xsk_descs);
tx_ring->xsk_descs = NULL;
if (tx_ring->desc) { if (tx_ring->desc) {
dma_free_coherent(tx_ring->dev, tx_ring->size, dma_free_coherent(tx_ring->dev, tx_ring->size,
@ -1433,13 +1431,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
if (!tx_ring->tx_bi) if (!tx_ring->tx_bi)
goto err; goto err;
if (ring_is_xdp(tx_ring)) {
tx_ring->xsk_descs = kcalloc(I40E_MAX_NUM_DESCRIPTORS, sizeof(*tx_ring->xsk_descs),
GFP_KERNEL);
if (!tx_ring->xsk_descs)
goto err;
}
u64_stats_init(&tx_ring->syncp); u64_stats_init(&tx_ring->syncp);
/* round up to nearest 4K */ /* round up to nearest 4K */
@ -1463,8 +1454,6 @@ int i40e_setup_tx_descriptors(struct i40e_ring *tx_ring)
return 0; return 0;
err: err:
kfree(tx_ring->xsk_descs);
tx_ring->xsk_descs = NULL;
kfree(tx_ring->tx_bi); kfree(tx_ring->tx_bi);
tx_ring->tx_bi = NULL; tx_ring->tx_bi = NULL;
return -ENOMEM; return -ENOMEM;

1
drivers/net/ethernet/intel/i40e/i40e_txrx.h
View file

@ -390,7 +390,6 @@ struct i40e_ring {
u16 rx_offset; u16 rx_offset;
struct xdp_rxq_info xdp_rxq; struct xdp_rxq_info xdp_rxq;
struct xsk_buff_pool *xsk_pool; struct xsk_buff_pool *xsk_pool;
struct xdp_desc *xsk_descs; /* For storing descriptors in the AF_XDP ZC path */
} ____cacheline_internodealigned_in_smp; } ____cacheline_internodealigned_in_smp;
static inline bool ring_uses_build_skb(struct i40e_ring *ring) static inline bool ring_uses_build_skb(struct i40e_ring *ring)

4
drivers/net/ethernet/intel/i40e/i40e_xsk.c
View file

@ -467,11 +467,11 @@ static void i40e_set_rs_bit(struct i40e_ring *xdp_ring)
**/ **/
static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget) static bool i40e_xmit_zc(struct i40e_ring *xdp_ring, unsigned int budget)
{ {
struct xdp_desc *descs = xdp_ring->xsk_descs; struct xdp_desc *descs = xdp_ring->xsk_pool->tx_descs;
u32 nb_pkts, nb_processed = 0; u32 nb_pkts, nb_processed = 0;
unsigned int total_bytes = 0; unsigned int total_bytes = 0;
nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, descs, budget); nb_pkts = xsk_tx_peek_release_desc_batch(xdp_ring->xsk_pool, budget);
if (!nb_pkts) if (!nb_pkts)
return true; return true;

5
include/net/xdp_sock_drv.h
View file

@ -13,7 +13,7 @@
void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries); void xsk_tx_completed(struct xsk_buff_pool *pool, u32 nb_entries);
bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc); bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc);
u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, u32 max); u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max);
void xsk_tx_release(struct xsk_buff_pool *pool); void xsk_tx_release(struct xsk_buff_pool *pool);
struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev, struct xsk_buff_pool *xsk_get_pool_from_qid(struct net_device *dev,
u16 queue_id); u16 queue_id);
@ -142,8 +142,7 @@ static inline bool xsk_tx_peek_desc(struct xsk_buff_pool *pool,
return false; return false;
} }
static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *desc, static inline u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max)
u32 max)
{ {
return 0; return 0;
} }

1
include/net/xsk_buff_pool.h
View file

@ -60,6 +60,7 @@ struct xsk_buff_pool {
*/ */
dma_addr_t *dma_pages; dma_addr_t *dma_pages;
struct xdp_buff_xsk *heads; struct xdp_buff_xsk *heads;
struct xdp_desc *tx_descs;
u64 chunk_mask; u64 chunk_mask;
u64 addrs_cnt; u64 addrs_cnt;
u32 free_list_cnt; u32 free_list_cnt;

13
net/xdp/xsk.c
View file

@ -343,9 +343,9 @@ bool xsk_tx_peek_desc(struct xsk_buff_pool *pool, struct xdp_desc *desc)
} }
EXPORT_SYMBOL(xsk_tx_peek_desc); EXPORT_SYMBOL(xsk_tx_peek_desc);
static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_desc *descs, static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, u32 max_entries)
u32 max_entries)
{ {
struct xdp_desc *descs = pool->tx_descs;
u32 nb_pkts = 0; u32 nb_pkts = 0;
while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts])) while (nb_pkts < max_entries && xsk_tx_peek_desc(pool, &descs[nb_pkts]))
@ -355,8 +355,7 @@ static u32 xsk_tx_peek_release_fallback(struct xsk_buff_pool *pool, struct xdp_d
return nb_pkts; return nb_pkts;
} }
u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *descs, u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, u32 max_entries)
u32 max_entries)
{ {
struct xdp_sock *xs; struct xdp_sock *xs;
u32 nb_pkts; u32 nb_pkts;
@ -365,7 +364,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
if (!list_is_singular(&pool->xsk_tx_list)) { if (!list_is_singular(&pool->xsk_tx_list)) {
/* Fallback to the non-batched version */ /* Fallback to the non-batched version */
rcu_read_unlock(); rcu_read_unlock();
return xsk_tx_peek_release_fallback(pool, descs, max_entries); return xsk_tx_peek_release_fallback(pool, max_entries);
} }
xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list); xs = list_first_or_null_rcu(&pool->xsk_tx_list, struct xdp_sock, tx_list);
@ -374,7 +373,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
goto out; goto out;
} }
nb_pkts = xskq_cons_peek_desc_batch(xs->tx, descs, pool, max_entries); nb_pkts = xskq_cons_peek_desc_batch(xs->tx, pool, max_entries);
if (!nb_pkts) { if (!nb_pkts) {
xs->tx->queue_empty_descs++; xs->tx->queue_empty_descs++;
goto out; goto out;
@ -386,7 +385,7 @@ u32 xsk_tx_peek_release_desc_batch(struct xsk_buff_pool *pool, struct xdp_desc *
* packets. This avoids having to implement any buffering in * packets. This avoids having to implement any buffering in
* the Tx path. * the Tx path.
*/ */
nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, descs, nb_pkts); nb_pkts = xskq_prod_reserve_addr_batch(pool->cq, pool->tx_descs, nb_pkts);
if (!nb_pkts) if (!nb_pkts)
goto out; goto out;

7
net/xdp/xsk_buff_pool.c
View file

@ -37,6 +37,7 @@ void xp_destroy(struct xsk_buff_pool *pool)
if (!pool) if (!pool)
return; return;
kvfree(pool->tx_descs);
kvfree(pool->heads); kvfree(pool->heads);
kvfree(pool); kvfree(pool);
} }
@ -58,6 +59,12 @@ struct xsk_buff_pool *xp_create_and_assign_umem(struct xdp_sock *xs,
if (!pool->heads) if (!pool->heads)
goto out; goto out;
if (xs->tx) {
pool->tx_descs = kcalloc(xs->tx->nentries, sizeof(*pool->tx_descs), GFP_KERNEL);
if (!pool->tx_descs)
goto out;
}
pool->chunk_mask = ~((u64)umem->chunk_size - 1); pool->chunk_mask = ~((u64)umem->chunk_size - 1);
pool->addrs_cnt = umem->size; pool->addrs_cnt = umem->size;
pool->heads_cnt = umem->chunks; pool->heads_cnt = umem->chunks;

12
net/xdp/xsk_queue.h
View file

@ -205,11 +205,11 @@ static inline bool xskq_cons_read_desc(struct xsk_queue *q,
return false; return false;
} }
static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, static inline u32 xskq_cons_read_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
struct xdp_desc *descs, u32 max)
struct xsk_buff_pool *pool, u32 max)
{ {
u32 cached_cons = q->cached_cons, nb_entries = 0; u32 cached_cons = q->cached_cons, nb_entries = 0;
struct xdp_desc *descs = pool->tx_descs;
while (cached_cons != q->cached_prod && nb_entries < max) { while (cached_cons != q->cached_prod && nb_entries < max) {
struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring; struct xdp_rxtx_ring *ring = (struct xdp_rxtx_ring *)q->ring;
@ -282,12 +282,12 @@ static inline bool xskq_cons_peek_desc(struct xsk_queue *q,
return xskq_cons_read_desc(q, desc, pool); return xskq_cons_read_desc(q, desc, pool);
} }
static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xdp_desc *descs, static inline u32 xskq_cons_peek_desc_batch(struct xsk_queue *q, struct xsk_buff_pool *pool,
struct xsk_buff_pool *pool, u32 max) u32 max)
{ {
u32 entries = xskq_cons_nb_entries(q, max); u32 entries = xskq_cons_nb_entries(q, max);
return xskq_cons_read_desc_batch(q, descs, pool, entries); return xskq_cons_read_desc_batch(q, pool, entries);
} }
/* To improve performance in the xskq_cons_release functions, only update local state here. /* To improve performance in the xskq_cons_release functions, only update local state here.