mirror of
https://github.com/torvalds/linux.git
synced 2025-11-04 10:40:15 +02:00
RDMA/mlx5: Move mlx5_ib_cont_pages() to the creation of the mlx5_ib_mr
For the user MR path, instead of calling this after getting the umem, call it as part of creating the struct mlx5_ib_mr and distill its output to a single page_shift stored inside the mr. This avoids passing around the tuple of its output. Based on the umem and page_shift, the output arguments can be computed using: count == ib_umem_num_pages(mr->umem) shift == mr->page_shift ncont == ib_umem_num_dma_blocks(mr->umem, 1 << mr->page_shift) order == order_base_2(ncont) And since mr->page_shift == umem_odp->page_shift then ncont == ib_umem_num_dma_blocks() == ib_umem_odp_num_pages() for ODP umems. Link: https://lore.kernel.org/r/20201026131936.1335664-5-leon@kernel.org Signed-off-by: Leon Romanovsky <leonro@nvidia.com> Signed-off-by: Jason Gunthorpe <jgg@nvidia.com>
This commit is contained in:
Jason Gunthorpe
parent
1c3d247eee
commit
f0093fb1a7
3 changed files with 69 additions and 78 deletions
|
|
@ -57,6 +57,17 @@ void mlx5_ib_cont_pages(struct ib_umem *umem, u64 addr,
|
|||
struct scatterlist *sg;
|
||||
int entry;
|
||||
|
||||
if (umem->is_odp) {
|
||||
struct ib_umem_odp *odp = to_ib_umem_odp(umem);
|
||||
|
||||
*shift = odp->page_shift;
|
||||
*ncont = ib_umem_odp_num_pages(odp);
|
||||
*count = *ncont << (*shift - PAGE_SHIFT);
|
||||
if (order)
|
||||
*order = ilog2(roundup_pow_of_two(*count));
|
||||
return;
|
||||
}
|
||||
|
||||
addr = addr >> PAGE_SHIFT;
|
||||
tmp = (unsigned long)addr;
|
||||
m = find_first_bit(&tmp, BITS_PER_LONG);
|
||||
|
|
|
|||
|
|
@ -597,6 +597,7 @@ struct mlx5_ib_mr {
|
|||
int max_descs;
|
||||
int desc_size;
|
||||
int access_mode;
|
||||
unsigned int page_shift;
|
||||
struct mlx5_core_mkey mmkey;
|
||||
struct ib_umem *umem;
|
||||
struct mlx5_shared_mr_info *smr_info;
|
||||
|
|
|
|||
|
|
@ -868,14 +868,11 @@ static int mr_cache_max_order(struct mlx5_ib_dev *dev)
|
|||
return MLX5_MAX_UMR_SHIFT;
|
||||
}
|
||||
|
||||
static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
|
||||
int access_flags, struct ib_umem **umem, int *npages,
|
||||
int *page_shift, int *ncont, int *order)
|
||||
static struct ib_umem *mr_umem_get(struct mlx5_ib_dev *dev, u64 start,
|
||||
u64 length, int access_flags)
|
||||
{
|
||||
struct ib_umem *u;
|
||||
|
||||
*umem = NULL;
|
||||
|
||||
if (access_flags & IB_ACCESS_ON_DEMAND) {
|
||||
struct ib_umem_odp *odp;
|
||||
|
||||
|
|
@ -884,39 +881,17 @@ static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length,
|
|||
if (IS_ERR(odp)) {
|
||||
mlx5_ib_dbg(dev, "umem get failed (%ld)\n",
|
||||
PTR_ERR(odp));
|
||||
return PTR_ERR(odp);
|
||||
return ERR_CAST(odp);
|
||||
}
|
||||
|
||||
u = &odp->umem;
|
||||
|
||||
*page_shift = odp->page_shift;
|
||||
*ncont = ib_umem_odp_num_pages(odp);
|
||||
*npages = *ncont << (*page_shift - PAGE_SHIFT);
|
||||
if (order)
|
||||
*order = ilog2(roundup_pow_of_two(*ncont));
|
||||
} else {
|
||||
u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
|
||||
if (IS_ERR(u)) {
|
||||
mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
|
||||
return PTR_ERR(u);
|
||||
}
|
||||
|
||||
mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages,
|
||||
page_shift, ncont, order);
|
||||
return &odp->umem;
|
||||
}
|
||||
|
||||
if (!*npages) {
|
||||
mlx5_ib_warn(dev, "avoid zero region\n");
|
||||
ib_umem_release(u);
|
||||
return -EINVAL;
|
||||
u = ib_umem_get(&dev->ib_dev, start, length, access_flags);
|
||||
if (IS_ERR(u)) {
|
||||
mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u));
|
||||
return u;
|
||||
}
|
||||
|
||||
*umem = u;
|
||||
|
||||
mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n",
|
||||
*npages, *ncont, *order, *page_shift);
|
||||
|
||||
return 0;
|
||||
return u;
|
||||
}
|
||||
|
||||
static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc)
|
||||
|
|
@ -975,15 +950,21 @@ static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev,
|
|||
return &cache->ent[order];
|
||||
}
|
||||
|
||||
static struct mlx5_ib_mr *
|
||||
alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
|
||||
u64 len, int npages, int page_shift, unsigned int order,
|
||||
int access_flags)
|
||||
static struct mlx5_ib_mr *alloc_mr_from_cache(struct ib_pd *pd,
|
||||
struct ib_umem *umem, u64 iova,
|
||||
int access_flags)
|
||||
{
|
||||
struct mlx5_ib_dev *dev = to_mdev(pd->device);
|
||||
struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order);
|
||||
struct mlx5_cache_ent *ent;
|
||||
struct mlx5_ib_mr *mr;
|
||||
int npages;
|
||||
int page_shift;
|
||||
int ncont;
|
||||
int order;
|
||||
|
||||
mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
|
||||
&page_shift, &ncont, &order);
|
||||
ent = mr_cache_ent_from_order(dev, order);
|
||||
if (!ent)
|
||||
return ERR_PTR(-E2BIG);
|
||||
|
||||
|
|
@ -1002,9 +983,10 @@ alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr,
|
|||
mr->umem = umem;
|
||||
mr->access_flags = access_flags;
|
||||
mr->desc_size = sizeof(struct mlx5_mtt);
|
||||
mr->mmkey.iova = virt_addr;
|
||||
mr->mmkey.size = len;
|
||||
mr->mmkey.iova = iova;
|
||||
mr->mmkey.size = umem->length;
|
||||
mr->mmkey.pd = to_mpd(pd)->pdn;
|
||||
mr->page_shift = page_shift;
|
||||
|
||||
return mr;
|
||||
}
|
||||
|
|
@ -1163,14 +1145,15 @@ int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages,
|
|||
* Else, the given ibmr will be used.
|
||||
*/
|
||||
static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
|
||||
u64 virt_addr, u64 length,
|
||||
struct ib_umem *umem, int npages,
|
||||
int page_shift, int access_flags,
|
||||
bool populate)
|
||||
struct ib_umem *umem, u64 iova,
|
||||
int access_flags, bool populate)
|
||||
{
|
||||
struct mlx5_ib_dev *dev = to_mdev(pd->device);
|
||||
struct mlx5_ib_mr *mr;
|
||||
int page_shift;
|
||||
__be64 *pas;
|
||||
int npages;
|
||||
int ncont;
|
||||
void *mkc;
|
||||
int inlen;
|
||||
u32 *in;
|
||||
|
|
@ -1181,6 +1164,10 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
|
|||
if (!mr)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
|
||||
mlx5_ib_cont_pages(umem, iova, MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
|
||||
&page_shift, &ncont, NULL);
|
||||
|
||||
mr->page_shift = page_shift;
|
||||
mr->ibmr.pd = pd;
|
||||
mr->access_flags = access_flags;
|
||||
|
||||
|
|
@ -1207,20 +1194,20 @@ static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd,
|
|||
MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap));
|
||||
|
||||
mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry);
|
||||
set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr,
|
||||
set_mkc_access_pd_addr_fields(mkc, access_flags, iova,
|
||||
populate ? pd : dev->umrc.pd);
|
||||
MLX5_SET(mkc, mkc, free, !populate);
|
||||
MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT);
|
||||
MLX5_SET(mkc, mkc, umr_en, 1);
|
||||
|
||||
MLX5_SET64(mkc, mkc, len, length);
|
||||
MLX5_SET64(mkc, mkc, len, umem->length);
|
||||
MLX5_SET(mkc, mkc, bsf_octword_size, 0);
|
||||
MLX5_SET(mkc, mkc, translations_octword_size,
|
||||
get_octo_len(virt_addr, length, page_shift));
|
||||
get_octo_len(iova, umem->length, page_shift));
|
||||
MLX5_SET(mkc, mkc, log_page_size, page_shift);
|
||||
if (populate) {
|
||||
MLX5_SET(create_mkey_in, in, translations_octword_actual_size,
|
||||
get_octo_len(virt_addr, length, page_shift));
|
||||
get_octo_len(iova, umem->length, page_shift));
|
||||
}
|
||||
|
||||
err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen);
|
||||
|
|
@ -1358,10 +1345,6 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
|||
struct mlx5_ib_mr *mr = NULL;
|
||||
bool xlt_with_umr;
|
||||
struct ib_umem *umem;
|
||||
int page_shift;
|
||||
int npages;
|
||||
int ncont;
|
||||
int order;
|
||||
int err;
|
||||
|
||||
if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM))
|
||||
|
|
@ -1389,23 +1372,20 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
|||
return &mr->ibmr;
|
||||
}
|
||||
|
||||
err = mr_umem_get(dev, start, length, access_flags, &umem,
|
||||
&npages, &page_shift, &ncont, &order);
|
||||
|
||||
if (err < 0)
|
||||
return ERR_PTR(err);
|
||||
umem = mr_umem_get(dev, start, length, access_flags);
|
||||
if (IS_ERR(umem))
|
||||
return ERR_CAST(umem);
|
||||
|
||||
if (xlt_with_umr) {
|
||||
mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont,
|
||||
page_shift, order, access_flags);
|
||||
mr = alloc_mr_from_cache(pd, umem, virt_addr, access_flags);
|
||||
if (IS_ERR(mr))
|
||||
mr = NULL;
|
||||
}
|
||||
|
||||
if (!mr) {
|
||||
mutex_lock(&dev->slow_path_mutex);
|
||||
mr = reg_create(NULL, pd, virt_addr, length, umem, ncont,
|
||||
page_shift, access_flags, !xlt_with_umr);
|
||||
mr = reg_create(NULL, pd, umem, virt_addr, access_flags,
|
||||
!xlt_with_umr);
|
||||
mutex_unlock(&dev->slow_path_mutex);
|
||||
}
|
||||
|
||||
|
|
@ -1428,8 +1408,10 @@ struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length,
|
|||
*/
|
||||
int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE;
|
||||
|
||||
err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift,
|
||||
update_xlt_flags);
|
||||
err = mlx5_ib_update_xlt(
|
||||
mr, 0,
|
||||
ib_umem_num_dma_blocks(umem, 1UL << mr->page_shift),
|
||||
mr->page_shift, update_xlt_flags);
|
||||
if (err) {
|
||||
dereg_mr(dev, mr);
|
||||
return ERR_PTR(err);
|
||||
|
|
@ -1519,11 +1501,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
|
|||
int access_flags = flags & IB_MR_REREG_ACCESS ?
|
||||
new_access_flags :
|
||||
mr->access_flags;
|
||||
int page_shift = 0;
|
||||
int upd_flags = 0;
|
||||
int npages = 0;
|
||||
int ncont = 0;
|
||||
int order = 0;
|
||||
u64 addr, len;
|
||||
int err;
|
||||
|
||||
|
|
@ -1553,12 +1531,12 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
|
|||
atomic_sub(ib_umem_num_pages(mr->umem),
|
||||
&dev->mdev->priv.reg_pages);
|
||||
ib_umem_release(mr->umem);
|
||||
mr->umem = NULL;
|
||||
|
||||
err = mr_umem_get(dev, addr, len, access_flags, &mr->umem,
|
||||
&npages, &page_shift, &ncont, &order);
|
||||
if (err)
|
||||
mr->umem = mr_umem_get(dev, addr, len, access_flags);
|
||||
if (IS_ERR(mr->umem)) {
|
||||
err = PTR_ERR(mr->umem);
|
||||
mr->umem = NULL;
|
||||
goto err;
|
||||
}
|
||||
atomic_add(ib_umem_num_pages(mr->umem),
|
||||
&dev->mdev->priv.reg_pages);
|
||||
}
|
||||
|
|
@ -1577,9 +1555,7 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
|
|||
if (err)
|
||||
goto err;
|
||||
|
||||
mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont,
|
||||
page_shift, access_flags, true);
|
||||
|
||||
mr = reg_create(ib_mr, pd, mr->umem, addr, access_flags, true);
|
||||
if (IS_ERR(mr)) {
|
||||
err = PTR_ERR(mr);
|
||||
mr = to_mmr(ib_mr);
|
||||
|
|
@ -1601,8 +1577,11 @@ int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start,
|
|||
upd_flags |= MLX5_IB_UPD_XLT_PD;
|
||||
if (flags & IB_MR_REREG_ACCESS)
|
||||
upd_flags |= MLX5_IB_UPD_XLT_ACCESS;
|
||||
err = mlx5_ib_update_xlt(mr, 0, npages, page_shift,
|
||||
upd_flags);
|
||||
err = mlx5_ib_update_xlt(
|
||||
mr, 0,
|
||||
ib_umem_num_dma_blocks(mr->umem,
|
||||
1UL << mr->page_shift),
|
||||
mr->page_shift, upd_flags);
|
||||
} else {
|
||||
err = rereg_umr(pd, mr, access_flags, flags);
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue