mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

nvme-pci: optimize mapping of small single segment requests

Browse source 
If a request is single segment and fits into one or two PRP entries we
do not have to create a scatterlist for it, but can just map the bio_vec
directly.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Keith Busch <keith.busch@intel.com>
Reviewed-by: Sagi Grimberg <sagi@grimberg.me>
Reviewed-by: Chaitanya Kulkarni <chaitanya.kulkarni@wdc.com>
This commit is contained in:
Christoph Hellwig 2019-03-05 05:49:34 -07:00
parent d43f1ccfad
commit dff824b2aa
1 changed files with 40 additions and 5 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

41
drivers/nvme/host/pci.c
View file

@ -221,6 +221,7 @@ struct nvme_iod {
int npages; /* In the PRP list. 0 means small pool in use */ int npages; /* In the PRP list. 0 means small pool in use */
int nents; /* Used in scatterlist */ int nents; /* Used in scatterlist */
dma_addr_t first_dma; dma_addr_t first_dma;
unsigned int dma_len; /* length of single DMA segment mapping */
dma_addr_t meta_dma; dma_addr_t meta_dma;
struct scatterlist *sg; struct scatterlist *sg;
}; };
@ -576,12 +577,17 @@ static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
dma_addr_t dma_addr = iod->first_dma, next_dma_addr; dma_addr_t dma_addr = iod->first_dma, next_dma_addr;
int i; int i;
if (iod->nents) { if (iod->dma_len) {
dma_unmap_page(dev->dev, dma_addr, iod->dma_len, dma_dir);
return;
}
WARN_ON_ONCE(!iod->nents);
/* P2PDMA requests do not need to be unmapped */ /* P2PDMA requests do not need to be unmapped */
if (!is_pci_p2pdma_page(sg_page(iod->sg))) if (!is_pci_p2pdma_page(sg_page(iod->sg)))
dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir); dma_unmap_sg(dev->dev, iod->sg, iod->nents, rq_dma_dir(req));
}
if (iod->npages == 0) if (iod->npages == 0)
dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0], dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
@ -795,6 +801,24 @@ static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
return BLK_STS_OK; return BLK_STS_OK;
} }
static blk_status_t nvme_setup_prp_simple(struct nvme_dev *dev,
struct request *req, struct nvme_rw_command *cmnd,
struct bio_vec *bv)
{
struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
unsigned int first_prp_len = dev->ctrl.page_size - bv->bv_offset;
iod->first_dma = dma_map_bvec(dev->dev, bv, rq_dma_dir(req), 0);
if (dma_mapping_error(dev->dev, iod->first_dma))
return BLK_STS_RESOURCE;
iod->dma_len = bv->bv_len;
cmnd->dptr.prp1 = cpu_to_le64(iod->first_dma);
if (bv->bv_len > first_prp_len)
cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma + first_prp_len);
return 0;
}
static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req, static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
struct nvme_command *cmnd) struct nvme_command *cmnd)
{ {
@ -805,6 +829,17 @@ static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
blk_status_t ret = BLK_STS_IOERR; blk_status_t ret = BLK_STS_IOERR;
int nr_mapped; int nr_mapped;
if (blk_rq_nr_phys_segments(req) == 1) {
struct bio_vec bv = req_bvec(req);
if (!is_pci_p2pdma_page(bv.bv_page)) {
if (bv.bv_offset + bv.bv_len <= dev->ctrl.page_size * 2)
return nvme_setup_prp_simple(dev, req,
&cmnd->rw, &bv);
}
}
iod->dma_len = 0;
iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC); iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
if (!iod->sg) if (!iod->sg)
return BLK_STS_RESOURCE; return BLK_STS_RESOURCE;