mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 02:30:34 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

swiotlb: refactor swiotlb_tbl_map_single

Browse source 
Split out a bunch of a self-contained helpers to make the function easier
to follow.

Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Jianxiong Gao <jxgao@google.com>
Tested-by: Jianxiong Gao <jxgao@google.com>
Signed-off-by: Konrad Rzeszutek Wilk <konrad.wilk@oracle.com>
This commit is contained in:
Christoph Hellwig 2021-02-04 11:08:35 +01:00 committed by Konrad Rzeszutek Wilk
parent ca10d0f8e5
commit 26a7e09478
1 changed files with 102 additions and 103 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

203
kernel/dma/swiotlb.c
View file

@ -468,19 +468,99 @@ static void swiotlb_bounce(phys_addr_t orig_addr, phys_addr_t tlb_addr,
} }
} }
phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr, #define slot_addr(start, idx) ((start) + ((idx) << IO_TLB_SHIFT))
/*
* Carefully handle integer overflow which can occur when boundary_mask == ~0UL.
*/
static inline unsigned long get_max_slots(unsigned long boundary_mask)
{
if (boundary_mask == ~0UL)
return 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
return nr_slots(boundary_mask + 1);
}
static unsigned int wrap_index(unsigned int index)
{
if (index >= io_tlb_nslabs)
return 0;
return index;
}
/*
* Find a suitable number of IO TLB entries size that will fit this request and
* allocate a buffer from that IO TLB pool.
*/
static int find_slots(struct device *dev, size_t alloc_size)
{
unsigned long boundary_mask = dma_get_seg_boundary(dev);
dma_addr_t tbl_dma_addr =
phys_to_dma_unencrypted(dev, io_tlb_start) & boundary_mask;
unsigned long max_slots = get_max_slots(boundary_mask);
unsigned int nslots = nr_slots(alloc_size), stride = 1;
unsigned int index, wrap, count = 0, i;
unsigned long flags;
BUG_ON(!nslots);
/*
* For mappings greater than or equal to a page, we limit the stride
* (and hence alignment) to a page size.
*/
if (alloc_size >= PAGE_SIZE)
stride <<= (PAGE_SHIFT - IO_TLB_SHIFT);
spin_lock_irqsave(&io_tlb_lock, flags);
if (unlikely(nslots > io_tlb_nslabs - io_tlb_used))
goto not_found;
index = wrap = wrap_index(ALIGN(io_tlb_index, stride));
do {
/*
* If we find a slot that indicates we have 'nslots' number of
* contiguous buffers, we allocate the buffers from that slot
* and mark the entries as '0' indicating unavailable.
*/
if (!iommu_is_span_boundary(index, nslots,
nr_slots(tbl_dma_addr),
max_slots)) {
if (io_tlb_list[index] >= nslots)
goto found;
}
index = wrap_index(index + stride);
} while (index != wrap);
not_found:
spin_unlock_irqrestore(&io_tlb_lock, flags);
return -1;
found:
for (i = index; i < index + nslots; i++)
io_tlb_list[i] = 0;
for (i = index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
io_tlb_list[i]; i--)
io_tlb_list[i] = ++count;
/*
* Update the indices to avoid searching in the next round.
*/
if (index + nslots < io_tlb_nslabs)
io_tlb_index = index + nslots;
else
io_tlb_index = 0;
io_tlb_used += nslots;
spin_unlock_irqrestore(&io_tlb_lock, flags);
return index;
}
phys_addr_t swiotlb_tbl_map_single(struct device *dev, phys_addr_t orig_addr,
size_t mapping_size, size_t alloc_size, size_t mapping_size, size_t alloc_size,
enum dma_data_direction dir, unsigned long attrs) enum dma_data_direction dir, unsigned long attrs)
{ {
dma_addr_t tbl_dma_addr = phys_to_dma_unencrypted(hwdev, io_tlb_start); unsigned int index, i;
unsigned long flags;
phys_addr_t tlb_addr; phys_addr_t tlb_addr;
unsigned int nslots, stride, index, wrap;
int i;
unsigned long mask;
unsigned long offset_slots;
unsigned long max_slots;
unsigned long tmp_io_tlb_used;
if (no_iotlb_memory) if (no_iotlb_memory)
panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer"); panic("Can not allocate SWIOTLB buffer earlier and can't now provide you with the DMA bounce buffer");
@ -489,113 +569,32 @@ phys_addr_t swiotlb_tbl_map_single(struct device *hwdev, phys_addr_t orig_addr,
pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n"); pr_warn_once("Memory encryption is active and system is using DMA bounce buffers\n");
if (mapping_size > alloc_size) { if (mapping_size > alloc_size) {
dev_warn_once(hwdev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)", dev_warn_once(dev, "Invalid sizes (mapping: %zd bytes, alloc: %zd bytes)",
mapping_size, alloc_size); mapping_size, alloc_size);
return (phys_addr_t)DMA_MAPPING_ERROR; return (phys_addr_t)DMA_MAPPING_ERROR;
} }
mask = dma_get_seg_boundary(hwdev); index = find_slots(dev, alloc_size);
if (index == -1) {
tbl_dma_addr &= mask; if (!(attrs & DMA_ATTR_NO_WARN))
dev_warn_ratelimited(dev,
offset_slots = nr_slots(tbl_dma_addr); "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
alloc_size, io_tlb_nslabs, io_tlb_used);
/*
* Carefully handle integer overflow which can occur when mask == ~0UL.
*/
max_slots = mask + 1
? nr_slots(mask + 1)
: 1UL << (BITS_PER_LONG - IO_TLB_SHIFT);
/*
* For mappings greater than or equal to a page, we limit the stride
* (and hence alignment) to a page size.
*/
nslots = nr_slots(alloc_size);
if (alloc_size >= PAGE_SIZE)
stride = (1 << (PAGE_SHIFT - IO_TLB_SHIFT));
else
stride = 1;
BUG_ON(!nslots);
/*
* Find suitable number of IO TLB entries size that will fit this
* request and allocate a buffer from that IO TLB pool.
*/
spin_lock_irqsave(&io_tlb_lock, flags);
if (unlikely(nslots > io_tlb_nslabs - io_tlb_used))
goto not_found;
index = ALIGN(io_tlb_index, stride);
if (index >= io_tlb_nslabs)
index = 0;
wrap = index;
do {
while (iommu_is_span_boundary(index, nslots, offset_slots,
max_slots)) {
index += stride;
if (index >= io_tlb_nslabs)
index = 0;
if (index == wrap)
goto not_found;
}
/*
* If we find a slot that indicates we have 'nslots' number of
* contiguous buffers, we allocate the buffers from that slot
* and mark the entries as '0' indicating unavailable.
*/
if (io_tlb_list[index] >= nslots) {
int count = 0;
for (i = index; i < (int) (index + nslots); i++)
io_tlb_list[i] = 0;
for (i = index - 1;
io_tlb_offset(i) != IO_TLB_SEGSIZE - 1 &&
io_tlb_list[i]; i--)
io_tlb_list[i] = ++count;
tlb_addr = io_tlb_start + (index << IO_TLB_SHIFT);
/*
* Update the indices to avoid searching in the next
* round.
*/
io_tlb_index = ((index + nslots) < io_tlb_nslabs
? (index + nslots) : 0);
goto found;
}
index += stride;
if (index >= io_tlb_nslabs)
index = 0;
} while (index != wrap);
not_found:
tmp_io_tlb_used = io_tlb_used;
spin_unlock_irqrestore(&io_tlb_lock, flags);
if (!(attrs & DMA_ATTR_NO_WARN) && printk_ratelimit())
dev_warn(hwdev, "swiotlb buffer is full (sz: %zd bytes), total %lu (slots), used %lu (slots)\n",
alloc_size, io_tlb_nslabs, tmp_io_tlb_used);
return (phys_addr_t)DMA_MAPPING_ERROR; return (phys_addr_t)DMA_MAPPING_ERROR;
found: }
io_tlb_used += nslots;
spin_unlock_irqrestore(&io_tlb_lock, flags);
/* /*
* Save away the mapping from the original address to the DMA address. * Save away the mapping from the original address to the DMA address.
* This is needed when we sync the memory. Then we sync the buffer if * This is needed when we sync the memory. Then we sync the buffer if
* needed. * needed.
*/ */
for (i = 0; i < nslots; i++) for (i = 0; i < nr_slots(alloc_size); i++)
io_tlb_orig_addr[index+i] = orig_addr + (i << IO_TLB_SHIFT); io_tlb_orig_addr[index + i] = slot_addr(orig_addr, i);
tlb_addr = slot_addr(io_tlb_start, index);
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL))
swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE); swiotlb_bounce(orig_addr, tlb_addr, mapping_size, DMA_TO_DEVICE);
return tlb_addr; return tlb_addr;
} }