nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

iommu/vt-d: Convert intel iommu driver to the iommu ops

Browse source 
Convert the intel iommu driver to the dma-iommu api. Remove the iova
handling and reserve region code from the intel iommu driver.

Signed-off-by: Tom Murphy <murphyt7@tcd.ie>
Signed-off-by: Lu Baolu <baolu.lu@linux.intel.com>
Tested-by: Logan Gunthorpe <logang@deltatee.com>
Link: https://lore.kernel.org/r/20201124082057.2614359-7-baolu.lu@linux.intel.com
Signed-off-by: Will Deacon <will@kernel.org>
This commit is contained in:
Tom Murphy 2020-11-24 16:20:56 +08:00 committed by Will Deacon
parent c062db039f
commit c588072bba
2 changed files with 43 additions and 703 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

1
drivers/iommu/intel/Kconfig
View file

@ -13,6 +13,7 @@ config INTEL_IOMMU
select DMAR_TABLE select DMAR_TABLE
select SWIOTLB select SWIOTLB
select IOASID select IOASID
select IOMMU_DMA
help help
DMA remapping (DMAR) devices support enables independent address DMA remapping (DMAR) devices support enables independent address
translations for Direct Memory Access (DMA) from devices. translations for Direct Memory Access (DMA) from devices.

745
drivers/iommu/intel/iommu.c
View file

@ -31,6 +31,7 @@
#include <linux/io.h> #include <linux/io.h>
#include <linux/iova.h> #include <linux/iova.h>
#include <linux/iommu.h> #include <linux/iommu.h>
#include <linux/dma-iommu.h>
#include <linux/intel-iommu.h> #include <linux/intel-iommu.h>
#include <linux/syscore_ops.h> #include <linux/syscore_ops.h>
#include <linux/tboot.h> #include <linux/tboot.h>
@ -41,7 +42,6 @@
#include <linux/dma-direct.h> #include <linux/dma-direct.h>
#include <linux/crash_dump.h> #include <linux/crash_dump.h>
#include <linux/numa.h> #include <linux/numa.h>
#include <linux/swiotlb.h>
#include <asm/irq_remapping.h> #include <asm/irq_remapping.h>
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/iommu.h> #include <asm/iommu.h>
@ -382,9 +382,6 @@ struct device_domain_info *get_domain_info(struct device *dev)
DEFINE_SPINLOCK(device_domain_lock); DEFINE_SPINLOCK(device_domain_lock);
static LIST_HEAD(device_domain_list); static LIST_HEAD(device_domain_list);
#define device_needs_bounce(d) (!intel_no_bounce && dev_is_pci(d) && \
to_pci_dev(d)->untrusted)
/* /*
* Iterate over elements in device_domain_list and call the specified * Iterate over elements in device_domain_list and call the specified
* callback @fn against each element. * callback @fn against each element.
@ -1289,13 +1286,6 @@ static void dma_free_pagelist(struct page *freelist)
} }
} }
static void iova_entry_free(unsigned long data)
{
struct page *freelist = (struct page *)data;
dma_free_pagelist(freelist);
}
/* iommu handling */ /* iommu handling */
static int iommu_alloc_root_entry(struct intel_iommu *iommu) static int iommu_alloc_root_entry(struct intel_iommu *iommu)
{ {
@ -1660,19 +1650,17 @@ static inline void __mapping_notify_one(struct intel_iommu *iommu,
iommu_flush_write_buffer(iommu); iommu_flush_write_buffer(iommu);
} }
static void iommu_flush_iova(struct iova_domain *iovad) static void intel_flush_iotlb_all(struct iommu_domain *domain)
{ {
struct dmar_domain *domain; struct dmar_domain *dmar_domain = to_dmar_domain(domain);
int idx; int idx;
domain = container_of(iovad, struct dmar_domain, iovad); for_each_domain_iommu(idx, dmar_domain) {
for_each_domain_iommu(idx, domain) {
struct intel_iommu *iommu = g_iommus[idx]; struct intel_iommu *iommu = g_iommus[idx];
u16 did = domain->iommu_did[iommu->seq_id]; u16 did = dmar_domain->iommu_did[iommu->seq_id];
if (domain_use_first_level(domain)) if (domain_use_first_level(dmar_domain))
domain_flush_piotlb(iommu, domain, 0, -1, 0); domain_flush_piotlb(iommu, dmar_domain, 0, -1, 0);
else else
iommu->flush.flush_iotlb(iommu, did, 0, 0, iommu->flush.flush_iotlb(iommu, did, 0, 0,
DMA_TLB_DSI_FLUSH); DMA_TLB_DSI_FLUSH);
@ -1954,48 +1942,6 @@ static int domain_detach_iommu(struct dmar_domain *domain,
return count; return count;
} }
static struct iova_domain reserved_iova_list;
static struct lock_class_key reserved_rbtree_key;
static int dmar_init_reserved_ranges(void)
{
struct pci_dev *pdev = NULL;
struct iova *iova;
int i;
init_iova_domain(&reserved_iova_list, VTD_PAGE_SIZE, IOVA_START_PFN);
lockdep_set_class(&reserved_iova_list.iova_rbtree_lock,
&reserved_rbtree_key);
/* IOAPIC ranges shouldn't be accessed by DMA */
iova = reserve_iova(&reserved_iova_list, IOVA_PFN(IOAPIC_RANGE_START),
IOVA_PFN(IOAPIC_RANGE_END));
if (!iova) {
pr_err("Reserve IOAPIC range failed\n");
return -ENODEV;
}
/* Reserve all PCI MMIO to avoid peer-to-peer access */
for_each_pci_dev(pdev) {
struct resource *r;
for (i = 0; i < PCI_NUM_RESOURCES; i++) {
r = &pdev->resource[i];
if (!r->flags || !(r->flags & IORESOURCE_MEM))
continue;
iova = reserve_iova(&reserved_iova_list,
IOVA_PFN(r->start),
IOVA_PFN(r->end));
if (!iova) {
pci_err(pdev, "Reserve iova for %pR failed\n", r);
return -ENODEV;
}
}
}
return 0;
}
static inline int guestwidth_to_adjustwidth(int gaw) static inline int guestwidth_to_adjustwidth(int gaw)
{ {
int agaw; int agaw;
@ -2018,7 +1964,7 @@ static void domain_exit(struct dmar_domain *domain)
/* destroy iovas */ /* destroy iovas */
if (domain->domain.type == IOMMU_DOMAIN_DMA) if (domain->domain.type == IOMMU_DOMAIN_DMA)
put_iova_domain(&domain->iovad); iommu_put_dma_cookie(&domain->domain);
if (domain->pgd) { if (domain->pgd) {
struct page *freelist; struct page *freelist;
@ -2552,16 +2498,6 @@ struct dmar_domain *find_domain(struct device *dev)
return NULL; return NULL;
} }
static void do_deferred_attach(struct device *dev)
{
struct iommu_domain *domain;
dev_iommu_priv_set(dev, NULL);
domain = iommu_get_domain_for_dev(dev);
if (domain)
intel_iommu_attach_device(domain, dev);
}
static inline struct device_domain_info * static inline struct device_domain_info *
dmar_search_domain_by_dev_info(int segment, int bus, int devfn) dmar_search_domain_by_dev_info(int segment, int bus, int devfn)
{ {
@ -3434,594 +3370,6 @@ static int __init init_dmars(void)
return ret; return ret;
} }
/* This takes a number of _MM_ pages, not VTD pages */
static unsigned long intel_alloc_iova(struct device *dev,
struct dmar_domain *domain,
unsigned long nrpages, uint64_t dma_mask)
{
unsigned long iova_pfn;
/*
* Restrict dma_mask to the width that the iommu can handle.
* First-level translation restricts the input-address to a
* canonical address (i.e., address bits 63:N have the same
* value as address bit [N-1], where N is 48-bits with 4-level
* paging and 57-bits with 5-level paging). Hence, skip bit
* [N-1].
*/
if (domain_use_first_level(domain))
dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw - 1),
dma_mask);
else
dma_mask = min_t(uint64_t, DOMAIN_MAX_ADDR(domain->gaw),
dma_mask);
/* Ensure we reserve the whole size-aligned region */
nrpages = __roundup_pow_of_two(nrpages);
if (!dmar_forcedac && dma_mask > DMA_BIT_MASK(32)) {
/*
* First try to allocate an io virtual address in
* DMA_BIT_MASK(32) and if that fails then try allocating
* from higher range
*/
iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
IOVA_PFN(DMA_BIT_MASK(32)), false);
if (iova_pfn)
return iova_pfn;
}
iova_pfn = alloc_iova_fast(&domain->iovad, nrpages,
IOVA_PFN(dma_mask), true);
if (unlikely(!iova_pfn)) {
dev_err_once(dev, "Allocating %ld-page iova failed\n",
nrpages);
return 0;
}
return iova_pfn;
}
static dma_addr_t __intel_map_single(struct device *dev, phys_addr_t paddr,
size_t size, int dir, u64 dma_mask)
{
struct dmar_domain *domain;
phys_addr_t start_paddr;
unsigned long iova_pfn;
int prot = 0;
int ret;
struct intel_iommu *iommu;
unsigned long paddr_pfn = paddr >> PAGE_SHIFT;
BUG_ON(dir == DMA_NONE);
if (unlikely(attach_deferred(dev)))
do_deferred_attach(dev);
domain = find_domain(dev);
if (!domain)
return DMA_MAPPING_ERROR;
iommu = domain_get_iommu(domain);
size = aligned_nrpages(paddr, size);
iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size), dma_mask);
if (!iova_pfn)
goto error;
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
*/
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
!cap_zlr(iommu->cap))
prot |= DMA_PTE_READ;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
/*
* paddr - (paddr + size) might be partial page, we should map the whole
* page. Note: if two part of one page are separately mapped, we
* might have two guest_addr mapping to the same host paddr, but this
* is not a big problem
*/
ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
mm_to_dma_pfn(paddr_pfn), size, prot);
if (ret)
goto error;
start_paddr = (phys_addr_t)iova_pfn << PAGE_SHIFT;
start_paddr += paddr & ~PAGE_MASK;
trace_map_single(dev, start_paddr, paddr, size << VTD_PAGE_SHIFT);
return start_paddr;
error:
if (iova_pfn)
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
dev_err(dev, "Device request: %zx@%llx dir %d --- failed\n",
size, (unsigned long long)paddr, dir);
return DMA_MAPPING_ERROR;
}
static dma_addr_t intel_map_page(struct device *dev, struct page *page,
unsigned long offset, size_t size,
enum dma_data_direction dir,
unsigned long attrs)
{
return __intel_map_single(dev, page_to_phys(page) + offset,
size, dir, *dev->dma_mask);
}
static dma_addr_t intel_map_resource(struct device *dev, phys_addr_t phys_addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
return __intel_map_single(dev, phys_addr, size, dir, *dev->dma_mask);
}
static void intel_unmap(struct device *dev, dma_addr_t dev_addr, size_t size)
{
struct dmar_domain *domain;
unsigned long start_pfn, last_pfn;
unsigned long nrpages;
unsigned long iova_pfn;
struct intel_iommu *iommu;
struct page *freelist;
struct pci_dev *pdev = NULL;
domain = find_domain(dev);
BUG_ON(!domain);
iommu = domain_get_iommu(domain);
iova_pfn = IOVA_PFN(dev_addr);
nrpages = aligned_nrpages(dev_addr, size);
start_pfn = mm_to_dma_pfn(iova_pfn);
last_pfn = start_pfn + nrpages - 1;
if (dev_is_pci(dev))
pdev = to_pci_dev(dev);
freelist = domain_unmap(domain, start_pfn, last_pfn, NULL);
if (intel_iommu_strict || (pdev && pdev->untrusted) ||
!has_iova_flush_queue(&domain->iovad)) {
iommu_flush_iotlb_psi(iommu, domain, start_pfn,
nrpages, !freelist, 0);
/* free iova */
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
dma_free_pagelist(freelist);
} else {
queue_iova(&domain->iovad, iova_pfn, nrpages,
(unsigned long)freelist);
/*
* queue up the release of the unmap to save the 1/6th of the
* cpu used up by the iotlb flush operation...
*/
}
trace_unmap_single(dev, dev_addr, size);
}
static void intel_unmap_page(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir,
unsigned long attrs)
{
intel_unmap(dev, dev_addr, size);
}
static void intel_unmap_resource(struct device *dev, dma_addr_t dev_addr,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
intel_unmap(dev, dev_addr, size);
}
static void *intel_alloc_coherent(struct device *dev, size_t size,
dma_addr_t *dma_handle, gfp_t flags,
unsigned long attrs)
{
struct page *page = NULL;
int order;
if (unlikely(attach_deferred(dev)))
do_deferred_attach(dev);
size = PAGE_ALIGN(size);
order = get_order(size);
if (gfpflags_allow_blocking(flags)) {
unsigned int count = size >> PAGE_SHIFT;
page = dma_alloc_from_contiguous(dev, count, order,
flags & __GFP_NOWARN);
}
if (!page)
page = alloc_pages(flags, order);
if (!page)
return NULL;
memset(page_address(page), 0, size);
*dma_handle = __intel_map_single(dev, page_to_phys(page), size,
DMA_BIDIRECTIONAL,
dev->coherent_dma_mask);
if (*dma_handle != DMA_MAPPING_ERROR)
return page_address(page);
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, order);
return NULL;
}
static void intel_free_coherent(struct device *dev, size_t size, void *vaddr,
dma_addr_t dma_handle, unsigned long attrs)
{
int order;
struct page *page = virt_to_page(vaddr);
size = PAGE_ALIGN(size);
order = get_order(size);
intel_unmap(dev, dma_handle, size);
if (!dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT))
__free_pages(page, order);
}
static void intel_unmap_sg(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir,
unsigned long attrs)
{
dma_addr_t startaddr = sg_dma_address(sglist) & PAGE_MASK;
unsigned long nrpages = 0;
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i) {
nrpages += aligned_nrpages(sg_dma_address(sg), sg_dma_len(sg));
}
intel_unmap(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
trace_unmap_sg(dev, startaddr, nrpages << VTD_PAGE_SHIFT);
}
static int intel_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, unsigned long attrs)
{
int i;
struct dmar_domain *domain;
size_t size = 0;
int prot = 0;
unsigned long iova_pfn;
int ret;
struct scatterlist *sg;
unsigned long start_vpfn;
struct intel_iommu *iommu;
BUG_ON(dir == DMA_NONE);
if (unlikely(attach_deferred(dev)))
do_deferred_attach(dev);
domain = find_domain(dev);
if (!domain)
return 0;
iommu = domain_get_iommu(domain);
for_each_sg(sglist, sg, nelems, i)
size += aligned_nrpages(sg->offset, sg->length);
iova_pfn = intel_alloc_iova(dev, domain, dma_to_mm_pfn(size),
*dev->dma_mask);
if (!iova_pfn) {
sglist->dma_length = 0;
return 0;
}
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
*/
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL || \
!cap_zlr(iommu->cap))
prot |= DMA_PTE_READ;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
start_vpfn = mm_to_dma_pfn(iova_pfn);
ret = domain_sg_mapping(domain, start_vpfn, sglist, size, prot);
if (unlikely(ret)) {
dma_pte_free_pagetable(domain, start_vpfn,
start_vpfn + size - 1,
agaw_to_level(domain->agaw) + 1);
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(size));
return 0;
}
for_each_sg(sglist, sg, nelems, i)
trace_map_sg(dev, i + 1, nelems, sg);
return nelems;
}
static u64 intel_get_required_mask(struct device *dev)
{
return DMA_BIT_MASK(32);
}
static const struct dma_map_ops intel_dma_ops = {
.alloc = intel_alloc_coherent,
.free = intel_free_coherent,
.map_sg = intel_map_sg,
.unmap_sg = intel_unmap_sg,
.map_page = intel_map_page,
.unmap_page = intel_unmap_page,
.map_resource = intel_map_resource,
.unmap_resource = intel_unmap_resource,
.dma_supported = dma_direct_supported,
.mmap = dma_common_mmap,
.get_sgtable = dma_common_get_sgtable,
.alloc_pages = dma_common_alloc_pages,
.free_pages = dma_common_free_pages,
.get_required_mask = intel_get_required_mask,
};
static void
bounce_sync_single(struct device *dev, dma_addr_t addr, size_t size,
enum dma_data_direction dir, enum dma_sync_target target)
{
struct dmar_domain *domain;
phys_addr_t tlb_addr;
domain = find_domain(dev);
if (WARN_ON(!domain))
return;
tlb_addr = intel_iommu_iova_to_phys(&domain->domain, addr);
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_sync_single(dev, tlb_addr, size, dir, target);
}
static dma_addr_t
bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size,
enum dma_data_direction dir, unsigned long attrs,
u64 dma_mask)
{
size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
struct dmar_domain *domain;
struct intel_iommu *iommu;
unsigned long iova_pfn;
unsigned long nrpages;
phys_addr_t tlb_addr;
int prot = 0;
int ret;
if (unlikely(attach_deferred(dev)))
do_deferred_attach(dev);
domain = find_domain(dev);
if (WARN_ON(dir == DMA_NONE || !domain))
return DMA_MAPPING_ERROR;
iommu = domain_get_iommu(domain);
if (WARN_ON(!iommu))
return DMA_MAPPING_ERROR;
nrpages = aligned_nrpages(0, size);
iova_pfn = intel_alloc_iova(dev, domain,
dma_to_mm_pfn(nrpages), dma_mask);
if (!iova_pfn)
return DMA_MAPPING_ERROR;
/*
* Check if DMAR supports zero-length reads on write only
* mappings..
*/
if (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL ||
!cap_zlr(iommu->cap))
prot |= DMA_PTE_READ;
if (dir == DMA_FROM_DEVICE || dir == DMA_BIDIRECTIONAL)
prot |= DMA_PTE_WRITE;
/*
* If both the physical buffer start address and size are
* page aligned, we don't need to use a bounce page.
*/
if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) {
tlb_addr = swiotlb_tbl_map_single(dev, paddr, size,
aligned_size, dir, attrs);
if (tlb_addr == DMA_MAPPING_ERROR) {
goto swiotlb_error;
} else {
/* Cleanup the padding area. */
void *padding_start = phys_to_virt(tlb_addr);
size_t padding_size = aligned_size;
if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
(dir == DMA_TO_DEVICE ||
dir == DMA_BIDIRECTIONAL)) {
padding_start += size;
padding_size -= size;
}
memset(padding_start, 0, padding_size);
}
} else {
tlb_addr = paddr;
}
ret = domain_pfn_mapping(domain, mm_to_dma_pfn(iova_pfn),
tlb_addr >> VTD_PAGE_SHIFT, nrpages, prot);
if (ret)
goto mapping_error;
trace_bounce_map_single(dev, iova_pfn << PAGE_SHIFT, paddr, size);
return (phys_addr_t)iova_pfn << PAGE_SHIFT;
mapping_error:
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_unmap_single(dev, tlb_addr, size,
aligned_size, dir, attrs);
swiotlb_error:
free_iova_fast(&domain->iovad, iova_pfn, dma_to_mm_pfn(nrpages));
dev_err(dev, "Device bounce map: %zx@%llx dir %d --- failed\n",
size, (unsigned long long)paddr, dir);
return DMA_MAPPING_ERROR;
}
static void
bounce_unmap_single(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
size_t aligned_size = ALIGN(size, VTD_PAGE_SIZE);
struct dmar_domain *domain;
phys_addr_t tlb_addr;
domain = find_domain(dev);
if (WARN_ON(!domain))
return;
tlb_addr = intel_iommu_iova_to_phys(&domain->domain, dev_addr);
if (WARN_ON(!tlb_addr))
return;
intel_unmap(dev, dev_addr, size);
if (is_swiotlb_buffer(tlb_addr))
swiotlb_tbl_unmap_single(dev, tlb_addr, size,
aligned_size, dir, attrs);
trace_bounce_unmap_single(dev, dev_addr, size);
}
static dma_addr_t
bounce_map_page(struct device *dev, struct page *page, unsigned long offset,
size_t size, enum dma_data_direction dir, unsigned long attrs)
{
return bounce_map_single(dev, page_to_phys(page) + offset,
size, dir, attrs, *dev->dma_mask);
}
static dma_addr_t
bounce_map_resource(struct device *dev, phys_addr_t phys_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
return bounce_map_single(dev, phys_addr, size,
dir, attrs, *dev->dma_mask);
}
static void
bounce_unmap_page(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
bounce_unmap_single(dev, dev_addr, size, dir, attrs);
}
static void
bounce_unmap_resource(struct device *dev, dma_addr_t dev_addr, size_t size,
enum dma_data_direction dir, unsigned long attrs)
{
bounce_unmap_single(dev, dev_addr, size, dir, attrs);
}
static void
bounce_unmap_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, unsigned long attrs)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_unmap_page(dev, sg->dma_address,
sg_dma_len(sg), dir, attrs);
}
static int
bounce_map_sg(struct device *dev, struct scatterlist *sglist, int nelems,
enum dma_data_direction dir, unsigned long attrs)
{
int i;
struct scatterlist *sg;
for_each_sg(sglist, sg, nelems, i) {
sg->dma_address = bounce_map_page(dev, sg_page(sg),
sg->offset, sg->length,
dir, attrs);
if (sg->dma_address == DMA_MAPPING_ERROR)
goto out_unmap;
sg_dma_len(sg) = sg->length;
}
for_each_sg(sglist, sg, nelems, i)
trace_bounce_map_sg(dev, i + 1, nelems, sg);
return nelems;
out_unmap:
bounce_unmap_sg(dev, sglist, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
return 0;
}
static void
bounce_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir)
{
bounce_sync_single(dev, addr, size, dir, SYNC_FOR_CPU);
}
static void
bounce_sync_single_for_device(struct device *dev, dma_addr_t addr,
size_t size, enum dma_data_direction dir)
{
bounce_sync_single(dev, addr, size, dir, SYNC_FOR_DEVICE);
}
static void
bounce_sync_sg_for_cpu(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_sync_single(dev, sg_dma_address(sg),
sg_dma_len(sg), dir, SYNC_FOR_CPU);
}
static void
bounce_sync_sg_for_device(struct device *dev, struct scatterlist *sglist,
int nelems, enum dma_data_direction dir)
{
struct scatterlist *sg;
int i;
for_each_sg(sglist, sg, nelems, i)
bounce_sync_single(dev, sg_dma_address(sg),
sg_dma_len(sg), dir, SYNC_FOR_DEVICE);
}
static const struct dma_map_ops bounce_dma_ops = {
.alloc = intel_alloc_coherent,
.free = intel_free_coherent,
.map_sg = bounce_map_sg,
.unmap_sg = bounce_unmap_sg,
.map_page = bounce_map_page,
.unmap_page = bounce_unmap_page,
.sync_single_for_cpu = bounce_sync_single_for_cpu,
.sync_single_for_device = bounce_sync_single_for_device,
.sync_sg_for_cpu = bounce_sync_sg_for_cpu,
.sync_sg_for_device = bounce_sync_sg_for_device,
.map_resource = bounce_map_resource,
.unmap_resource = bounce_unmap_resource,
.alloc_pages = dma_common_alloc_pages,
.free_pages = dma_common_free_pages,
.dma_supported = dma_direct_supported,
};
static inline int iommu_domain_cache_init(void) static inline int iommu_domain_cache_init(void)
{ {
int ret = 0; int ret = 0;
@ -4689,7 +4037,7 @@ static void free_all_cpu_cached_iovas(unsigned int cpu)
if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA) if (!domain || domain->domain.type != IOMMU_DOMAIN_DMA)
continue; continue;
free_cpu_cached_iovas(cpu, &domain->iovad); iommu_dma_free_cpu_cached_iovas(cpu, &domain->domain);
} }
} }
} }
@ -4961,12 +4309,6 @@ int __init intel_iommu_init(void)
if (list_empty(&dmar_atsr_units)) if (list_empty(&dmar_atsr_units))
pr_info("No ATSR found\n"); pr_info("No ATSR found\n");
if (dmar_init_reserved_ranges()) {
if (force_on)
panic("tboot: Failed to reserve iommu ranges\n");
goto out_free_reserved_range;
}
if (dmar_map_gfx) if (dmar_map_gfx)
intel_iommu_gfx_mapped = 1; intel_iommu_gfx_mapped = 1;
@ -4977,7 +4319,7 @@ int __init intel_iommu_init(void)
if (force_on) if (force_on)
panic("tboot: Failed to initialize DMARs\n"); panic("tboot: Failed to initialize DMARs\n");
pr_err("Initialization failed\n"); pr_err("Initialization failed\n");
goto out_free_reserved_range; goto out_free_dmar;
} }
up_write(&dmar_global_lock); up_write(&dmar_global_lock);
@ -5018,8 +4360,6 @@ int __init intel_iommu_init(void)
return 0; return 0;
out_free_reserved_range:
put_iova_domain(&reserved_iova_list);
out_free_dmar: out_free_dmar:
intel_iommu_free_dmars(); intel_iommu_free_dmars();
up_write(&dmar_global_lock); up_write(&dmar_global_lock);
@ -5117,17 +4457,6 @@ static int md_domain_init(struct dmar_domain *domain, int guest_width)
return 0; return 0;
} }
static void intel_init_iova_domain(struct dmar_domain *dmar_domain)
{
init_iova_domain(&dmar_domain->iovad, VTD_PAGE_SIZE, IOVA_START_PFN);
copy_reserved_iova(&reserved_iova_list, &dmar_domain->iovad);
if (!intel_iommu_strict &&
init_iova_flush_queue(&dmar_domain->iovad,
iommu_flush_iova, iova_entry_free))
pr_info("iova flush queue initialization failed\n");
}
static struct iommu_domain *intel_iommu_domain_alloc(unsigned type) static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
{ {
struct dmar_domain *dmar_domain; struct dmar_domain *dmar_domain;
@ -5147,8 +4476,9 @@ static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
return NULL; return NULL;
} }
if (type == IOMMU_DOMAIN_DMA) if (type == IOMMU_DOMAIN_DMA &&
intel_init_iova_domain(dmar_domain); iommu_get_dma_cookie(&dmar_domain->domain))
return NULL;
domain = &dmar_domain->domain; domain = &dmar_domain->domain;
domain->geometry.aperture_start = 0; domain->geometry.aperture_start = 0;
@ -5777,13 +5107,13 @@ static void intel_iommu_release_device(struct device *dev)
static void intel_iommu_probe_finalize(struct device *dev) static void intel_iommu_probe_finalize(struct device *dev)
{ {
struct iommu_domain *domain; dma_addr_t base = IOVA_START_PFN << VTD_PAGE_SHIFT;
struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
domain = iommu_get_domain_for_dev(dev); if (domain && domain->type == IOMMU_DOMAIN_DMA)
if (device_needs_bounce(dev)) iommu_setup_dma_ops(dev, base,
set_dma_ops(dev, &bounce_dma_ops); __DOMAIN_MAX_ADDR(dmar_domain->gaw) - base);
else if (domain && domain->type == IOMMU_DOMAIN_DMA)
set_dma_ops(dev, &intel_dma_ops);
else else
set_dma_ops(dev, NULL); set_dma_ops(dev, NULL);
} }
@ -5896,19 +5226,6 @@ int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev)
return ret; return ret;
} }
static void intel_iommu_apply_resv_region(struct device *dev,
struct iommu_domain *domain,
struct iommu_resv_region *region)
{
struct dmar_domain *dmar_domain = to_dmar_domain(domain);
unsigned long start, end;
start = IOVA_PFN(region->start);
end = IOVA_PFN(region->start + region->length - 1);
WARN_ON_ONCE(!reserve_iova(&dmar_domain->iovad, start, end));
}
static struct iommu_group *intel_iommu_device_group(struct device *dev) static struct iommu_group *intel_iommu_device_group(struct device *dev)
{ {
if (dev_is_pci(dev)) if (dev_is_pci(dev))
@ -6097,6 +5414,27 @@ intel_iommu_domain_set_attr(struct iommu_domain *domain,
return ret; return ret;
} }
static int
intel_iommu_domain_get_attr(struct iommu_domain *domain,
enum iommu_attr attr, void *data)
{
switch (domain->type) {
case IOMMU_DOMAIN_UNMANAGED:
return -ENODEV;
case IOMMU_DOMAIN_DMA:
switch (attr) {
case DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE:
*(int *)data = !intel_iommu_strict;
return 0;
default:
return -ENODEV;
}
break;
default:
return -EINVAL;
}
}
/* /*
* Check that the device does not live on an external facing PCI port that is * Check that the device does not live on an external facing PCI port that is
* marked as untrusted. Such devices should not be able to apply quirks and * marked as untrusted. Such devices should not be able to apply quirks and
@ -6118,6 +5456,7 @@ const struct iommu_ops intel_iommu_ops = {
.capable = intel_iommu_capable, .capable = intel_iommu_capable,
.domain_alloc = intel_iommu_domain_alloc, .domain_alloc = intel_iommu_domain_alloc,
.domain_free = intel_iommu_domain_free, .domain_free = intel_iommu_domain_free,
.domain_get_attr = intel_iommu_domain_get_attr,
.domain_set_attr = intel_iommu_domain_set_attr, .domain_set_attr = intel_iommu_domain_set_attr,
.attach_dev = intel_iommu_attach_device, .attach_dev = intel_iommu_attach_device,
.detach_dev = intel_iommu_detach_device, .detach_dev = intel_iommu_detach_device,
@ -6126,6 +5465,7 @@ const struct iommu_ops intel_iommu_ops = {
.aux_get_pasid = intel_iommu_aux_get_pasid, .aux_get_pasid = intel_iommu_aux_get_pasid,
.map = intel_iommu_map, .map = intel_iommu_map,
.unmap = intel_iommu_unmap, .unmap = intel_iommu_unmap,
.flush_iotlb_all = intel_flush_iotlb_all,
.iotlb_sync = intel_iommu_tlb_sync, .iotlb_sync = intel_iommu_tlb_sync,
.iova_to_phys = intel_iommu_iova_to_phys, .iova_to_phys = intel_iommu_iova_to_phys,
.probe_device = intel_iommu_probe_device, .probe_device = intel_iommu_probe_device,
@ -6133,7 +5473,6 @@ const struct iommu_ops intel_iommu_ops = {
.release_device = intel_iommu_release_device, .release_device = intel_iommu_release_device,
.get_resv_regions = intel_iommu_get_resv_regions, .get_resv_regions = intel_iommu_get_resv_regions,
.put_resv_regions = generic_iommu_put_resv_regions, .put_resv_regions = generic_iommu_put_resv_regions,
.apply_resv_region = intel_iommu_apply_resv_region,
.device_group = intel_iommu_device_group, .device_group = intel_iommu_device_group,
.dev_has_feat = intel_iommu_dev_has_feat, .dev_has_feat = intel_iommu_dev_has_feat,
.dev_feat_enabled = intel_iommu_dev_feat_enabled, .dev_feat_enabled = intel_iommu_dev_feat_enabled,