iommu/dma: Provide an interface to allow preallocate IOVA

Separate IOVA allocation to dedicated callback so it will allow
cache of IOVA and reuse it in fast paths for devices which support
ODP (on-demand-paging) mechanism.

Signed-off-by: Leon Romanovsky <leonro@nvidia.com>

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index 2a9fa0c8cc00feb20bc9d520406fe286ad36e318..ed9edd2d27346fb9009e8853b76a313594eb8ab5 100644 (file)
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -358,7 +358,7 @@ int iommu_dma_init_fq(struct iommu_domain *domain)
        atomic_set(&cookie->fq_timer_on, 0);
        /*
         * Prevent incomplete fq state being observable. Pairs with path from
-        * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
+        * __iommu_dma_unmap() through __iommu_dma_free_iova() to queue_iova()
         */
        smp_wmb();
        WRITE_ONCE(cookie->fq_domain, domain);
@@ -759,7 +759,7 @@ static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
        }
 }
 
-static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
+static dma_addr_t __iommu_dma_alloc_iova(struct iommu_domain *domain,
                size_t size, u64 dma_limit, struct device *dev)
 {
        struct iommu_dma_cookie *cookie = domain->iova_cookie;
@@ -805,7 +805,7 @@ done:
        return (dma_addr_t)iova << shift;
 }
 
-static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
+static void __iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
                dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
 {
        struct iova_domain *iovad = &cookie->iovad;
@@ -842,7 +842,7 @@ static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
 
        if (!iotlb_gather.queued)
                iommu_iotlb_sync(domain, &iotlb_gather);
-       iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
+       __iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
 }
 
 static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
@@ -865,12 +865,12 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
 
        size = iova_align(iovad, size + iova_off);
 
-       iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev);
+       iova = __iommu_dma_alloc_iova(domain, size, dma_mask, dev);
        if (!iova)
                return DMA_MAPPING_ERROR;
 
        if (iommu_map(domain, iova, phys - iova_off, size, prot, GFP_ATOMIC)) {
-               iommu_dma_free_iova(cookie, iova, size, NULL);
+               __iommu_dma_free_iova(cookie, iova, size, NULL);
                return DMA_MAPPING_ERROR;
        }
        return iova + iova_off;
@@ -973,7 +973,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
                return NULL;
 
        size = iova_align(iovad, size);
-       iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
+       iova = __iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
        if (!iova)
                goto out_free_pages;
 
@@ -1007,7 +1007,7 @@ static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
 out_free_sg:
        sg_free_table(sgt);
 out_free_iova:
-       iommu_dma_free_iova(cookie, iova, size, NULL);
+       __iommu_dma_free_iova(cookie, iova, size, NULL);
 out_free_pages:
        __iommu_dma_free_pages(pages, count);
        return NULL;
@@ -1467,7 +1467,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
        if (!iova_len)
                return __finalise_sg(dev, sg, nents, 0);
 
-       iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
+       iova = __iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
        if (!iova) {
                ret = -ENOMEM;
                goto out_restore_sg;
@@ -1484,7 +1484,7 @@ int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
        return __finalise_sg(dev, sg, nents, iova);
 
 out_free_iova:
-       iommu_dma_free_iova(cookie, iova, iova_len, NULL);
+       __iommu_dma_free_iova(cookie, iova, iova_len, NULL);
 out_restore_sg:
        __invalidate_sg(sg, nents);
 out:
@@ -1744,6 +1744,39 @@ size_t iommu_dma_max_mapping_size(struct device *dev)
        return SIZE_MAX;
 }
 
+int iommu_dma_alloc_iova(struct dma_iova_state *state, phys_addr_t phys,
+                        size_t size)
+{
+       struct iommu_domain *domain = iommu_get_dma_domain(state->dev);
+       struct iommu_dma_cookie *cookie = domain->iova_cookie;
+       struct iova_domain *iovad = &cookie->iovad;
+       dma_addr_t addr;
+
+       size = iova_align(iovad, size + iova_offset(iovad, phys));
+       addr = __iommu_dma_alloc_iova(domain, size, dma_get_mask(state->dev),
+                                     state->dev);
+       if (addr == DMA_MAPPING_ERROR)
+               return -EINVAL;
+
+       state->addr = addr;
+       state->size = size;
+       return 0;
+}
+
+void iommu_dma_free_iova(struct dma_iova_state *state)
+{
+       struct iommu_domain *domain = iommu_get_dma_domain(state->dev);
+       struct iommu_dma_cookie *cookie = domain->iova_cookie;
+       struct iova_domain *iovad = &cookie->iovad;
+       size_t iova_off = iova_offset(iovad, state->addr);
+       struct iommu_iotlb_gather iotlb_gather;
+
+       iommu_iotlb_gather_init(&iotlb_gather);
+       __iommu_dma_free_iova(cookie, state->addr - iova_off,
+                             iova_align(iovad, state->size + iova_off),
+                             &iotlb_gather);
+}
+
 void iommu_setup_dma_ops(struct device *dev)
 {
        struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
@@ -1780,7 +1813,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
        if (!msi_page)
                return NULL;
 
-       iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
+       iova = __iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
        if (!iova)
                goto out_free_page;
 
@@ -1794,7 +1827,7 @@ static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
        return msi_page;
 
 out_free_iova:
-       iommu_dma_free_iova(cookie, iova, size, NULL);
+       __iommu_dma_free_iova(cookie, iova, size, NULL);
 out_free_page:
        kfree(msi_page);
        return NULL;

diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h
index d83e02f5f68a456674c33b546bbbe21158a8f63f..55c5c334ca76b773d61337bfa7209ace5207b2c5 100644 (file)
--- a/include/linux/dma-mapping.h
+++ b/include/linux/dma-mapping.h
@@ -78,6 +78,8 @@
 
 struct dma_iova_state {
        struct device *dev;
+       dma_addr_t addr;
+       size_t size;
        enum dma_data_direction dir;
 };
 

diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h
index 508beaa44c39e82567ae22a30ed806813a951cc4..85d2b2fcbfd9533601e8c51d1b6ce4b9b3086edc 100644 (file)
--- a/include/linux/iommu-dma.h
+++ b/include/linux/iommu-dma.h
@@ -8,6 +8,7 @@
 #define _LINUX_IOMMU_DMA_H
 
 #include <linux/dma-direction.h>
+#include <linux/dma-mapping.h>
 
 #ifdef CONFIG_IOMMU_DMA
 static inline bool use_dma_iommu(struct device *dev)
@@ -65,5 +66,7 @@ void iommu_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sgl,
                int nelems, enum dma_data_direction dir);
 void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl,
                int nelems, enum dma_data_direction dir);
-
+int iommu_dma_alloc_iova(struct dma_iova_state *state, phys_addr_t phys,
+               size_t size);
+void iommu_dma_free_iova(struct dma_iova_state *state);
 #endif /* _LINUX_IOMMU_DMA_H */