iommu: Add gfp parameter to iommu_ops::map

Add a gfp_t parameter to the iommu_ops::map function.
Remove the needless locking in the AMD iommu driver.

The iommu_ops::map function (or the iommu_map function which calls it)
was always supposed to be sleepable (according to Joerg's comment in
this thread: https://lore.kernel.org/patchwork/patch/977520/ ) and so
should probably have had a "might_sleep()" since it was written. However
currently the dma-iommu api can call iommu_map in an atomic context,
which it shouldn't do. This doesn't cause any problems because any iommu
driver which uses the dma-iommu api uses gfp_atomic in it's
iommu_ops::map function. But doing this wastes the memory allocators
atomic pools.

Signed-off-by: Tom Murphy <murphyt7@tcd.ie>
Reviewed-by: Robin Murphy <robin.murphy@arm.com>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Signed-off-by: Joerg Roedel <jroedel@suse.de>

diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c
index 97ca4ed4ce0c15f98d3eadbcb6904615f8ffca89..1444757f0ff93e8c686ed2be742898f49a364101 100644 (file)
--- a/drivers/iommu/amd_iommu.c
+++ b/drivers/iommu/amd_iommu.c
@@ -3106,7 +3106,8 @@ static int amd_iommu_attach_device(struct iommu_domain *dom,
 }
 
 static int amd_iommu_map(struct iommu_domain *dom, unsigned long iova,
-                        phys_addr_t paddr, size_t page_size, int iommu_prot)
+                        phys_addr_t paddr, size_t page_size, int iommu_prot,
+                        gfp_t gfp)
 {
        struct protection_domain *domain = to_pdomain(dom);
        int prot = 0;

diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 8da93e730d6fd9f7eb6f1d9a2e8072232c7e3754..f1bdaaa8c5deecf2260b5c45927115058daf70d1 100644 (file)
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -2448,7 +2448,7 @@ out_unlock:
 }
 
 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
-                       phys_addr_t paddr, size_t size, int prot)
+                       phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
 

diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c
index b18aac4c105eef088d03cb624011dcc25284f9e5..e85bc0e8d5640d40c37bcb967d8d1bfaeb774b2f 100644 (file)
--- a/drivers/iommu/arm-smmu.c
+++ b/drivers/iommu/arm-smmu.c
@@ -1159,7 +1159,7 @@ rpm_put:
 }
 
 static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova,
-                       phys_addr_t paddr, size_t size, int prot)
+                       phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
        struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;

diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c
index f321279baf9e8984effbc212697862f2f771e1e8..cc3bf5cf0a90feea3afe986afd4c6b3713b7d3f2 100644 (file)
--- a/drivers/iommu/dma-iommu.c
+++ b/drivers/iommu/dma-iommu.c
@@ -476,7 +476,7 @@ static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
        if (!iova)
                return DMA_MAPPING_ERROR;
 
-       if (iommu_map(domain, iova, phys - iova_off, size, prot)) {
+       if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
                iommu_dma_free_iova(cookie, iova, size);
                return DMA_MAPPING_ERROR;
        }
@@ -611,7 +611,7 @@ static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
                        arch_dma_prep_coherent(sg_page(sg), sg->length);
        }
 
-       if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot)
+       if (iommu_map_sg_atomic(domain, iova, sgt.sgl, sgt.orig_nents, ioprot)
                        < size)
                goto out_free_sg;
 
@@ -871,7 +871,7 @@ static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
         * We'll leave any physical concatenation to the IOMMU driver's
         * implementation - it knows better than we do.
         */
-       if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)
+       if (iommu_map_sg_atomic(domain, iova, sg, nents, prot) < iova_len)
                goto out_free_iova;
 
        return __finalise_sg(dev, sg, nents, iova);

diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c
index 9c94e16fb1277f793bb8d24062b09afb8ab49d74..186ff5cc975ca14cc33484daa4562da4d73cde30 100644 (file)
--- a/drivers/iommu/exynos-iommu.c
+++ b/drivers/iommu/exynos-iommu.c
@@ -1073,7 +1073,7 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size,
  */
 static int exynos_iommu_map(struct iommu_domain *iommu_domain,
                            unsigned long l_iova, phys_addr_t paddr, size_t size,
-                           int prot)
+                           int prot, gfp_t gfp)
 {
        struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain);
        sysmmu_pte_t *entry;

diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c
index 3f974919d3bdb4c7ee78adc091f585c4cc7a6da7..615495aa613ecb0bd6dce4f90360fc368489bb05 100644 (file)
--- a/drivers/iommu/intel-iommu.c
+++ b/drivers/iommu/intel-iommu.c
@@ -5432,7 +5432,7 @@ static void intel_iommu_aux_detach_device(struct iommu_domain *domain,
 
 static int intel_iommu_map(struct iommu_domain *domain,
                           unsigned long iova, phys_addr_t hpa,
-                          size_t size, int iommu_prot)
+                          size_t size, int iommu_prot, gfp_t gfp)
 {
        struct dmar_domain *dmar_domain = to_dmar_domain(domain);
        u64 max_addr;

diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c
index d658c7c6a2ab0a6d3d6fcffdb89edf03f9b9bda2..f8853dbf1c4eb0be9de8ecb211883344fd786e11 100644 (file)
--- a/drivers/iommu/iommu.c
+++ b/drivers/iommu/iommu.c
@@ -1854,8 +1854,8 @@ static size_t iommu_pgsize(struct iommu_domain *domain,
        return pgsize;
 }
 
-int iommu_map(struct iommu_domain *domain, unsigned long iova,
-             phys_addr_t paddr, size_t size, int prot)
+int __iommu_map(struct iommu_domain *domain, unsigned long iova,
+             phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        const struct iommu_ops *ops = domain->ops;
        unsigned long orig_iova = iova;
@@ -1892,8 +1892,8 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
 
                pr_debug("mapping: iova 0x%lx pa %pa pgsize 0x%zx\n",
                         iova, &paddr, pgsize);
+               ret = ops->map(domain, iova, paddr, pgsize, prot, gfp);
 
-               ret = ops->map(domain, iova, paddr, pgsize, prot);
                if (ret)
                        break;
 
@@ -1913,8 +1913,22 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova,
 
        return ret;
 }
+
+int iommu_map(struct iommu_domain *domain, unsigned long iova,
+             phys_addr_t paddr, size_t size, int prot)
+{
+       might_sleep();
+       return __iommu_map(domain, iova, paddr, size, prot, GFP_KERNEL);
+}
 EXPORT_SYMBOL_GPL(iommu_map);
 
+int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
+             phys_addr_t paddr, size_t size, int prot)
+{
+       return __iommu_map(domain, iova, paddr, size, prot, GFP_ATOMIC);
+}
+EXPORT_SYMBOL_GPL(iommu_map_atomic);
+
 static size_t __iommu_unmap(struct iommu_domain *domain,
                            unsigned long iova, size_t size,
                            struct iommu_iotlb_gather *iotlb_gather)
@@ -1991,8 +2005,9 @@ size_t iommu_unmap_fast(struct iommu_domain *domain,
 }
 EXPORT_SYMBOL_GPL(iommu_unmap_fast);
 
-size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
-                   struct scatterlist *sg, unsigned int nents, int prot)
+size_t __iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+                   struct scatterlist *sg, unsigned int nents, int prot,
+                   gfp_t gfp)
 {
        size_t len = 0, mapped = 0;
        phys_addr_t start;
@@ -2003,7 +2018,9 @@ size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
                phys_addr_t s_phys = sg_phys(sg);
 
                if (len && s_phys != start + len) {
-                       ret = iommu_map(domain, iova + mapped, start, len, prot);
+                       ret = __iommu_map(domain, iova + mapped, start,
+                                       len, prot, gfp);
+
                        if (ret)
                                goto out_err;
 
@@ -2031,8 +2048,22 @@ out_err:
        return 0;
 
 }
+
+size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
+                   struct scatterlist *sg, unsigned int nents, int prot)
+{
+       might_sleep();
+       return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_KERNEL);
+}
 EXPORT_SYMBOL_GPL(iommu_map_sg);
 
+size_t iommu_map_sg_atomic(struct iommu_domain *domain, unsigned long iova,
+                   struct scatterlist *sg, unsigned int nents, int prot)
+{
+       return __iommu_map_sg(domain, iova, sg, nents, prot, GFP_ATOMIC);
+}
+EXPORT_SYMBOL_GPL(iommu_map_sg_atomic);
+
 int iommu_domain_window_enable(struct iommu_domain *domain, u32 wnd_nr,
                               phys_addr_t paddr, u64 size, int prot)
 {

diff --git a/drivers/iommu/ipmmu-vmsa.c b/drivers/iommu/ipmmu-vmsa.c
index 9da8309f71708f213f91dbe99374681e0da3f652..35f3edaba0e2924eef2df8886038640a66a4b2bf 100644 (file)
--- a/drivers/iommu/ipmmu-vmsa.c
+++ b/drivers/iommu/ipmmu-vmsa.c
@@ -724,7 +724,7 @@ static void ipmmu_detach_device(struct iommu_domain *io_domain,
 }
 
 static int ipmmu_map(struct iommu_domain *io_domain, unsigned long iova,
-                    phys_addr_t paddr, size_t size, int prot)
+                    phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct ipmmu_vmsa_domain *domain = to_vmsa_domain(io_domain);
 

diff --git a/drivers/iommu/msm_iommu.c b/drivers/iommu/msm_iommu.c
index be99d408cf35dc7b8d26612b979d4c584de64c4a..93f14bca26ee0292ce51578236c59131e58bfc85 100644 (file)
--- a/drivers/iommu/msm_iommu.c
+++ b/drivers/iommu/msm_iommu.c
@@ -504,7 +504,7 @@ fail:
 }
 
 static int msm_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                        phys_addr_t pa, size_t len, int prot)
+                        phys_addr_t pa, size_t len, int prot, gfp_t gfp)
 {
        struct msm_priv *priv = to_msm_priv(domain);
        unsigned long flags;

diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c
index 67a483c1a9357c7629d569ae1456ca8181b92c89..556beaa110b3b55e7f16eba12b0feefc69e7a701 100644 (file)
--- a/drivers/iommu/mtk_iommu.c
+++ b/drivers/iommu/mtk_iommu.c
@@ -412,7 +412,7 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain,
 }
 
 static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                        phys_addr_t paddr, size_t size, int prot)
+                        phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
        struct mtk_iommu_data *data = mtk_iommu_get_m4u_data();

diff --git a/drivers/iommu/mtk_iommu_v1.c b/drivers/iommu/mtk_iommu_v1.c
index b5efd6dac953999a8780f19a9836a7353acd6b0f..e93b94ecac459c4120bbc09dd18d95c5bc523238 100644 (file)
--- a/drivers/iommu/mtk_iommu_v1.c
+++ b/drivers/iommu/mtk_iommu_v1.c
@@ -295,7 +295,7 @@ static void mtk_iommu_detach_device(struct iommu_domain *domain,
 }
 
 static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                        phys_addr_t paddr, size_t size, int prot)
+                        phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct mtk_iommu_domain *dom = to_mtk_domain(domain);
        unsigned int page_num = size >> MT2701_IOMMU_PAGE_SHIFT;

diff --git a/drivers/iommu/omap-iommu.c b/drivers/iommu/omap-iommu.c
index 09c6e1c680db980aeb67dae51986b2a84e88d99a..be551cc34be4507521eac485cedb202cd7fdb664 100644 (file)
--- a/drivers/iommu/omap-iommu.c
+++ b/drivers/iommu/omap-iommu.c
@@ -1339,7 +1339,7 @@ static u32 iotlb_init_entry(struct iotlb_entry *e, u32 da, u32 pa, int pgsz)
 }
 
 static int omap_iommu_map(struct iommu_domain *domain, unsigned long da,
-                         phys_addr_t pa, size_t bytes, int prot)
+                         phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
 {
        struct omap_iommu_domain *omap_domain = to_omap_domain(domain);
        struct device *dev = omap_domain->dev;

diff --git a/drivers/iommu/qcom_iommu.c b/drivers/iommu/qcom_iommu.c
index c31e7bc4ccbec2f7083ba30541e2a2b70df7431a..6ad5f934f00ff260be454b3146673a4ff2d28f5e 100644 (file)
--- a/drivers/iommu/qcom_iommu.c
+++ b/drivers/iommu/qcom_iommu.c
@@ -423,7 +423,7 @@ static void qcom_iommu_detach_dev(struct iommu_domain *domain, struct device *de
 }
 
 static int qcom_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                         phys_addr_t paddr, size_t size, int prot)
+                         phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        int ret;
        unsigned long flags;

diff --git a/drivers/iommu/rockchip-iommu.c b/drivers/iommu/rockchip-iommu.c
index 26290f310f90bd8c6d05da9b255ec02d144cd56f..a4f6425fd12d300928cb9f1ca95bf6d1155c55e7 100644 (file)
--- a/drivers/iommu/rockchip-iommu.c
+++ b/drivers/iommu/rockchip-iommu.c
@@ -757,7 +757,7 @@ unwind:
 }
 
 static int rk_iommu_map(struct iommu_domain *domain, unsigned long _iova,
-                       phys_addr_t paddr, size_t size, int prot)
+                       phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct rk_iommu_domain *rk_domain = to_rk_domain(domain);
        unsigned long flags;

diff --git a/drivers/iommu/s390-iommu.c b/drivers/iommu/s390-iommu.c
index 3b0b18e23187c19d7c787389fb33d73ca957a429..1137f3ddcb851a19572ef970905836d676ec495f 100644 (file)
--- a/drivers/iommu/s390-iommu.c
+++ b/drivers/iommu/s390-iommu.c
@@ -265,7 +265,7 @@ undo_cpu_trans:
 }
 
 static int s390_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                         phys_addr_t paddr, size_t size, int prot)
+                         phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct s390_domain *s390_domain = to_s390_domain(domain);
        int flags = ZPCI_PTE_VALID, rc = 0;

diff --git a/drivers/iommu/tegra-gart.c b/drivers/iommu/tegra-gart.c
index 3924f7c055440edd3ee64be0b02710102bd7efcd..3fb7ba72507def82303475aef334a88f12d43454 100644 (file)
--- a/drivers/iommu/tegra-gart.c
+++ b/drivers/iommu/tegra-gart.c
@@ -178,7 +178,7 @@ static inline int __gart_iommu_map(struct gart_device *gart, unsigned long iova,
 }
 
 static int gart_iommu_map(struct iommu_domain *domain, unsigned long iova,
-                         phys_addr_t pa, size_t bytes, int prot)
+                         phys_addr_t pa, size_t bytes, int prot, gfp_t gfp)
 {
        struct gart_device *gart = gart_handle;
        int ret;

diff --git a/drivers/iommu/tegra-smmu.c b/drivers/iommu/tegra-smmu.c
index 7293fc3f796d69f61f8cbe8a4ef3176d3b9ff5b0..99f85fb5a704e41584da7831e5bf865b78ea2e51 100644 (file)
--- a/drivers/iommu/tegra-smmu.c
+++ b/drivers/iommu/tegra-smmu.c
@@ -650,7 +650,7 @@ static void tegra_smmu_set_pte(struct tegra_smmu_as *as, unsigned long iova,
 }
 
 static int tegra_smmu_map(struct iommu_domain *domain, unsigned long iova,
-                         phys_addr_t paddr, size_t size, int prot)
+                         phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        struct tegra_smmu_as *as = to_smmu_as(domain);
        dma_addr_t pte_dma;

diff --git a/drivers/iommu/virtio-iommu.c b/drivers/iommu/virtio-iommu.c
index 3ea9d7682999528d1348a6d2693f6136556eea57..2ebcf4bae0c6b6b9c3038d08ede2e3add385591e 100644 (file)
--- a/drivers/iommu/virtio-iommu.c
+++ b/drivers/iommu/virtio-iommu.c
@@ -713,7 +713,7 @@ static int viommu_attach_dev(struct iommu_domain *domain, struct device *dev)
 }
 
 static int viommu_map(struct iommu_domain *domain, unsigned long iova,
-                     phys_addr_t paddr, size_t size, int prot)
+                     phys_addr_t paddr, size_t size, int prot, gfp_t gfp)
 {
        int ret;
        u32 flags;

diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 29bac5345563aca434bfb86ab00df539ab80d080..6ca3fb2873d7d8575166c66aaa0007a12741f3ab 100644 (file)
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -256,7 +256,7 @@ struct iommu_ops {
        int (*attach_dev)(struct iommu_domain *domain, struct device *dev);
        void (*detach_dev)(struct iommu_domain *domain, struct device *dev);
        int (*map)(struct iommu_domain *domain, unsigned long iova,
-                  phys_addr_t paddr, size_t size, int prot);
+                  phys_addr_t paddr, size_t size, int prot, gfp_t gfp);
        size_t (*unmap)(struct iommu_domain *domain, unsigned long iova,
                     size_t size, struct iommu_iotlb_gather *iotlb_gather);
        void (*flush_iotlb_all)(struct iommu_domain *domain);
@@ -421,6 +421,8 @@ extern struct iommu_domain *iommu_get_domain_for_dev(struct device *dev);
 extern struct iommu_domain *iommu_get_dma_domain(struct device *dev);
 extern int iommu_map(struct iommu_domain *domain, unsigned long iova,
                     phys_addr_t paddr, size_t size, int prot);
+extern int iommu_map_atomic(struct iommu_domain *domain, unsigned long iova,
+                           phys_addr_t paddr, size_t size, int prot);
 extern size_t iommu_unmap(struct iommu_domain *domain, unsigned long iova,
                          size_t size);
 extern size_t iommu_unmap_fast(struct iommu_domain *domain,
@@ -428,6 +430,9 @@ extern size_t iommu_unmap_fast(struct iommu_domain *domain,
                               struct iommu_iotlb_gather *iotlb_gather);
 extern size_t iommu_map_sg(struct iommu_domain *domain, unsigned long iova,
                           struct scatterlist *sg,unsigned int nents, int prot);
+extern size_t iommu_map_sg_atomic(struct iommu_domain *domain,
+                                 unsigned long iova, struct scatterlist *sg,
+                                 unsigned int nents, int prot);
 extern phys_addr_t iommu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova);
 extern void iommu_set_fault_handler(struct iommu_domain *domain,
                        iommu_fault_handler_t handler, void *token);
@@ -662,6 +667,13 @@ static inline int iommu_map(struct iommu_domain *domain, unsigned long iova,
        return -ENODEV;
 }
 
+static inline int iommu_map_atomic(struct iommu_domain *domain,
+                                  unsigned long iova, phys_addr_t paddr,
+                                  size_t size, int prot)
+{
+       return -ENODEV;
+}
+
 static inline size_t iommu_unmap(struct iommu_domain *domain,
                                 unsigned long iova, size_t size)
 {
@@ -682,6 +694,13 @@ static inline size_t iommu_map_sg(struct iommu_domain *domain,
        return 0;
 }
 
+static inline size_t iommu_map_sg_atomic(struct iommu_domain *domain,
+                                 unsigned long iova, struct scatterlist *sg,
+                                 unsigned int nents, int prot)
+{
+       return 0;
+}
+
 static inline void iommu_flush_tlb_all(struct iommu_domain *domain)
 {
 }