From: Christoph Hellwig Date: Sat, 5 Oct 2024 09:21:12 +0000 (+0200) Subject: dma-mapping: add a dma_unlink_and_free_iova API X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=d86a03930160ddad48801a97253b62b277c13f4d;p=users%2Fhch%2Fblock.git dma-mapping: add a dma_unlink_and_free_iova API This combines the unmapping and freeing of the IOVA so that the father structured and queued mode can be used for the common case. Signed-off-by: Christoph Hellwig --- diff --git a/drivers/iommu/dma-iommu.c b/drivers/iommu/dma-iommu.c index 9076b1b1d9c0..ce5b54dac513 100644 --- a/drivers/iommu/dma-iommu.c +++ b/drivers/iommu/dma-iommu.c @@ -1852,6 +1852,32 @@ void iommu_dma_unlink_range(struct device *dev, dma_addr_t start, size_t size, WARN_ON(unmapped != size); } +void iommu_dma_unlink_and_free_iova(struct dma_iova_state *state, + dma_addr_t dma_addr, size_t size) +{ + struct iommu_domain *domain = iommu_get_dma_domain(state->dev); + struct iommu_dma_cookie *cookie = domain->iova_cookie; + struct iommu_iotlb_gather iotlb_gather; + size_t unmapped; + + if (WARN_ON_ONCE(dma_addr + size > state->addr + state->range_size)) + return; + + iommu_iotlb_gather_init(&iotlb_gather); + iotlb_gather.queued = READ_ONCE(cookie->fq_domain); + + if (!dev_is_dma_coherent(state->dev)) + iommu_sync_dma_for_cpu(domain, dma_addr, size, state->dir); + + unmapped = iommu_unmap_fast(domain, state->addr, state->range_size, + &iotlb_gather); + WARN_ON_ONCE(unmapped != state->range_size); + + if (!iotlb_gather.queued) + iommu_iotlb_sync(domain, &iotlb_gather); + __iommu_dma_free_iova(cookie, state->addr, state->size, &iotlb_gather); +} + bool iommu_can_use_iova(struct device *dev, struct page *page, size_t size, enum dma_data_direction dir) { diff --git a/drivers/vfio/pci/mlx5/cmd.c b/drivers/vfio/pci/mlx5/cmd.c index 2a846bb6d8be..1cb966c094f9 100644 --- a/drivers/vfio/pci/mlx5/cmd.c +++ b/drivers/vfio/pci/mlx5/cmd.c @@ -364,7 +364,7 @@ static void unregister_dma_pages(struct mlx5_core_dev *mdev, u32 npages, WARN_ON_ONCE(state->dir == DMA_NONE); if (dma_can_use_iova(state)) { - dma_unlink_range(state); + dma_unlink_and_free_iova(state, state->addr, state->range_size); } else { mtt = (__be64 *)MLX5_ADDR_OF(create_mkey_in, mkey_in, klm_pas_mtt); @@ -373,7 +373,6 @@ static void unregister_dma_pages(struct mlx5_core_dev *mdev, u32 npages, dma_unmap_page(state->dev, addr, PAGE_SIZE, state->dir); } } - dma_free_iova(state); } static int register_dma_pages(struct mlx5_core_dev *mdev, u32 npages, diff --git a/include/linux/dma-mapping.h b/include/linux/dma-mapping.h index 0638918a74f8..bfbd1092980c 100644 --- a/include/linux/dma-mapping.h +++ b/include/linux/dma-mapping.h @@ -123,6 +123,8 @@ static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) int dma_alloc_iova_unaligned(struct dma_iova_state *state, phys_addr_t phys, size_t size); void dma_free_iova(struct dma_iova_state *state); +void dma_unlink_and_free_iova(struct dma_iova_state *state, + dma_addr_t dma_addr, size_t size); dma_addr_t dma_hmm_link_page(struct dma_iova_state *state, unsigned long *pfn, dma_addr_t dma_offset); void dma_hmm_unlink_page(struct dma_iova_state *state, unsigned long *pfn, @@ -183,7 +185,6 @@ int dma_start_range(struct dma_iova_state *state); void dma_end_range(struct dma_iova_state *state); dma_addr_t dma_link_range_attrs(struct dma_iova_state *state, phys_addr_t phys, size_t size, unsigned long attrs); -void dma_unlink_range_attrs(struct dma_iova_state *state, unsigned long attrs); #else /* CONFIG_HAS_DMA */ static inline int dma_alloc_iova_unaligned(struct dma_iova_state *state, phys_addr_t phys, size_t size) @@ -193,6 +194,10 @@ static inline int dma_alloc_iova_unaligned(struct dma_iova_state *state, static inline void dma_free_iova(struct dma_iova_state *state) { } +static inline void dma_unlink_and_free_iova(struct dma_iova_state *state, + dma_addr_t dma_addr, size_t size) +{ +} static inline dma_addr_t dma_hmm_link_page(struct dma_iova_state *state, unsigned long *pfn, dma_addr_t dma_offset) @@ -354,10 +359,6 @@ static inline dma_addr_t dma_link_range_attrs(struct dma_iova_state *state, { return DMA_MAPPING_ERROR; } -static inline void dma_unlink_range_attrs(struct dma_iova_state *state, - unsigned long attrs) -{ -} #endif /* CONFIG_HAS_DMA */ #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) @@ -553,7 +554,6 @@ static inline void dma_sync_sgtable_for_device(struct device *dev, #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) #define dma_link_range(d, p, o) dma_link_range_attrs(d, p, o, 0) -#define dma_unlink_range(d) dma_unlink_range_attrs(d, 0) bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size); diff --git a/include/linux/iommu-dma.h b/include/linux/iommu-dma.h index 0832b115a436..90c8b4a02498 100644 --- a/include/linux/iommu-dma.h +++ b/include/linux/iommu-dma.h @@ -69,6 +69,8 @@ void iommu_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, int iommu_dma_alloc_iova(struct dma_iova_state *state, phys_addr_t phys, size_t size); void iommu_dma_free_iova(struct dma_iova_state *state); +void iommu_dma_unlink_and_free_iova(struct dma_iova_state *state, + dma_addr_t dma_addr, size_t size); int iommu_dma_start_range(struct device *dev); void iommu_dma_end_range(struct device *dev); dma_addr_t iommu_dma_link_range(struct dma_iova_state *state, phys_addr_t phys, diff --git a/kernel/dma/mapping.c b/kernel/dma/mapping.c index c4880d458147..152078d0ffe2 100644 --- a/kernel/dma/mapping.c +++ b/kernel/dma/mapping.c @@ -1001,6 +1001,15 @@ void dma_free_iova(struct dma_iova_state *state) } EXPORT_SYMBOL_GPL(dma_free_iova); +void dma_unlink_and_free_iova(struct dma_iova_state *state, + dma_addr_t dma_addr, size_t size) +{ + if (!use_dma_iommu(state->dev)) + return; + iommu_dma_unlink_and_free_iova(state, dma_addr, size); +} +EXPORT_SYMBOL_GPL(dma_unlink_and_free_iova); + /** * dma_set_iova_state - Set the IOVA state for the given page and size * @state: IOVA state @@ -1083,20 +1092,6 @@ dma_addr_t dma_link_range_attrs(struct dma_iova_state *state, phys_addr_t phys, } EXPORT_SYMBOL_GPL(dma_link_range_attrs); -/** - * dma_unlink_range_attrs - Unlink a range of IOVA space - * @state: IOVA state - * @attrs: attributes of mapping properties - * - * Unlink a range of IOVA space for the given IOVA state. - */ -void dma_unlink_range_attrs(struct dma_iova_state *state, unsigned long attrs) -{ - iommu_dma_unlink_range(state->dev, state->addr, state->range_size, - state->dir, attrs); -} -EXPORT_SYMBOL_GPL(dma_unlink_range_attrs); - /** * dma_hmm_link_page - Link a physical HMM page to DMA address * @state: IOVA state