vfio: powerpc/spapr: Move page pinning from arch code to VFIO IOMMU driver

This moves page pinning (get_user_pages_fast()/put_page()) code out of
the platform IOMMU code and puts it to VFIO IOMMU driver where it belongs
to as the platform code does not deal with page pinning.

This makes iommu_take_ownership()/iommu_release_ownership() deal with
the IOMMU table bitmap only.

This removes page unpinning from iommu_take_ownership() as the actual
TCE table might contain garbage and doing put_page() on it is undefined
behaviour.

Besides the last part, the rest of the patch is mechanical.

Signed-off-by: Alexey Kardashevskiy <aik@ozlabs.ru>
[aw: for the vfio related changes]
Acked-by: Alex Williamson <alex.williamson@redhat.com>
Reviewed-by: David Gibson <david@gibson.dropbear.id.au>
Reviewed-by: Gavin Shan <gwshan@linux.vnet.ibm.com>
Signed-off-by: Michael Ellerman <mpe@ellerman.id.au>

diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h
index 8353c865db6ffccc9a97490147a0646e657bc2d5..e94a5e3c5a2f6e1f0a8b217bdd67f2b3d5ad0faf 100644 (file)
--- a/arch/powerpc/include/asm/iommu.h
+++ b/arch/powerpc/include/asm/iommu.h
@@ -194,10 +194,6 @@ extern int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
                unsigned long hwaddr, enum dma_data_direction direction);
 extern unsigned long iommu_clear_tce(struct iommu_table *tbl,
                unsigned long entry);
-extern int iommu_clear_tces_and_put_pages(struct iommu_table *tbl,
-               unsigned long entry, unsigned long pages);
-extern int iommu_put_tce_user_mode(struct iommu_table *tbl,
-               unsigned long entry, unsigned long tce);
 
 extern void iommu_flush_tce(struct iommu_table *tbl);
 extern int iommu_take_ownership(struct iommu_table *tbl);

diff --git a/arch/powerpc/kernel/iommu.c b/arch/powerpc/kernel/iommu.c
index 73eb39a7a9e1972f807131bfb5e872ba16c0ad83..0019c80aa81d9414a089dd97ff0389b5bb1c13f5 100644 (file)
--- a/arch/powerpc/kernel/iommu.c
+++ b/arch/powerpc/kernel/iommu.c
@@ -986,30 +986,6 @@ unsigned long iommu_clear_tce(struct iommu_table *tbl, unsigned long entry)
 }
 EXPORT_SYMBOL_GPL(iommu_clear_tce);
 
-int iommu_clear_tces_and_put_pages(struct iommu_table *tbl,
-               unsigned long entry, unsigned long pages)
-{
-       unsigned long oldtce;
-       struct page *page;
-
-       for ( ; pages; --pages, ++entry) {
-               oldtce = iommu_clear_tce(tbl, entry);
-               if (!oldtce)
-                       continue;
-
-               page = pfn_to_page(oldtce >> PAGE_SHIFT);
-               WARN_ON(!page);
-               if (page) {
-                       if (oldtce & TCE_PCI_WRITE)
-                               SetPageDirty(page);
-                       put_page(page);
-               }
-       }
-
-       return 0;
-}
-EXPORT_SYMBOL_GPL(iommu_clear_tces_and_put_pages);
-
 /*
  * hwaddr is a kernel virtual address here (0xc... bazillion),
  * tce_build converts it to a physical address.
@@ -1039,35 +1015,6 @@ int iommu_tce_build(struct iommu_table *tbl, unsigned long entry,
 }
 EXPORT_SYMBOL_GPL(iommu_tce_build);
 
-int iommu_put_tce_user_mode(struct iommu_table *tbl, unsigned long entry,
-               unsigned long tce)
-{
-       int ret;
-       struct page *page = NULL;
-       unsigned long hwaddr, offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
-       enum dma_data_direction direction = iommu_tce_direction(tce);
-
-       ret = get_user_pages_fast(tce & PAGE_MASK, 1,
-                       direction != DMA_TO_DEVICE, &page);
-       if (unlikely(ret != 1)) {
-               /* pr_err("iommu_tce: get_user_pages_fast failed tce=%lx ioba=%lx ret=%d\n",
-                               tce, entry << tbl->it_page_shift, ret); */
-               return -EFAULT;
-       }
-       hwaddr = (unsigned long) page_address(page) + offset;
-
-       ret = iommu_tce_build(tbl, entry, hwaddr, direction);
-       if (ret)
-               put_page(page);
-
-       if (ret < 0)
-               pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%d\n",
-                       __func__, entry << tbl->it_page_shift, tce, ret);
-
-       return ret;
-}
-EXPORT_SYMBOL_GPL(iommu_put_tce_user_mode);
-
 int iommu_take_ownership(struct iommu_table *tbl)
 {
        unsigned long sz = (tbl->it_size + 7) >> 3;
@@ -1081,7 +1028,6 @@ int iommu_take_ownership(struct iommu_table *tbl)
        }
 
        memset(tbl->it_map, 0xff, sz);
-       iommu_clear_tces_and_put_pages(tbl, tbl->it_offset, tbl->it_size);
 
        /*
         * Disable iommu bypass, otherwise the user can DMA to all of
@@ -1099,7 +1045,6 @@ void iommu_release_ownership(struct iommu_table *tbl)
 {
        unsigned long sz = (tbl->it_size + 7) >> 3;
 
-       iommu_clear_tces_and_put_pages(tbl, tbl->it_offset, tbl->it_size);
        memset(tbl->it_map, 0, sz);
 
        /* Restore bit#0 set by iommu_init_table() */

diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c
index 730b4ef3e0cc3543382930456f923135858cb181..b95fa2b64680c0a7374ae18c65e8b686c4ecc75f 100644 (file)
--- a/drivers/vfio/vfio_iommu_spapr_tce.c
+++ b/drivers/vfio/vfio_iommu_spapr_tce.c
@@ -147,6 +147,67 @@ static void tce_iommu_release(void *iommu_data)
        kfree(container);
 }
 
+static int tce_iommu_clear(struct tce_container *container,
+               struct iommu_table *tbl,
+               unsigned long entry, unsigned long pages)
+{
+       unsigned long oldtce;
+       struct page *page;
+
+       for ( ; pages; --pages, ++entry) {
+               oldtce = iommu_clear_tce(tbl, entry);
+               if (!oldtce)
+                       continue;
+
+               page = pfn_to_page(oldtce >> PAGE_SHIFT);
+               WARN_ON(!page);
+               if (page) {
+                       if (oldtce & TCE_PCI_WRITE)
+                               SetPageDirty(page);
+                       put_page(page);
+               }
+       }
+
+       return 0;
+}
+
+static long tce_iommu_build(struct tce_container *container,
+               struct iommu_table *tbl,
+               unsigned long entry, unsigned long tce, unsigned long pages)
+{
+       long i, ret = 0;
+       struct page *page = NULL;
+       unsigned long hva;
+       enum dma_data_direction direction = iommu_tce_direction(tce);
+
+       for (i = 0; i < pages; ++i) {
+               unsigned long offset = tce & IOMMU_PAGE_MASK(tbl) & ~PAGE_MASK;
+
+               ret = get_user_pages_fast(tce & PAGE_MASK, 1,
+                               direction != DMA_TO_DEVICE, &page);
+               if (unlikely(ret != 1)) {
+                       ret = -EFAULT;
+                       break;
+               }
+               hva = (unsigned long) page_address(page) + offset;
+
+               ret = iommu_tce_build(tbl, entry + i, hva, direction);
+               if (ret) {
+                       put_page(page);
+                       pr_err("iommu_tce: %s failed ioba=%lx, tce=%lx, ret=%ld\n",
+                                       __func__, entry << tbl->it_page_shift,
+                                       tce, ret);
+                       break;
+               }
+               tce += IOMMU_PAGE_SIZE_4K;
+       }
+
+       if (ret)
+               tce_iommu_clear(container, tbl, entry, i);
+
+       return ret;
+}
+
 static long tce_iommu_ioctl(void *iommu_data,
                                 unsigned int cmd, unsigned long arg)
 {
@@ -195,7 +256,7 @@ static long tce_iommu_ioctl(void *iommu_data,
        case VFIO_IOMMU_MAP_DMA: {
                struct vfio_iommu_type1_dma_map param;
                struct iommu_table *tbl = container->tbl;
-               unsigned long tce, i;
+               unsigned long tce;
 
                if (!tbl)
                        return -ENXIO;
@@ -229,17 +290,9 @@ static long tce_iommu_ioctl(void *iommu_data,
                if (ret)
                        return ret;
 
-               for (i = 0; i < (param.size >> IOMMU_PAGE_SHIFT_4K); ++i) {
-                       ret = iommu_put_tce_user_mode(tbl,
-                                       (param.iova >> IOMMU_PAGE_SHIFT_4K) + i,
-                                       tce);
-                       if (ret)
-                               break;
-                       tce += IOMMU_PAGE_SIZE_4K;
-               }
-               if (ret)
-                       iommu_clear_tces_and_put_pages(tbl,
-                                       param.iova >> IOMMU_PAGE_SHIFT_4K, i);
+               ret = tce_iommu_build(container, tbl,
+                               param.iova >> IOMMU_PAGE_SHIFT_4K,
+                               tce, param.size >> IOMMU_PAGE_SHIFT_4K);
 
                iommu_flush_tce(tbl);
 
@@ -273,7 +326,7 @@ static long tce_iommu_ioctl(void *iommu_data,
                if (ret)
                        return ret;
 
-               ret = iommu_clear_tces_and_put_pages(tbl,
+               ret = tce_iommu_clear(container, tbl,
                                param.iova >> IOMMU_PAGE_SHIFT_4K,
                                param.size >> IOMMU_PAGE_SHIFT_4K);
                iommu_flush_tce(tbl);
@@ -357,6 +410,7 @@ static void tce_iommu_detach_group(void *iommu_data,
                /* pr_debug("tce_vfio: detaching group #%u from iommu %p\n",
                                iommu_group_id(iommu_group), iommu_group); */
                container->tbl = NULL;
+               tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size);
                iommu_release_ownership(tbl);
        }
        mutex_unlock(&container->lock);