mm: add device coherent vma selection for memory migration

This case is used to migrate pages from device memory, back to system
memory.  Device coherent type memory is cache coherent from device and CPU
point of view.

Link: https://lkml.kernel.org/r/20220715150521.18165-6-alex.sierra@amd.com
Signed-off-by: Alex Sierra <alex.sierra@amd.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Acked-by: Felix Kuehling <Felix.Kuehling@amd.com>
Reviewed-by: Alistair Poppple <apopple@nvidia.com>
Reviewed-by: David Hildenbrand <david@redhat.com>
Cc: Jason Gunthorpe <jgg@nvidia.com>
Cc: Jerome Glisse <jglisse@redhat.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Ralph Campbell <rcampbell@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 069a89e847f345a9e0cd26caab90d07ece2d63d9..b84908debe5cdd1f2a79e758586ac5fafff43cc2 100644 (file)
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -148,6 +148,7 @@ static inline unsigned long migrate_pfn(unsigned long pfn)
 enum migrate_vma_direction {
        MIGRATE_VMA_SELECT_SYSTEM = 1 << 0,
        MIGRATE_VMA_SELECT_DEVICE_PRIVATE = 1 << 1,
+       MIGRATE_VMA_SELECT_DEVICE_COHERENT = 1 << 2,
 };
 
 struct migrate_vma {

diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index a4847ad65da3c1f0186f8b1d6a9bc0482f81d0ec..18bc6483f63a2663652209d6c7fd1f57b43831a8 100644 (file)
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -148,15 +148,21 @@ again:
                        if (is_writable_device_private_entry(entry))
                                mpfn |= MIGRATE_PFN_WRITE;
                } else {
-                       if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
-                               goto next;
                        pfn = pte_pfn(pte);
-                       if (is_zero_pfn(pfn)) {
+                       if (is_zero_pfn(pfn) &&
+                           (migrate->flags & MIGRATE_VMA_SELECT_SYSTEM)) {
                                mpfn = MIGRATE_PFN_MIGRATE;
                                migrate->cpages++;
                                goto next;
                        }
                        page = vm_normal_page(migrate->vma, addr, pte);
+                       if (page && !is_zone_device_page(page) &&
+                           !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
+                               goto next;
+                       else if (page && is_device_coherent_page(page) &&
+                           (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) ||
+                            page->pgmap->owner != migrate->pgmap_owner))
+                               goto next;
                        mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
                        mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
                }