mm: allow compound zone device pages

Zone device pages are used to represent various type of device memory
managed by device drivers.  Currently compound zone device pages are not
supported.  This is because MEMORY_DEVICE_FS_DAX pages are the only user
of higher order zone device pages and have their own page reference
counting.

A future change will unify FS DAX reference counting with normal page
reference counting rules and remove the special FS DAX reference counting.
Supporting that requires compound zone device pages.

Supporting compound zone device pages requires compound_head() to
distinguish between head and tail pages whilst still preserving the
special struct page fields that are specific to zone device pages.

A tail page is distinguished by having bit zero being set in
page->compound_head, with the remaining bits pointing to the head page.
For zone device pages page->compound_head is shared with page->pgmap.

The page->pgmap field must be common to all pages within a folio, even if
the folio spans memory sections.  Therefore pgmap is the same for both
head and tail pages and can be moved into the folio and we can use the
standard scheme to find compound_head from a tail page.

Link: https://lkml.kernel.org/r/5106a9ab1b2eaa8e6f1429fcb3cd1e86ecbb0a29.1739850794.git-series.apopple@nvidia.com
Signed-off-by: Alistair Popple <apopple@nvidia.com>
Reviewed-by: Jason Gunthorpe <jgg@nvidia.com>
Reviewed-by: Dan Williams <dan.j.williams@intel.com>
Acked-by: David Hildenbrand <david@redhat.com>
Tested-by: Alison Schofield <alison.schofield@intel.com>
Cc: Alexander Gordeev <agordeev@linux.ibm.com>
Cc: Asahi Lina <lina@asahilina.net>
Cc: Balbir Singh <balbirs@nvidia.com>
Cc: Bjorn Helgaas <bhelgaas@google.com>
Cc: Catalin Marinas <catalin.marinas@arm.com>
Cc: Christian Borntraeger <borntraeger@linux.ibm.com>
Cc: Christoph Hellwig <hch@lst.de>
Cc: Chunyan Zhang <zhang.lyra@gmail.com>
Cc: "Darrick J. Wong" <djwong@kernel.org>
Cc: Dave Chinner <david@fromorbit.com>
Cc: Dave Hansen <dave.hansen@linux.intel.com>
Cc: Dave Jiang <dave.jiang@intel.com>
Cc: Gerald Schaefer <gerald.schaefer@linux.ibm.com>
Cc: Heiko Carstens <hca@linux.ibm.com>
Cc: Huacai Chen <chenhuacai@kernel.org>
Cc: Ira Weiny <ira.weiny@intel.com>
Cc: Jan Kara <jack@suse.cz>
Cc: Jason Gunthorpe <jgg@ziepe.ca>
Cc: John Hubbard <jhubbard@nvidia.com>
Cc: Logan Gunthorpe <logang@deltatee.com>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Miaohe Lin <linmiaohe@huawei.com>
Cc: Michael Ellerman <mpe@ellerman.id.au>
Cc: Nicholas Piggin <npiggin@gmail.com>
Cc: Peter Xu <peterx@redhat.com>
Cc: Sven Schnelle <svens@linux.ibm.com>
Cc: Ted Ts'o <tytso@mit.edu>
Cc: Vasily Gorbik <gor@linux.ibm.com>
Cc: Vishal Verma <vishal.l.verma@intel.com>
Cc: Vivek Goyal <vgoyal@redhat.com>
Cc: WANG Xuerui <kernel@xen0n.name>
Cc: Will Deacon <will@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/drivers/gpu/drm/nouveau/nouveau_dmem.c b/drivers/gpu/drm/nouveau/nouveau_dmem.c
index 1a072568cef6816d08f99b130bc0bb3111b16ada..61d0f411ef841ae4976ff164ccf5d087c7fd2463 100644 (file)
--- a/drivers/gpu/drm/nouveau/nouveau_dmem.c
+++ b/drivers/gpu/drm/nouveau/nouveau_dmem.c
@@ -88,7 +88,8 @@ struct nouveau_dmem {
 
 static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page)
 {
-       return container_of(page->pgmap, struct nouveau_dmem_chunk, pagemap);
+       return container_of(page_pgmap(page), struct nouveau_dmem_chunk,
+                           pagemap);
 }
 
 static struct nouveau_drm *page_to_drm(struct page *page)

diff --git a/drivers/pci/p2pdma.c b/drivers/pci/p2pdma.c
index 04773a865819d49816b7fa5c25594007ccd16e37..19214ec81fbb8344f33d8e6f79c2ffb33e36c1a4 100644 (file)
--- a/drivers/pci/p2pdma.c
+++ b/drivers/pci/p2pdma.c
@@ -202,7 +202,7 @@ static const struct attribute_group p2pmem_group = {
 
 static void p2pdma_page_free(struct page *page)
 {
-       struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page->pgmap);
+       struct pci_p2pdma_pagemap *pgmap = to_p2p_pgmap(page_pgmap(page));
        /* safe to dereference while a reference is held to the percpu ref */
        struct pci_p2pdma *p2pdma =
                rcu_dereference_protected(pgmap->provider->p2pdma, 1);
@@ -1025,8 +1025,8 @@ enum pci_p2pdma_map_type
 pci_p2pdma_map_segment(struct pci_p2pdma_map_state *state, struct device *dev,
                       struct scatterlist *sg)
 {
-       if (state->pgmap != sg_page(sg)->pgmap) {
-               state->pgmap = sg_page(sg)->pgmap;
+       if (state->pgmap != page_pgmap(sg_page(sg))) {
+               state->pgmap = page_pgmap(sg_page(sg));
                state->map = pci_p2pdma_map_type(state->pgmap, dev);
                state->bus_off = to_p2p_pgmap(state->pgmap)->bus_offset;
        }

diff --git a/include/linux/memremap.h b/include/linux/memremap.h
index 3f7143ade32c049d55f9f2025cdfb712dda9b613..0256a4218dc341967b71575ef4635192edfef99f 100644 (file)
--- a/include/linux/memremap.h
+++ b/include/linux/memremap.h
@@ -161,7 +161,7 @@ static inline bool is_device_private_page(const struct page *page)
 {
        return IS_ENABLED(CONFIG_DEVICE_PRIVATE) &&
                is_zone_device_page(page) &&
-               page->pgmap->type == MEMORY_DEVICE_PRIVATE;
+               page_pgmap(page)->type == MEMORY_DEVICE_PRIVATE;
 }
 
 static inline bool folio_is_device_private(const struct folio *folio)
@@ -173,13 +173,13 @@ static inline bool is_pci_p2pdma_page(const struct page *page)
 {
        return IS_ENABLED(CONFIG_PCI_P2PDMA) &&
                is_zone_device_page(page) &&
-               page->pgmap->type == MEMORY_DEVICE_PCI_P2PDMA;
+               page_pgmap(page)->type == MEMORY_DEVICE_PCI_P2PDMA;
 }
 
 static inline bool is_device_coherent_page(const struct page *page)
 {
        return is_zone_device_page(page) &&
-               page->pgmap->type == MEMORY_DEVICE_COHERENT;
+               page_pgmap(page)->type == MEMORY_DEVICE_COHERENT;
 }
 
 static inline bool folio_is_device_coherent(const struct folio *folio)

diff --git a/include/linux/migrate.h b/include/linux/migrate.h
index 29919faea2f10e97748a1f80bd09fe57966a7d62..61899ec7a9a33fdcfcea22b47f9c6beb1b576195 100644 (file)
--- a/include/linux/migrate.h
+++ b/include/linux/migrate.h
@@ -205,8 +205,8 @@ struct migrate_vma {
        unsigned long           end;
 
        /*
-        * Set to the owner value also stored in page->pgmap->owner for
-        * migrating out of device private memory. The flags also need to
+        * Set to the owner value also stored in page_pgmap(page)->owner
+        * for migrating out of device private memory. The flags also need to
         * be set to MIGRATE_VMA_SELECT_DEVICE_PRIVATE.
         * The caller should always set this field when using mmu notifier
         * callbacks to avoid device MMU invalidations for device private

diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h
index 369f76a8f00cdb96bcb0c61831462e7991ab310c..6f2d6bbfc0dbe6cf5740b49fd28e3168686dfd3a 100644 (file)
--- a/include/linux/mm_types.h
+++ b/include/linux/mm_types.h
@@ -130,8 +130,11 @@ struct page {
                        unsigned long compound_head;    /* Bit zero is set */
                };
                struct {        /* ZONE_DEVICE pages */
-                       /** @pgmap: Points to the hosting device page map. */
-                       struct dev_pagemap *pgmap;
+                       /*
+                        * The first word is used for compound_head or folio
+                        * pgmap
+                        */
+                       void *_unused_pgmap_compound_head;
                        void *zone_device_data;
                        /*
                         * ZONE_DEVICE private pages are counted as being
@@ -300,6 +303,7 @@ typedef struct {
  * @_refcount: Do not access this member directly.  Use folio_ref_count()
  *    to find how many references there are to this folio.
  * @memcg_data: Memory Control Group data.
+ * @pgmap: Metadata for ZONE_DEVICE mappings
  * @virtual: Virtual address in the kernel direct map.
  * @_last_cpupid: IDs of last CPU and last process that accessed the folio.
  * @_entire_mapcount: Do not use directly, call folio_entire_mapcount().
@@ -338,6 +342,7 @@ struct folio {
        /* private: */
                                };
        /* public: */
+                               struct dev_pagemap *pgmap;
                        };
                        struct address_space *mapping;
                        pgoff_t index;

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 9540b41894da6d67751506614e90bed142e127b4..8aecbbb0b685f379bcbb2b330ec90bc5f8869f26 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -1158,6 +1158,12 @@ static inline bool is_zone_device_page(const struct page *page)
        return page_zonenum(page) == ZONE_DEVICE;
 }
 
+static inline struct dev_pagemap *page_pgmap(const struct page *page)
+{
+       VM_WARN_ON_ONCE_PAGE(!is_zone_device_page(page), page);
+       return page_folio(page)->pgmap;
+}
+
 /*
  * Consecutive zone device pages should not be merged into the same sgl
  * or bvec segment with other types of pages or if they belong to different
@@ -1173,7 +1179,7 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
                return false;
        if (!is_zone_device_page(a))
                return true;
-       return a->pgmap == b->pgmap;
+       return page_pgmap(a) == page_pgmap(b);
 }
 
 extern void memmap_init_zone_device(struct zone *, unsigned long,
@@ -1188,6 +1194,10 @@ static inline bool zone_device_pages_have_same_pgmap(const struct page *a,
 {
        return true;
 }
+static inline struct dev_pagemap *page_pgmap(const struct page *page)
+{
+       return NULL;
+}
 #endif
 
 static inline bool folio_is_zone_device(const struct folio *folio)

diff --git a/lib/test_hmm.c b/lib/test_hmm.c
index e4afca8d18802d8896456f219c8993b712f50164..155b18cd9f2afba75aa506858bafafe701e63428 100644 (file)
--- a/lib/test_hmm.c
+++ b/lib/test_hmm.c
@@ -195,7 +195,8 @@ static int dmirror_fops_release(struct inode *inode, struct file *filp)
 
 static struct dmirror_chunk *dmirror_page_to_chunk(struct page *page)
 {
-       return container_of(page->pgmap, struct dmirror_chunk, pagemap);
+       return container_of(page_pgmap(page), struct dmirror_chunk,
+                           pagemap);
 }
 
 static struct dmirror_device *dmirror_page_to_device(struct page *page)

diff --git a/mm/hmm.c b/mm/hmm.c
index 7e0229ae4a5a6bfc06bc8fde06abf31f6198093c..082f7b7c0b9ebc40fd4cc1867ca55699348d46f8 100644 (file)
--- a/mm/hmm.c
+++ b/mm/hmm.c
@@ -248,7 +248,7 @@ static int hmm_vma_handle_pte(struct mm_walk *walk, unsigned long addr,
                 * just report the PFN.
                 */
                if (is_device_private_entry(entry) &&
-                   pfn_swap_entry_to_page(entry)->pgmap->owner ==
+                   page_pgmap(pfn_swap_entry_to_page(entry))->owner ==
                    range->dev_private_owner) {
                        cpu_flags = HMM_PFN_VALID;
                        if (is_writable_device_private_entry(entry))

diff --git a/mm/memory.c b/mm/memory.c
index f1d676c7ca586fbb0cce28e808c952a6a2d3c983..f0fa2a6340696de1afc673f55f33444ce08bae4f 100644 (file)
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -4318,6 +4318,7 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
                        vmf->page = pfn_swap_entry_to_page(entry);
                        ret = remove_device_exclusive_entry(vmf);
                } else if (is_device_private_entry(entry)) {
+                       struct dev_pagemap *pgmap;
                        if (vmf->flags & FAULT_FLAG_VMA_LOCK) {
                                /*
                                 * migrate_to_ram is not yet ready to operate
@@ -4342,7 +4343,8 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
                         */
                        get_page(vmf->page);
                        pte_unmap_unlock(vmf->pte, vmf->ptl);
-                       ret = vmf->page->pgmap->ops->migrate_to_ram(vmf);
+                       pgmap = page_pgmap(vmf->page);
+                       ret = pgmap->ops->migrate_to_ram(vmf);
                        put_page(vmf->page);
                } else if (is_hwpoison_entry(entry)) {
                        ret = VM_FAULT_HWPOISON;

diff --git a/mm/memremap.c b/mm/memremap.c
index 07bbe0eed084a9bccbd9bed7210b82af3f83dbeb..68099af9df4cdf1288f19e82a91e1883f8731f0d 100644 (file)
--- a/mm/memremap.c
+++ b/mm/memremap.c
@@ -458,8 +458,8 @@ EXPORT_SYMBOL_GPL(get_dev_pagemap);
 
 void free_zone_device_folio(struct folio *folio)
 {
-       if (WARN_ON_ONCE(!folio->page.pgmap->ops ||
-                       !folio->page.pgmap->ops->page_free))
+       if (WARN_ON_ONCE(!folio->pgmap->ops ||
+                       !folio->pgmap->ops->page_free))
                return;
 
        mem_cgroup_uncharge(folio);
@@ -486,12 +486,12 @@ void free_zone_device_folio(struct folio *folio)
         * to clear folio->mapping.
         */
        folio->mapping = NULL;
-       folio->page.pgmap->ops->page_free(folio_page(folio, 0));
+       folio->pgmap->ops->page_free(folio_page(folio, 0));
 
-       switch (folio->page.pgmap->type) {
+       switch (folio->pgmap->type) {
        case MEMORY_DEVICE_PRIVATE:
        case MEMORY_DEVICE_COHERENT:
-               put_dev_pagemap(folio->page.pgmap);
+               put_dev_pagemap(folio->pgmap);
                break;
 
        case MEMORY_DEVICE_FS_DAX:
@@ -514,7 +514,7 @@ void zone_device_page_init(struct page *page)
         * Drivers shouldn't be allocating pages after calling
         * memunmap_pages().
         */
-       WARN_ON_ONCE(!percpu_ref_tryget_live(&page->pgmap->ref));
+       WARN_ON_ONCE(!percpu_ref_tryget_live(&page_pgmap(page)->ref));
        set_page_count(page, 1);
        lock_page(page);
 }
@@ -523,7 +523,7 @@ EXPORT_SYMBOL_GPL(zone_device_page_init);
 #ifdef CONFIG_FS_DAX
 bool __put_devmap_managed_folio_refs(struct folio *folio, int refs)
 {
-       if (folio->page.pgmap->type != MEMORY_DEVICE_FS_DAX)
+       if (folio->pgmap->type != MEMORY_DEVICE_FS_DAX)
                return false;
 
        /*

diff --git a/mm/migrate_device.c b/mm/migrate_device.c
index 5bd888223cc8b8d0ef89419507a87e5ba82c291a..6771893d46014b467390f34abef4b9dc0278e525 100644 (file)
--- a/mm/migrate_device.c
+++ b/mm/migrate_device.c
@@ -106,6 +106,7 @@ again:
        arch_enter_lazy_mmu_mode();
 
        for (; addr < end; addr += PAGE_SIZE, ptep++) {
+               struct dev_pagemap *pgmap;
                unsigned long mpfn = 0, pfn;
                struct folio *folio;
                struct page *page;
@@ -133,9 +134,10 @@ again:
                                goto next;
 
                        page = pfn_swap_entry_to_page(entry);
+                       pgmap = page_pgmap(page);
                        if (!(migrate->flags &
                                MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
-                           page->pgmap->owner != migrate->pgmap_owner)
+                           pgmap->owner != migrate->pgmap_owner)
                                goto next;
 
                        mpfn = migrate_pfn(page_to_pfn(page)) |
@@ -151,12 +153,13 @@ again:
                                goto next;
                        }
                        page = vm_normal_page(migrate->vma, addr, pte);
+                       pgmap = page_pgmap(page);
                        if (page && !is_zone_device_page(page) &&
                            !(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
                                goto next;
                        else if (page && is_device_coherent_page(page) &&
                            (!(migrate->flags & MIGRATE_VMA_SELECT_DEVICE_COHERENT) ||
-                            page->pgmap->owner != migrate->pgmap_owner))
+                            pgmap->owner != migrate->pgmap_owner))
                                goto next;
                        mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
                        mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;

diff --git a/mm/mlock.c b/mm/mlock.c
index cde076fa7d5e5a7ee3897fa66812d907156aede1..3cb72b579ffd3304f52579e22f61e5c1a8e6b08f 100644 (file)
--- a/mm/mlock.c
+++ b/mm/mlock.c
@@ -368,6 +368,8 @@ static int mlock_pte_range(pmd_t *pmd, unsigned long addr,
                if (is_huge_zero_pmd(*pmd))
                        goto out;
                folio = pmd_folio(*pmd);
+               if (folio_is_zone_device(folio))
+                       goto out;
                if (vma->vm_flags & VM_LOCKED)
                        mlock_folio(folio);
                else

diff --git a/mm/mm_init.c b/mm/mm_init.c
index 6be97965aecce4edf8544ba1321a6a93bc364247..d0b5bef39fadc41dea2a631b0e1f098283713e92 100644 (file)
--- a/mm/mm_init.c
+++ b/mm/mm_init.c
@@ -998,7 +998,7 @@ static void __ref __init_zone_device_page(struct page *page, unsigned long pfn,
         * and zone_device_data.  It is a bug if a ZONE_DEVICE page is
         * ever freed or placed on a driver-private list.
         */
-       page->pgmap = pgmap;
+       page_folio(page)->pgmap = pgmap;
        page->zone_device_data = NULL;
 
        /*