mm, page_owner: record page owner for each subpage

Patch series "debug_pagealloc improvements through page_owner", v2.

The debug_pagealloc functionality serves a similar purpose on the page
allocator level that slub_debug does on the kmalloc level, which is to
detect bad users.  One notable feature that slub_debug has is storing
stack traces of who last allocated and freed the object.  On page level we
track allocations via page_owner, but that info is discarded when freeing,
and we don't track freeing at all.  This series improves those aspects.
With both debug_pagealloc and page_owner enabled, we can then get bug
reports such as the example in Patch 4.

SLUB debug tracking additionally stores cpu, pid and timestamp.  This could
be added later, if deemed useful enough to justify the additional page_ext
structure size.

This patch (of 3):

Currently, page owner info is only recorded for the first page of a
high-order allocation, and copied to tail pages in the event of a split
page.  With the plan to keep previous owner info after freeing the page,
it would be benefical to record page owner for each subpage upon
allocation.  This increases the overhead for high orders, but that should
be acceptable for a debugging option.

The order stored for each subpage is the order of the whole allocation.
This makes it possible to calculate the "head" pfn and to recognize "tail"
pages (quoted because not all high-order allocations are compound pages
with true head and tail pages).  When reading the page_owner debugfs file,
keep skipping the "tail" pages so that stats gathered by existing scripts
don't get inflated.

Link: http://lkml.kernel.org/r/20190820131828.22684-3-vbabka@suse.cz
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Cc: Kirill A. Shutemov <kirill@shutemov.name>
Cc: Matthew Wilcox <willy@infradead.org>
Cc: Mel Gorman <mgorman@techsingularity.net>
Cc: Michal Hocko <mhocko@kernel.org>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/mm/page_owner.c b/mm/page_owner.c
index addcbb2ae4e4f7e8efa3bc1df75a2f77bf9a094a..813fcb70547b74843d9872816d2fd9a6b3c8c7b4 100644 (file)
--- a/mm/page_owner.c
+++ b/mm/page_owner.c
@@ -154,18 +154,23 @@ static noinline depot_stack_handle_t save_stack(gfp_t flags)
        return handle;
 }
 
-static inline void __set_page_owner_handle(struct page_ext *page_ext,
-       depot_stack_handle_t handle, unsigned int order, gfp_t gfp_mask)
+static inline void __set_page_owner_handle(struct page *page,
+       struct page_ext *page_ext, depot_stack_handle_t handle,
+       unsigned int order, gfp_t gfp_mask)
 {
        struct page_owner *page_owner;
+       int i;
 
-       page_owner = get_page_owner(page_ext);
-       page_owner->handle = handle;
-       page_owner->order = order;
-       page_owner->gfp_mask = gfp_mask;
-       page_owner->last_migrate_reason = -1;
+       for (i = 0; i < (1 << order); i++) {
+               page_owner = get_page_owner(page_ext);
+               page_owner->handle = handle;
+               page_owner->order = order;
+               page_owner->gfp_mask = gfp_mask;
+               page_owner->last_migrate_reason = -1;
+               __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
 
-       __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
+               page_ext = lookup_page_ext(page + i);
+       }
 }
 
 noinline void __set_page_owner(struct page *page, unsigned int order,
@@ -178,7 +183,7 @@ noinline void __set_page_owner(struct page *page, unsigned int order,
                return;
 
        handle = save_stack(gfp_mask);
-       __set_page_owner_handle(page_ext, handle, order, gfp_mask);
+       __set_page_owner_handle(page, page_ext, handle, order, gfp_mask);
 }
 
 void __set_page_owner_migrate_reason(struct page *page, int reason)
@@ -204,8 +209,11 @@ void __split_page_owner(struct page *page, unsigned int order)
 
        page_owner = get_page_owner(page_ext);
        page_owner->order = 0;
-       for (i = 1; i < (1 << order); i++)
-               __copy_page_owner(page, page + i);
+       for (i = 1; i < (1 << order); i++) {
+               page_ext = lookup_page_ext(page + i);
+               page_owner = get_page_owner(page_ext);
+               page_owner->order = 0;
+       }
 }
 
 void __copy_page_owner(struct page *oldpage, struct page *newpage)
@@ -483,6 +491,13 @@ read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
 
                page_owner = get_page_owner(page_ext);
 
+               /*
+                * Don't print "tail" pages of high-order allocations as that
+                * would inflate the stats.
+                */
+               if (!IS_ALIGNED(pfn, 1 << page_owner->order))
+                       continue;
+
                /*
                 * Access to page_ext->handle isn't synchronous so we should
                 * be careful to access it.
@@ -562,7 +577,8 @@ static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
                                continue;
 
                        /* Found early allocated page */
-                       __set_page_owner_handle(page_ext, early_handle, 0, 0);
+                       __set_page_owner_handle(page, page_ext, early_handle,
+                                               0, 0);
                        count++;
                }
                cond_resched();