slab: Allocate frozen pages

Since slab does not use the page refcount, it can allocate and
free frozen pages, saving one atomic operation per free.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: William Kucharski <william.kucharski@oracle.com>

diff --git a/mm/slab.c b/mm/slab.c
index 7a269db050eed88b5273d2c3820ff8e453de518b..87225fa142f7047278cf9acee015711d9272d25d 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -1355,25 +1355,25 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid)
 static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
                                                                int nodeid)
 {
-       struct folio *folio;
+       struct page *page;
        struct slab *slab;
 
        flags |= cachep->allocflags;
 
-       folio = (struct folio *) __alloc_pages_node(nodeid, flags, cachep->gfporder);
-       if (!folio) {
+       page = __alloc_frozen_pages(flags, cachep->gfporder, nodeid, NULL);
+       if (!page) {
                slab_out_of_memory(cachep, flags, nodeid);
                return NULL;
        }
 
-       slab = folio_slab(folio);
-
-       account_slab(slab, cachep->gfporder, cachep, flags);
-       __folio_set_slab(folio);
+       __SetPageSlab(page);
        /* Make the flag visible before any changes to folio->mapping */
        smp_wmb();
+       slab = (struct slab *)page;
+
+       account_slab(slab, cachep->gfporder, cachep, flags);
        /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */
-       if (sk_memalloc_socks() && page_is_pfmemalloc(folio_page(folio, 0)))
+       if (sk_memalloc_socks() && page_is_pfmemalloc(page))
                slab_set_pfmemalloc(slab);
 
        return slab;
@@ -1385,20 +1385,19 @@ static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags,
 static void kmem_freepages(struct kmem_cache *cachep, struct slab *slab)
 {
        int order = cachep->gfporder;
-       struct folio *folio = slab_folio(slab);
+       struct page *page = (struct page *)slab;
 
-       BUG_ON(!folio_test_slab(folio));
        __slab_clear_pfmemalloc(slab);
-       page_mapcount_reset(folio_page(folio, 0));
-       folio->mapping = NULL;
+       page_mapcount_reset(page);
+       page->mapping = NULL;
        /* Make the mapping reset visible before clearing the flag */
        smp_wmb();
-       __folio_clear_slab(folio);
+       __ClearPageSlab(page);
 
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += 1 << order;
        unaccount_slab(slab, order, cachep);
-       __free_pages(folio_page(folio, 0), order);
+       free_frozen_pages(page, order);
 }
 
 static void kmem_rcu_free(struct rcu_head *head)