From c1eac6aedd0ecec280328df9da5574876fda40ca Mon Sep 17 00:00:00 2001 From: "Matthew Wilcox (Oracle)" Date: Mon, 30 May 2022 23:02:44 -0400 Subject: [PATCH] slab: Allocate frozen pages Since slab does not use the page refcount, it can allocate and free frozen pages, saving one atomic operation per free. Signed-off-by: Matthew Wilcox (Oracle) Reviewed-by: William Kucharski --- mm/slab.c | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/mm/slab.c b/mm/slab.c index 7a269db050eed..87225fa142f70 100644 --- a/mm/slab.c +++ b/mm/slab.c @@ -1355,25 +1355,25 @@ slab_out_of_memory(struct kmem_cache *cachep, gfp_t gfpflags, int nodeid) static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, int nodeid) { - struct folio *folio; + struct page *page; struct slab *slab; flags |= cachep->allocflags; - folio = (struct folio *) __alloc_pages_node(nodeid, flags, cachep->gfporder); - if (!folio) { + page = __alloc_frozen_pages(flags, cachep->gfporder, nodeid, NULL); + if (!page) { slab_out_of_memory(cachep, flags, nodeid); return NULL; } - slab = folio_slab(folio); - - account_slab(slab, cachep->gfporder, cachep, flags); - __folio_set_slab(folio); + __SetPageSlab(page); /* Make the flag visible before any changes to folio->mapping */ smp_wmb(); + slab = (struct slab *)page; + + account_slab(slab, cachep->gfporder, cachep, flags); /* Record if ALLOC_NO_WATERMARKS was set when allocating the slab */ - if (sk_memalloc_socks() && page_is_pfmemalloc(folio_page(folio, 0))) + if (sk_memalloc_socks() && page_is_pfmemalloc(page)) slab_set_pfmemalloc(slab); return slab; @@ -1385,20 +1385,19 @@ static struct slab *kmem_getpages(struct kmem_cache *cachep, gfp_t flags, static void kmem_freepages(struct kmem_cache *cachep, struct slab *slab) { int order = cachep->gfporder; - struct folio *folio = slab_folio(slab); + struct page *page = (struct page *)slab; - BUG_ON(!folio_test_slab(folio)); __slab_clear_pfmemalloc(slab); - page_mapcount_reset(folio_page(folio, 0)); - folio->mapping = NULL; + page_mapcount_reset(page); + page->mapping = NULL; /* Make the mapping reset visible before clearing the flag */ smp_wmb(); - __folio_clear_slab(folio); + __ClearPageSlab(page); if (current->reclaim_state) current->reclaim_state->reclaimed_slab += 1 << order; unaccount_slab(slab, order, cachep); - __free_pages(folio_page(folio, 0), order); + free_frozen_pages(page, order); } static void kmem_rcu_free(struct rcu_head *head) -- 2.49.0