mm/slub: Convert kfree() to use a struct slab

With kfree() using a struct slab, we can also convert slab_free()
and do_slab_free() to use a slab instead of a page.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>

diff --git a/mm/slub.c b/mm/slub.c
index 050a0610b3ef6cdd97d1feef541aa3b15602dac8..15996ea165acad556a66b4e3066ece890049d942 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -3402,11 +3402,11 @@ slab_empty:
  * with all sorts of special processing.
  *
  * Bulk free of a freelist with several objects (all pointing to the
- * same page) possible by specifying head and tail ptr, plus objects
+ * same slab) possible by specifying head and tail ptr, plus objects
  * count (cnt). Bulk free indicated by tail pointer being set.
  */
 static __always_inline void do_slab_free(struct kmem_cache *s,
-                               struct page *page, void *head, void *tail,
+                               struct slab *slab, void *head, void *tail,
                                int cnt, unsigned long addr)
 {
        void *tail_obj = tail ? : head;
@@ -3427,7 +3427,7 @@ redo:
        /* Same with comment on barrier() in slab_alloc_node() */
        barrier();
 
-       if (likely(page == c->page)) {
+       if (likely(slab_page(slab) == c->page)) {
 #ifndef CONFIG_PREEMPT_RT
                void **freelist = READ_ONCE(c->freelist);
 
@@ -3453,7 +3453,7 @@ redo:
 
                local_lock(&s->cpu_slab->lock);
                c = this_cpu_ptr(s->cpu_slab);
-               if (unlikely(page != c->page)) {
+               if (unlikely(slab_page(slab) != c->page)) {
                        local_unlock(&s->cpu_slab->lock);
                        goto redo;
                }
@@ -3468,11 +3468,11 @@ redo:
 #endif
                stat(s, FREE_FASTPATH);
        } else
-               __slab_free(s, page, head, tail_obj, cnt, addr);
+               __slab_free(s, slab_page(slab), head, tail_obj, cnt, addr);
 
 }
 
-static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
+static __always_inline void slab_free(struct kmem_cache *s, struct slab *slab,
                                      void *head, void *tail, int cnt,
                                      unsigned long addr)
 {
@@ -3481,13 +3481,13 @@ static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
         * to remove objects, whose reuse must be delayed.
         */
        if (slab_free_freelist_hook(s, &head, &tail))
-               do_slab_free(s, page, head, tail, cnt, addr);
+               do_slab_free(s, slab, head, tail, cnt, addr);
 }
 
 #ifdef CONFIG_KASAN_GENERIC
 void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
 {
-       do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
+       do_slab_free(cache, virt_to_slab(x), x, NULL, 1, addr);
 }
 #endif
 
@@ -3496,7 +3496,7 @@ void kmem_cache_free(struct kmem_cache *s, void *x)
        s = cache_from_obj(s, x);
        if (!s)
                return;
-       slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
+       slab_free(s, virt_to_slab(x), x, NULL, 1, _RET_IP_);
        trace_kmem_cache_free(_RET_IP_, x, s->name);
 }
 EXPORT_SYMBOL(kmem_cache_free);
@@ -3621,7 +3621,7 @@ void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
                if (!df.slab)
                        continue;
 
-               slab_free(df.s, slab_page(df.slab), df.freelist, df.tail, df.cnt, _RET_IP_);
+               slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt, _RET_IP_);
        } while (likely(size));
 }
 EXPORT_SYMBOL(kmem_cache_free_bulk);
@@ -4527,7 +4527,7 @@ EXPORT_SYMBOL(__ksize);
 
 void kfree(const void *x)
 {
-       struct page *page;
+       struct slab *slab;
        void *object = (void *)x;
 
        trace_kfree(_RET_IP_, x);
@@ -4535,12 +4535,12 @@ void kfree(const void *x)
        if (unlikely(ZERO_OR_NULL_PTR(x)))
                return;
 
-       page = virt_to_head_page(x);
-       if (unlikely(!PageSlab(page))) {
-               free_nonslab_page(page, object);
+       slab = virt_to_slab(x);
+       if (unlikely(!SlabAllocation(slab))) {
+               free_nonslab_page(slab_page(slab), object);
                return;
        }
-       slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
+       slab_free(slab->slab_cache, slab, object, NULL, 1, _RET_IP_);
 }
 EXPORT_SYMBOL(kfree);