mm, swap: use a single page for swap table when the size fits

We have a cluster size of 512 slots.  Each slot consumes 8 bytes in swap
table so the swap table size of each cluster is exactly one page (4K).

If that condition is true, allocate one page direct and disable the slab
cache to reduce the memory usage of swap table and avoid fragmentation.

Link: https://lkml.kernel.org/r/20250910160833.3464-16-ryncsn@gmail.com
Co-developed-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Chris Li <chrisl@kernel.org>
Signed-off-by: Kairui Song <kasong@tencent.com>
Acked-by: Chris Li <chrisl@kernel.org>
Cc: Baolin Wang <baolin.wang@linux.alibaba.com>
Cc: Baoquan He <bhe@redhat.com>
Cc: Barry Song <baohua@kernel.org>
Cc: David Hildenbrand <david@redhat.com>
Cc: "Huang, Ying" <ying.huang@linux.alibaba.com>
Cc: Hugh Dickins <hughd@google.com>
Cc: Johannes Weiner <hannes@cmpxchg.org>
Cc: Kemeng Shi <shikemeng@huaweicloud.com>
Cc: Lorenzo Stoakes <lorenzo.stoakes@oracle.com>
Cc: Matthew Wilcox (Oracle) <willy@infradead.org>
Cc: Nhat Pham <nphamcs@gmail.com>
Cc: Yosry Ahmed <yosryahmed@google.com>
Cc: Zi Yan <ziy@nvidia.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>

diff --git a/mm/swap_table.h b/mm/swap_table.h
index 52254e455304c588ff5e62955df733cad780cdbb..ea244a57a5b7a3ca6c545a791129dd3e35b3b412 100644 (file)
--- a/mm/swap_table.h
+++ b/mm/swap_table.h
@@ -11,6 +11,8 @@ struct swap_table {
        atomic_long_t entries[SWAPFILE_CLUSTER];
 };
 
+#define SWP_TABLE_USE_PAGE (sizeof(struct swap_table) == PAGE_SIZE)
+
 /*
  * A swap table entry represents the status of a swap slot on a swap
  * (physical or virtual) device. The swap table in each cluster is a

diff --git a/mm/swapfile.c b/mm/swapfile.c
index faf867a6c5c19e564d5a21fb29fd0108179a236e..cebf2dcfaab0220dfe92366be7a16d89e7c23aa2 100644 (file)
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -430,6 +430,38 @@ static inline unsigned int cluster_offset(struct swap_info_struct *si,
        return cluster_index(si, ci) * SWAPFILE_CLUSTER;
 }
 
+static struct swap_table *swap_table_alloc(gfp_t gfp)
+{
+       struct folio *folio;
+
+       if (!SWP_TABLE_USE_PAGE)
+               return kmem_cache_zalloc(swap_table_cachep, gfp);
+
+       folio = folio_alloc(gfp | __GFP_ZERO, 0);
+       if (folio)
+               return folio_address(folio);
+       return NULL;
+}
+
+static void swap_table_free_folio_rcu_cb(struct rcu_head *head)
+{
+       struct folio *folio;
+
+       folio = page_folio(container_of(head, struct page, rcu_head));
+       folio_put(folio);
+}
+
+static void swap_table_free(struct swap_table *table)
+{
+       if (!SWP_TABLE_USE_PAGE) {
+               kmem_cache_free(swap_table_cachep, table);
+               return;
+       }
+
+       call_rcu(&(folio_page(virt_to_folio(table), 0)->rcu_head),
+                swap_table_free_folio_rcu_cb);
+}
+
 static void swap_cluster_free_table(struct swap_cluster_info *ci)
 {
        unsigned int ci_off;
@@ -443,7 +475,7 @@ static void swap_cluster_free_table(struct swap_cluster_info *ci)
        table = (void *)rcu_dereference_protected(ci->table, true);
        rcu_assign_pointer(ci->table, NULL);
 
-       kmem_cache_free(swap_table_cachep, table);
+       swap_table_free(table);
 }
 
 /*
@@ -467,8 +499,7 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
        lockdep_assert_held(&ci->lock);
        lockdep_assert_held(&this_cpu_ptr(&percpu_swap_cluster)->lock);
 
-       table = kmem_cache_zalloc(swap_table_cachep,
-                                 __GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
+       table = swap_table_alloc(__GFP_HIGH | __GFP_NOMEMALLOC | __GFP_NOWARN);
        if (table) {
                rcu_assign_pointer(ci->table, table);
                return ci;
@@ -483,7 +514,7 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
        if (!(si->flags & SWP_SOLIDSTATE))
                spin_unlock(&si->global_cluster_lock);
        local_unlock(&percpu_swap_cluster.lock);
-       table = kmem_cache_zalloc(swap_table_cachep, __GFP_HIGH | GFP_KERNEL);
+       table = swap_table_alloc(__GFP_HIGH | GFP_KERNEL);
 
        local_lock(&percpu_swap_cluster.lock);
        if (!(si->flags & SWP_SOLIDSTATE))
@@ -520,7 +551,7 @@ swap_cluster_alloc_table(struct swap_info_struct *si,
 
 free_table:
        if (table)
-               kmem_cache_free(swap_table_cachep, table);
+               swap_table_free(table);
        return ci;
 }
 
@@ -738,7 +769,7 @@ static int inc_cluster_info_page(struct swap_info_struct *si,
 
        ci = cluster_info + idx;
        if (!ci->table) {
-               table = kmem_cache_zalloc(swap_table_cachep, GFP_KERNEL);
+               table = swap_table_alloc(GFP_KERNEL);
                if (!table)
                        return -ENOMEM;
                rcu_assign_pointer(ci->table, table);
@@ -4076,9 +4107,10 @@ static int __init swapfile_init(void)
         * only, and all swap cache readers (swap_cache_*) verifies
         * the content before use. So it's safe to use RCU slab here.
         */
-       swap_table_cachep = kmem_cache_create("swap_table",
-                           sizeof(struct swap_table),
-                           0, SLAB_PANIC | SLAB_TYPESAFE_BY_RCU, NULL);
+       if (!SWP_TABLE_USE_PAGE)
+               swap_table_cachep = kmem_cache_create("swap_table",
+                                   sizeof(struct swap_table),
+                                   0, SLAB_PANIC | SLAB_TYPESAFE_BY_RCU, NULL);
 
 #ifdef CONFIG_MIGRATION
        if (swapfile_maximum_size >= (1UL << SWP_MIG_TOTAL_BITS))