mm/memcg: Add folio_lruvec_lock() and similar functions

These are the folio equivalents of lock_page_lruvec() and similar
functions.  Also convert lruvec_memcg_debug() to take a folio.

Signed-off-by: Matthew Wilcox (Oracle) <willy@infradead.org>
Reviewed-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: David Howells <dhowells@redhat.com>
Acked-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h
index 49b8eff3b685c8f46ac24c96d10760ec6b2711bb..8c24f5b8638b48c60e3b8fc8752990816cd1d05b 100644 (file)
--- a/include/linux/memcontrol.h
+++ b/include/linux/memcontrol.h
@@ -767,15 +767,16 @@ struct mem_cgroup *mem_cgroup_from_task(struct task_struct *p);
 
 struct mem_cgroup *get_mem_cgroup_from_mm(struct mm_struct *mm);
 
-struct lruvec *lock_page_lruvec(struct page *page);
-struct lruvec *lock_page_lruvec_irq(struct page *page);
-struct lruvec *lock_page_lruvec_irqsave(struct page *page,
+struct lruvec *folio_lruvec_lock(struct folio *folio);
+struct lruvec *folio_lruvec_lock_irq(struct folio *folio);
+struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
                                                unsigned long *flags);
 
 #ifdef CONFIG_DEBUG_VM
-void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page);
+void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio);
 #else
-static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
+static inline
+void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
 {
 }
 #endif
@@ -1230,7 +1231,8 @@ static inline struct lruvec *folio_lruvec(struct folio *folio)
        return &pgdat->__lruvec;
 }
 
-static inline void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
+static inline
+void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
 {
 }
 
@@ -1260,26 +1262,26 @@ static inline void mem_cgroup_put(struct mem_cgroup *memcg)
 {
 }
 
-static inline struct lruvec *lock_page_lruvec(struct page *page)
+static inline struct lruvec *folio_lruvec_lock(struct folio *folio)
 {
-       struct pglist_data *pgdat = page_pgdat(page);
+       struct pglist_data *pgdat = folio_pgdat(folio);
 
        spin_lock(&pgdat->__lruvec.lru_lock);
        return &pgdat->__lruvec;
 }
 
-static inline struct lruvec *lock_page_lruvec_irq(struct page *page)
+static inline struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
 {
-       struct pglist_data *pgdat = page_pgdat(page);
+       struct pglist_data *pgdat = folio_pgdat(folio);
 
        spin_lock_irq(&pgdat->__lruvec.lru_lock);
        return &pgdat->__lruvec;
 }
 
-static inline struct lruvec *lock_page_lruvec_irqsave(struct page *page,
+static inline struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
                unsigned long *flagsp)
 {
-       struct pglist_data *pgdat = page_pgdat(page);
+       struct pglist_data *pgdat = folio_pgdat(folio);
 
        spin_lock_irqsave(&pgdat->__lruvec.lru_lock, *flagsp);
        return &pgdat->__lruvec;
@@ -1536,6 +1538,7 @@ static inline bool page_matches_lruvec(struct page *page, struct lruvec *lruvec)
 static inline struct lruvec *relock_page_lruvec_irq(struct page *page,
                struct lruvec *locked_lruvec)
 {
+       struct folio *folio = page_folio(page);
        if (locked_lruvec) {
                if (page_matches_lruvec(page, locked_lruvec))
                        return locked_lruvec;
@@ -1543,13 +1546,14 @@ static inline struct lruvec *relock_page_lruvec_irq(struct page *page,
                unlock_page_lruvec_irq(locked_lruvec);
        }
 
-       return lock_page_lruvec_irq(page);
+       return folio_lruvec_lock_irq(folio);
 }
 
 /* Don't lock again iff page's lruvec locked */
 static inline struct lruvec *relock_page_lruvec_irqsave(struct page *page,
                struct lruvec *locked_lruvec, unsigned long *flags)
 {
+       struct folio *folio = page_folio(page);
        if (locked_lruvec) {
                if (page_matches_lruvec(page, locked_lruvec))
                        return locked_lruvec;
@@ -1557,7 +1561,7 @@ static inline struct lruvec *relock_page_lruvec_irqsave(struct page *page,
                unlock_page_lruvec_irqrestore(locked_lruvec, *flags);
        }
 
-       return lock_page_lruvec_irqsave(page, flags);
+       return folio_lruvec_lock_irqsave(folio, flags);
 }
 
 #ifdef CONFIG_CGROUP_WRITEBACK

diff --git a/mm/compaction.c b/mm/compaction.c
index a88f7b893f80a2a51794bdd1de8ca4904ef87c28..6f77577be2483716d319bbf1e75440dc52207c76 100644 (file)
--- a/mm/compaction.c
+++ b/mm/compaction.c
@@ -1038,7 +1038,7 @@ isolate_migratepages_block(struct compact_control *cc, unsigned long low_pfn,
                        compact_lock_irqsave(&lruvec->lru_lock, &flags, cc);
                        locked = lruvec;
 
-                       lruvec_memcg_debug(lruvec, page);
+                       lruvec_memcg_debug(lruvec, page_folio(page));
 
                        /* Try get exclusive access under lock */
                        if (!skip_updated) {

diff --git a/mm/huge_memory.c b/mm/huge_memory.c
index ecb1fb1f5f3e14757d3d496a1941e8ad35d5dedf..763bf687ca92a370dba4bc6f6d6da301dfaa9874 100644 (file)
--- a/mm/huge_memory.c
+++ b/mm/huge_memory.c
@@ -2431,7 +2431,8 @@ static void __split_huge_page_tail(struct page *head, int tail,
 static void __split_huge_page(struct page *page, struct list_head *list,
                pgoff_t end)
 {
-       struct page *head = compound_head(page);
+       struct folio *folio = page_folio(page);
+       struct page *head = &folio->page;
        struct lruvec *lruvec;
        struct address_space *swap_cache = NULL;
        unsigned long offset = 0;
@@ -2450,7 +2451,7 @@ static void __split_huge_page(struct page *page, struct list_head *list,
        }
 
        /* lock lru list/PageCompound, ref frozen by page_ref_freeze */
-       lruvec = lock_page_lruvec(head);
+       lruvec = folio_lruvec_lock(folio);
 
        for (i = nr - 1; i >= 1; i--) {
                __split_huge_page_tail(head, i, lruvec, list);

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 915a7b359075da02d5b3050c714649a193b13151..c954fda9d7f4127c4182e8b51d47eac58a06cfbd 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -1158,67 +1158,88 @@ int mem_cgroup_scan_tasks(struct mem_cgroup *memcg,
 }
 
 #ifdef CONFIG_DEBUG_VM
-void lruvec_memcg_debug(struct lruvec *lruvec, struct page *page)
+void lruvec_memcg_debug(struct lruvec *lruvec, struct folio *folio)
 {
        struct mem_cgroup *memcg;
 
        if (mem_cgroup_disabled())
                return;
 
-       memcg = page_memcg(page);
+       memcg = folio_memcg(folio);
 
        if (!memcg)
-               VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != root_mem_cgroup, page);
+               VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != root_mem_cgroup, folio);
        else
-               VM_BUG_ON_PAGE(lruvec_memcg(lruvec) != memcg, page);
+               VM_BUG_ON_FOLIO(lruvec_memcg(lruvec) != memcg, folio);
 }
 #endif
 
 /**
- * lock_page_lruvec - lock and return lruvec for a given page.
- * @page: the page
+ * folio_lruvec_lock - Lock the lruvec for a folio.
+ * @folio: Pointer to the folio.
  *
  * These functions are safe to use under any of the following conditions:
- * - page locked
- * - PageLRU cleared
- * - lock_page_memcg()
- * - page->_refcount is zero
+ * - folio locked
+ * - folio_test_lru false
+ * - folio_memcg_lock()
+ * - folio frozen (refcount of 0)
+ *
+ * Return: The lruvec this folio is on with its lock held.
  */
-struct lruvec *lock_page_lruvec(struct page *page)
+struct lruvec *folio_lruvec_lock(struct folio *folio)
 {
-       struct folio *folio = page_folio(page);
-       struct lruvec *lruvec;
+       struct lruvec *lruvec = folio_lruvec(folio);
 
-       lruvec = folio_lruvec(folio);
        spin_lock(&lruvec->lru_lock);
-
-       lruvec_memcg_debug(lruvec, page);
+       lruvec_memcg_debug(lruvec, folio);
 
        return lruvec;
 }
 
-struct lruvec *lock_page_lruvec_irq(struct page *page)
+/**
+ * folio_lruvec_lock_irq - Lock the lruvec for a folio.
+ * @folio: Pointer to the folio.
+ *
+ * These functions are safe to use under any of the following conditions:
+ * - folio locked
+ * - folio_test_lru false
+ * - folio_memcg_lock()
+ * - folio frozen (refcount of 0)
+ *
+ * Return: The lruvec this folio is on with its lock held and interrupts
+ * disabled.
+ */
+struct lruvec *folio_lruvec_lock_irq(struct folio *folio)
 {
-       struct folio *folio = page_folio(page);
-       struct lruvec *lruvec;
+       struct lruvec *lruvec = folio_lruvec(folio);
 
-       lruvec = folio_lruvec(folio);
        spin_lock_irq(&lruvec->lru_lock);
-
-       lruvec_memcg_debug(lruvec, page);
+       lruvec_memcg_debug(lruvec, folio);
 
        return lruvec;
 }
 
-struct lruvec *lock_page_lruvec_irqsave(struct page *page, unsigned long *flags)
+/**
+ * folio_lruvec_lock_irqsave - Lock the lruvec for a folio.
+ * @folio: Pointer to the folio.
+ * @flags: Pointer to irqsave flags.
+ *
+ * These functions are safe to use under any of the following conditions:
+ * - folio locked
+ * - folio_test_lru false
+ * - folio_memcg_lock()
+ * - folio frozen (refcount of 0)
+ *
+ * Return: The lruvec this folio is on with its lock held and interrupts
+ * disabled.
+ */
+struct lruvec *folio_lruvec_lock_irqsave(struct folio *folio,
+               unsigned long *flags)
 {
-       struct folio *folio = page_folio(page);
-       struct lruvec *lruvec;
+       struct lruvec *lruvec = folio_lruvec(folio);
 
-       lruvec = folio_lruvec(folio);
        spin_lock_irqsave(&lruvec->lru_lock, *flags);
-
-       lruvec_memcg_debug(lruvec, page);
+       lruvec_memcg_debug(lruvec, folio);
 
        return lruvec;
 }

diff --git a/mm/rmap.c b/mm/rmap.c
index b9eb5c12f3fe10bc83ce0d26d3f804d7f252f517..1df8683c4c4ccbef8c1197c107ad264e64b1864c 100644 (file)
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -33,7 +33,7 @@
  *                 mapping->private_lock (in __set_page_dirty_buffers)
  *                   lock_page_memcg move_lock (in __set_page_dirty_buffers)
  *                     i_pages lock (widely used)
- *                       lruvec->lru_lock (in lock_page_lruvec_irq)
+ *                       lruvec->lru_lock (in folio_lruvec_lock_irq)
  *                 inode->i_lock (in set_page_dirty's __mark_inode_dirty)
  *                 bdi.wb->list_lock (in set_page_dirty's __mark_inode_dirty)
  *                   sb_lock (within inode_lock in fs/fs-writeback.c)

diff --git a/mm/swap.c b/mm/swap.c
index 4ba77fc8da4f85d8207cbee8eff089adedc253a6..6d0d2bfca48e9626a674100a936d758d4e29605b 100644 (file)
--- a/mm/swap.c
+++ b/mm/swap.c
@@ -80,10 +80,11 @@ static DEFINE_PER_CPU(struct lru_pvecs, lru_pvecs) = {
 static void __page_cache_release(struct page *page)
 {
        if (PageLRU(page)) {
+               struct folio *folio = page_folio(page);
                struct lruvec *lruvec;
                unsigned long flags;
 
-               lruvec = lock_page_lruvec_irqsave(page, &flags);
+               lruvec = folio_lruvec_lock_irqsave(folio, &flags);
                del_page_from_lru_list(page, lruvec);
                __clear_page_lru_flags(page);
                unlock_page_lruvec_irqrestore(lruvec, flags);
@@ -372,11 +373,12 @@ static inline void activate_page_drain(int cpu)
 
 static void activate_page(struct page *page)
 {
+       struct folio *folio = page_folio(page);
        struct lruvec *lruvec;
 
-       page = compound_head(page);
+       page = &folio->page;
        if (TestClearPageLRU(page)) {
-               lruvec = lock_page_lruvec_irq(page);
+               lruvec = folio_lruvec_lock_irq(folio);
                __activate_page(page, lruvec);
                unlock_page_lruvec_irq(lruvec);
                SetPageLRU(page);

diff --git a/mm/vmscan.c b/mm/vmscan.c
index 4620df62f0ffa1e6e631c041f8f236af389619f4..0d48306d37dc89bb7bd2631b125b2017fe0020a6 100644 (file)
--- a/mm/vmscan.c
+++ b/mm/vmscan.c
@@ -1965,6 +1965,7 @@ static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  */
 int isolate_lru_page(struct page *page)
 {
+       struct folio *folio = page_folio(page);
        int ret = -EBUSY;
 
        VM_BUG_ON_PAGE(!page_count(page), page);
@@ -1974,7 +1975,7 @@ int isolate_lru_page(struct page *page)
                struct lruvec *lruvec;
 
                get_page(page);
-               lruvec = lock_page_lruvec_irq(page);
+               lruvec = folio_lruvec_lock_irq(folio);
                del_page_from_lru_list(page, lruvec);
                unlock_page_lruvec_irq(lruvec);
                ret = 0;