From: David Woodhouse <dwmw@amazon.co.uk>
Date: Wed, 2 Apr 2025 17:58:36 +0000 (+0100)
Subject: mm: Implement for_each_valid_pfn() for CONFIG_SPARSEMEM
X-Git-Url: https://www.infradead.org/git/?a=commitdiff_plain;h=cf474911fb10bc953fa5e8a6b3eb13ff76481b46;p=users%2Fdwmw2%2Flinux.git
mm: Implement for_each_valid_pfn() for CONFIG_SPARSEMEM
Implement for_each_valid_pfn() based on two helper functions.
The first_valid_pfn() function largely mirrors pfn_valid(), calling into
a pfn_section_first_valid() helper which is trivial for the !VMEMMAP case,
and in the VMEMMAP case will skip to the next subsection as needed.
Since next_valid_pfn() knows that its argument *is* a valid PFN, it
doesn't need to do any checking at all while iterating over the low bits
within a (sub)section mask; the whole (sub)section is either present or
not.
Note that the VMEMMAP version of pfn_section_first_valid() may return a
value *higher* than end_pfn when skipping to the next subsection, and
first_valid_pfn() happily returns that higher value. This is fine.
Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Previous-revision-reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>
---
diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h
index 74d0077cc5faa..044536da33905 100644
--- a/include/asm-generic/memory_model.h
+++ b/include/asm-generic/memory_model.h
@@ -31,12 +31,28 @@ static inline int pfn_valid(unsigned long pfn)
}
#define pfn_valid pfn_valid
+static inline bool first_valid_pfn(unsigned long *pfn)
+{
+ /* avoid <linux/mm.h> include hell */
+ extern unsigned long max_mapnr;
+ unsigned long pfn_offset = ARCH_PFN_OFFSET;
+
+ if (*pfn < pfn_offset) {
+ *pfn = pfn_offset;
+ return true;
+ }
+
+ if ((*pfn - pfn_offset) < max_mapnr)
+ return true;
+
+ return false;
+}
+
#ifndef for_each_valid_pfn
-#define for_each_valid_pfn(pfn, start_pfn, end_pfn) \
- for ((pfn) = max_t(unsigned long, (start_pfn), ARCH_PFN_OFFSET); \
- (pfn) < min_t(unsigned long, (end_pfn), \
- ARCH_PFN_OFFSET + max_mapnr); \
- (pfn)++)
+#define for_each_valid_pfn(pfn, start_pfn, end_pfn) \
+ for (pfn = max_t(unsigned long, start_pfn, ARCH_PFN_OFFSET); \
+ pfn < min_t(unsigned long, end_pfn, ARCH_PFN_OFFSET + max_mapnr); \
+ pfn++)
#endif /* for_each_valid_pfn */
#endif /* valid_pfn */
diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 230a29c2ed1a9..dab1d31477d7c 100644
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2075,11 +2075,37 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
return usage ? test_bit(idx, usage->subsection_map) : 0;
}
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+ struct mem_section_usage *usage = READ_ONCE(ms->usage);
+ int idx = subsection_map_index(*pfn);
+ unsigned long bit;
+
+ if (!usage)
+ return false;
+
+ if (test_bit(idx, usage->subsection_map))
+ return true;
+
+ /* Find the next subsection that exists */
+ bit = find_next_bit(usage->subsection_map, SUBSECTIONS_PER_SECTION, idx);
+ if (bit == SUBSECTIONS_PER_SECTION)
+ return false;
+
+ *pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION);
+ return true;
+}
#else
static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
{
return 1;
}
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+ return true;
+}
#endif
void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
@@ -2128,6 +2154,58 @@ static inline int pfn_valid(unsigned long pfn)
return ret;
}
+
+/* Returns end_pfn or higher if no valid PFN remaining in range */
+static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+ unsigned long nr = pfn_to_section_nr(pfn);
+
+ rcu_read_lock_sched();
+
+ while (nr <= __highest_present_section_nr && pfn < end_pfn) {
+ struct mem_section *ms = __pfn_to_section(pfn);
+
+ if (valid_section(ms) &&
+ (early_section(ms) || pfn_section_first_valid(ms, &pfn))) {
+ rcu_read_unlock_sched();
+ return pfn;
+ }
+
+ /* Nothing left in this section? Skip to next section */
+ nr++;
+ pfn = section_nr_to_pfn(nr);
+ }
+
+ rcu_read_unlock_sched();
+ return end_pfn;
+}
+
+static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+ pfn++;
+
+ if (pfn >= end_pfn)
+ return end_pfn;
+
+ /*
+ * Either every PFN within the section (or subsection for VMEMMAP) is
+ * valid, or none of them are. So there's no point repeating the check
+ * for every PFN; only call first_valid_pfn() the first time, and when
+ * crossing a (sub)section boundary (i.e. !(pfn & ~PFN_VALID_MASK)).
+ */
+ if (pfn & (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) ?
+ PAGE_SUBSECTION_MASK : PAGE_SECTION_MASK))
+ return pfn;
+
+ return first_valid_pfn(pfn, end_pfn);
+}
+
+
+#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn) \
+ for ((_pfn) = first_valid_pfn((_start_pfn), (_end_pfn)); \
+ (_pfn) < (_end_pfn); \
+ (_pfn) = next_valid_pfn((_pfn), (_end_pfn)))
+
#endif
static inline int pfn_in_present_section(unsigned long pfn)