mm: Implement for_each_valid_pfn() for CONFIG_SPARSEMEM

Implement for_each_valid_pfn() based on two helper functions.

The first_valid_pfn() function largely mirrors pfn_valid(), calling into
a pfn_section_first_valid() helper which is trivial for the !VMEMMAP case,
and in the VMEMMAP case will skip to the next subsection as needed.

Since next_valid_pfn() knows that its argument *is* a valid PFN, it
doesn't need to do any checking at all while iterating over the low bits
within a (sub)section mask; the whole (sub)section is either present or
not.

Note that the VMEMMAP version of pfn_section_first_valid() may return a
value *higher* than end_pfn when skipping to the next subsection, and
first_valid_pfn() happily returns that higher value. This is fine.

Signed-off-by: David Woodhouse <dwmw@amazon.co.uk>
Previous-revision-reviewed-by: Mike Rapoport (Microsoft) <rppt@kernel.org>

diff --git a/include/asm-generic/memory_model.h b/include/asm-generic/memory_model.h
index 74d0077cc5faa9beb5efe4b134e71777aba8434b..044536da33905031d03dbf5a016ea7517ce2c141 100644 (file)
--- a/include/asm-generic/memory_model.h
+++ b/include/asm-generic/memory_model.h
@@ -31,12 +31,28 @@ static inline int pfn_valid(unsigned long pfn)
 }
 #define pfn_valid pfn_valid
 
+static inline bool first_valid_pfn(unsigned long *pfn)
+{
+       /* avoid <linux/mm.h> include hell */
+       extern unsigned long max_mapnr;
+       unsigned long pfn_offset = ARCH_PFN_OFFSET;
+
+       if (*pfn < pfn_offset) {
+               *pfn = pfn_offset;
+               return true;
+       }
+
+       if ((*pfn - pfn_offset) < max_mapnr)
+               return true;
+
+       return false;
+}
+
 #ifndef for_each_valid_pfn
-#define for_each_valid_pfn(pfn, start_pfn, end_pfn)                     \
-       for ((pfn) = max_t(unsigned long, (start_pfn), ARCH_PFN_OFFSET); \
-            (pfn) < min_t(unsigned long, (end_pfn),                     \
-                          ARCH_PFN_OFFSET + max_mapnr);                 \
-            (pfn)++)
+#define for_each_valid_pfn(pfn, start_pfn, end_pfn)                           \
+       for (pfn = max_t(unsigned long, start_pfn, ARCH_PFN_OFFSET);    \
+            pfn < min_t(unsigned long, end_pfn, ARCH_PFN_OFFSET + max_mapnr); \
+                        pfn++)
 #endif /* for_each_valid_pfn */
 #endif /* valid_pfn */
 

diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h
index 230a29c2ed1a9b1d83775f81f554d874fe097f7c..dab1d31477d7ccb96bb59e4a72fcd903b6dff4c5 100644 (file)
--- a/include/linux/mmzone.h
+++ b/include/linux/mmzone.h
@@ -2075,11 +2075,37 @@ static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 
        return usage ? test_bit(idx, usage->subsection_map) : 0;
 }
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+       struct mem_section_usage *usage = READ_ONCE(ms->usage);
+       int idx = subsection_map_index(*pfn);
+       unsigned long bit;
+
+       if (!usage)
+               return false;
+
+       if (test_bit(idx, usage->subsection_map))
+               return true;
+
+       /* Find the next subsection that exists */
+       bit = find_next_bit(usage->subsection_map, SUBSECTIONS_PER_SECTION, idx);
+       if (bit == SUBSECTIONS_PER_SECTION)
+               return false;
+
+       *pfn = (*pfn & PAGE_SECTION_MASK) + (bit * PAGES_PER_SUBSECTION);
+       return true;
+}
 #else
 static inline int pfn_section_valid(struct mem_section *ms, unsigned long pfn)
 {
        return 1;
 }
+
+static inline bool pfn_section_first_valid(struct mem_section *ms, unsigned long *pfn)
+{
+       return true;
+}
 #endif
 
 void sparse_init_early_section(int nid, struct page *map, unsigned long pnum,
@@ -2128,6 +2154,58 @@ static inline int pfn_valid(unsigned long pfn)
 
        return ret;
 }
+
+/* Returns end_pfn or higher if no valid PFN remaining in range */
+static inline unsigned long first_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+       unsigned long nr = pfn_to_section_nr(pfn);
+
+       rcu_read_lock_sched();
+
+       while (nr <= __highest_present_section_nr && pfn < end_pfn) {
+               struct mem_section *ms = __pfn_to_section(pfn);
+
+               if (valid_section(ms) &&
+                   (early_section(ms) || pfn_section_first_valid(ms, &pfn))) {
+                       rcu_read_unlock_sched();
+                       return pfn;
+               }
+
+               /* Nothing left in this section? Skip to next section */
+               nr++;
+               pfn = section_nr_to_pfn(nr);
+       }
+
+       rcu_read_unlock_sched();
+       return end_pfn;
+}
+
+static inline unsigned long next_valid_pfn(unsigned long pfn, unsigned long end_pfn)
+{
+       pfn++;
+
+       if (pfn >= end_pfn)
+               return end_pfn;
+
+       /*
+        * Either every PFN within the section (or subsection for VMEMMAP) is
+        * valid, or none of them are. So there's no point repeating the check
+        * for every PFN; only call first_valid_pfn() the first time, and when
+        * crossing a (sub)section boundary (i.e. !(pfn & ~PFN_VALID_MASK)).
+        */
+       if (pfn & (IS_ENABLED(CONFIG_SPARSEMEM_VMEMMAP) ?
+                  PAGE_SUBSECTION_MASK : PAGE_SECTION_MASK))
+               return pfn;
+
+       return first_valid_pfn(pfn, end_pfn);
+}
+
+
+#define for_each_valid_pfn(_pfn, _start_pfn, _end_pfn)                 \
+       for ((_pfn) = first_valid_pfn((_start_pfn), (_end_pfn));        \
+            (_pfn) < (_end_pfn);                                       \
+            (_pfn) = next_valid_pfn((_pfn), (_end_pfn)))
+
 #endif
 
 static inline int pfn_in_present_section(unsigned long pfn)