percpu: introduce nr_empty_pop_pages to help empty page accounting

pcpu_nr_empty_pop_pages is used to ensure there are a handful of free
pages around to serve atomic allocations. A new field, nr_empty_pop_pages,
is added to the pcpu_chunk struct to keep track of the number of empty
pages. This field is needed as the number of empty populated pages is
globally tracked and deltas are used to update in the bitmap allocator.
Pages that contain a hidden area are not considered to be empty. This
new field is exposed in percpu_stats.

Signed-off-by: Dennis Zhou <dennisszhou@gmail.com>
Reviewed-by: Josef Bacik <jbacik@fb.com>
Signed-off-by: Tejun Heo <tj@kernel.org>

diff --git a/mm/percpu-internal.h b/mm/percpu-internal.h
index 34cb9799e32442a46711e33a365a375d84be721e..c4c8fc49780bd893381610b638b7f3a597e1ccce 100644 (file)
--- a/mm/percpu-internal.h
+++ b/mm/percpu-internal.h
@@ -32,6 +32,7 @@ struct pcpu_chunk {
 
        int                     nr_pages;       /* # of pages served by this chunk */
        int                     nr_populated;   /* # of populated pages */
+       int                     nr_empty_pop_pages; /* # of empty populated pages */
        unsigned long           populated[];    /* populated bitmap */
 };
 

diff --git a/mm/percpu-stats.c b/mm/percpu-stats.c
index ffbdb96cdbeb4932bd4c9a3a717a70f3dd9eaa3a..e146b585fd185149902a5cbb45983bfc41c25260 100644 (file)
--- a/mm/percpu-stats.c
+++ b/mm/percpu-stats.c
@@ -100,6 +100,7 @@ static void chunk_map_stats(struct seq_file *m, struct pcpu_chunk *chunk,
 
        P("nr_alloc", chunk->nr_alloc);
        P("max_alloc_size", chunk->max_alloc_size);
+       P("empty_pop_pages", chunk->nr_empty_pop_pages);
        P("free_size", chunk->free_size);
        P("contig_hint", chunk->contig_hint);
        P("sum_frag", sum_frag);

diff --git a/mm/percpu.c b/mm/percpu.c
index 773dafea181ef7b79f0f1510f0fed93faadc27f8..657ab0821cf00a817f3968b19e3a2725db992321 100644 (file)
--- a/mm/percpu.c
+++ b/mm/percpu.c
@@ -757,11 +757,14 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
        chunk->immutable = true;
        bitmap_fill(chunk->populated, chunk->nr_pages);
        chunk->nr_populated = chunk->nr_pages;
+       chunk->nr_empty_pop_pages = chunk->nr_pages;
 
        chunk->contig_hint = chunk->free_size = map_size;
 
        if (chunk->start_offset) {
                /* hide the beginning of the bitmap */
+               chunk->nr_empty_pop_pages--;
+
                chunk->map[0] = 1;
                chunk->map[1] = chunk->start_offset;
                chunk->map_used = 1;
@@ -773,6 +776,8 @@ static struct pcpu_chunk * __init pcpu_alloc_first_chunk(unsigned long tmp_addr,
 
        if (chunk->end_offset) {
                /* hide the end of the bitmap */
+               chunk->nr_empty_pop_pages--;
+
                chunk->map[++chunk->map_used] = region_size | 1;
        }
 
@@ -836,6 +841,7 @@ static void pcpu_chunk_populated(struct pcpu_chunk *chunk,
 
        bitmap_set(chunk->populated, page_start, nr);
        chunk->nr_populated += nr;
+       chunk->nr_empty_pop_pages += nr;
        pcpu_nr_empty_pop_pages += nr;
 }
 
@@ -858,6 +864,7 @@ static void pcpu_chunk_depopulated(struct pcpu_chunk *chunk,
 
        bitmap_clear(chunk->populated, page_start, nr);
        chunk->nr_populated -= nr;
+       chunk->nr_empty_pop_pages -= nr;
        pcpu_nr_empty_pop_pages -= nr;
 }
 
@@ -1782,9 +1789,7 @@ int __init pcpu_setup_first_chunk(const struct pcpu_alloc_info *ai,
 
        /* link the first chunk in */
        pcpu_first_chunk = chunk;
-       i = (pcpu_first_chunk->start_offset) ? 1 : 0;
-       pcpu_nr_empty_pop_pages +=
-               pcpu_count_occupied_pages(pcpu_first_chunk, i);
+       pcpu_nr_empty_pop_pages = pcpu_first_chunk->nr_empty_pop_pages;
        pcpu_chunk_relocate(pcpu_first_chunk, -1);
 
        pcpu_stats_chunk_alloc();