lib/generic-radix-tree.c: Make nodes more reasonably sized

this code originally used the page allocator directly, but most code
shouldn't do that - PAGE_SIZE varies with architecture, and slab is
faster.

4k is also on the large side for typical usage, 512 bytes is a better
choice for typical usage that might be somewhat sparse.

Signed-off-by: Kent Overstreet <kent.overstreet@linux.dev>

diff --git a/include/linux/generic-radix-tree.h b/include/linux/generic-radix-tree.h
index 8474131647388b954812515930c81b0c83315f3a..f3512fddf3d738b2f085b037e07ef96b0e908642 100644 (file)
--- a/include/linux/generic-radix-tree.h
+++ b/include/linux/generic-radix-tree.h
@@ -5,7 +5,7 @@
  * DOC: Generic radix trees/sparse arrays
  *
  * Very simple and minimalistic, supporting arbitrary size entries up to
- * PAGE_SIZE.
+ * GENRADIX_NODE_SIZE.
  *
  * A genradix is defined with the type it will store, like so:
  *
@@ -45,12 +45,15 @@
 
 struct genradix_root;
 
+#define GENRADIX_NODE_SHIFT    9
+#define GENRADIX_NODE_SIZE     (1U << GENRADIX_NODE_SHIFT)
+
 struct __genradix {
        struct genradix_root            *root;
 };
 
 /*
- * NOTE: currently, sizeof(_type) must not be larger than PAGE_SIZE:
+ * NOTE: currently, sizeof(_type) must not be larger than GENRADIX_NODE_SIZE:
  */
 
 #define __GENRADIX_INITIALIZER                                 \
@@ -101,14 +104,14 @@ void __genradix_free(struct __genradix *);
 static inline size_t __idx_to_offset(size_t idx, size_t obj_size)
 {
        if (__builtin_constant_p(obj_size))
-               BUILD_BUG_ON(obj_size > PAGE_SIZE);
+               BUILD_BUG_ON(obj_size > GENRADIX_NODE_SIZE);
        else
-               BUG_ON(obj_size > PAGE_SIZE);
+               BUG_ON(obj_size > GENRADIX_NODE_SIZE);
 
        if (!is_power_of_2(obj_size)) {
-               size_t objs_per_page = PAGE_SIZE / obj_size;
+               size_t objs_per_page = GENRADIX_NODE_SIZE / obj_size;
 
-               return (idx / objs_per_page) * PAGE_SIZE +
+               return (idx / objs_per_page) * GENRADIX_NODE_SIZE +
                        (idx % objs_per_page) * obj_size;
        } else {
                return idx * obj_size;
@@ -118,9 +121,9 @@ static inline size_t __idx_to_offset(size_t idx, size_t obj_size)
 #define __genradix_cast(_radix)                (typeof((_radix)->type[0]) *)
 #define __genradix_obj_size(_radix)    sizeof((_radix)->type[0])
 #define __genradix_objs_per_page(_radix)                       \
-       (PAGE_SIZE / sizeof((_radix)->type[0]))
+       (GENRADIX_NODE_SIZE / sizeof((_radix)->type[0]))
 #define __genradix_page_remainder(_radix)                      \
-       (PAGE_SIZE % sizeof((_radix)->type[0]))
+       (GENRADIX_NODE_SIZE % sizeof((_radix)->type[0]))
 
 #define __genradix_idx_to_offset(_radix, _idx)                 \
        __idx_to_offset(_idx, __genradix_obj_size(_radix))
@@ -217,8 +220,8 @@ static inline void __genradix_iter_advance(struct genradix_iter *iter,
        iter->offset += obj_size;
 
        if (!is_power_of_2(obj_size) &&
-           (iter->offset & (PAGE_SIZE - 1)) + obj_size > PAGE_SIZE)
-               iter->offset = round_up(iter->offset, PAGE_SIZE);
+           (iter->offset & (GENRADIX_NODE_SIZE - 1)) + obj_size > GENRADIX_NODE_SIZE)
+               iter->offset = round_up(iter->offset, GENRADIX_NODE_SIZE);
 
        iter->pos++;
 }
@@ -235,8 +238,8 @@ static inline void __genradix_iter_rewind(struct genradix_iter *iter,
                return;
        }
 
-       if ((iter->offset & (PAGE_SIZE - 1)) == 0)
-               iter->offset -= PAGE_SIZE % obj_size;
+       if ((iter->offset & (GENRADIX_NODE_SIZE - 1)) == 0)
+               iter->offset -= GENRADIX_NODE_SIZE % obj_size;
 
        iter->offset -= obj_size;
        iter->pos--;
@@ -263,7 +266,7 @@ static inline void __genradix_iter_rewind(struct genradix_iter *iter,
        genradix_for_each_from(_radix, _iter, _p, 0)
 
 #define genradix_last_pos(_radix)                              \
-       (SIZE_MAX / PAGE_SIZE * __genradix_objs_per_page(_radix) - 1)
+       (SIZE_MAX / GENRADIX_NODE_SIZE * __genradix_objs_per_page(_radix) - 1)
 
 /**
  * genradix_for_each_reverse - iterate over entry in a genradix, reverse order

diff --git a/lib/generic-radix-tree.c b/lib/generic-radix-tree.c
index 41f1bcdc44886c1cef8f25c080c006d22a45339d..aaefb9b678c8e4e1277411a27c554604686d3ddc 100644 (file)
--- a/lib/generic-radix-tree.c
+++ b/lib/generic-radix-tree.c
@@ -5,7 +5,7 @@
 #include <linux/gfp.h>
 #include <linux/kmemleak.h>
 
-#define GENRADIX_ARY           (PAGE_SIZE / sizeof(struct genradix_node *))
+#define GENRADIX_ARY           (GENRADIX_NODE_SIZE / sizeof(struct genradix_node *))
 #define GENRADIX_ARY_SHIFT     ilog2(GENRADIX_ARY)
 
 struct genradix_node {
@@ -14,13 +14,13 @@ struct genradix_node {
                struct genradix_node    *children[GENRADIX_ARY];
 
                /* Leaf: */
-               u8                      data[PAGE_SIZE];
+               u8                      data[GENRADIX_NODE_SIZE];
        };
 };
 
 static inline int genradix_depth_shift(unsigned depth)
 {
-       return PAGE_SHIFT + GENRADIX_ARY_SHIFT * depth;
+       return GENRADIX_NODE_SHIFT + GENRADIX_ARY_SHIFT * depth;
 }
 
 /*
@@ -33,7 +33,7 @@ static inline size_t genradix_depth_size(unsigned depth)
 
 /* depth that's needed for a genradix that can address up to ULONG_MAX: */
 #define GENRADIX_MAX_DEPTH     \
-       DIV_ROUND_UP(BITS_PER_LONG - PAGE_SHIFT, GENRADIX_ARY_SHIFT)
+       DIV_ROUND_UP(BITS_PER_LONG - GENRADIX_NODE_SHIFT, GENRADIX_ARY_SHIFT)
 
 #define GENRADIX_DEPTH_MASK                            \
        ((unsigned long) (roundup_pow_of_two(GENRADIX_MAX_DEPTH + 1) - 1))
@@ -79,23 +79,12 @@ EXPORT_SYMBOL(__genradix_ptr);
 
 static inline struct genradix_node *genradix_alloc_node(gfp_t gfp_mask)
 {
-       struct genradix_node *node;
-
-       node = (struct genradix_node *)__get_free_page(gfp_mask|__GFP_ZERO);
-
-       /*
-        * We're using pages (not slab allocations) directly for kernel data
-        * structures, so we need to explicitly inform kmemleak of them in order
-        * to avoid false positive memory leak reports.
-        */
-       kmemleak_alloc(node, PAGE_SIZE, 1, gfp_mask);
-       return node;
+       return kzalloc(GENRADIX_NODE_SIZE, gfp_mask);
 }
 
 static inline void genradix_free_node(struct genradix_node *node)
 {
-       kmemleak_free(node);
-       free_page((unsigned long)node);
+       kfree(node);
 }
 
 /*
@@ -200,7 +189,7 @@ restart:
                        i++;
                        iter->offset = round_down(iter->offset + objs_per_ptr,
                                                  objs_per_ptr);
-                       iter->pos = (iter->offset >> PAGE_SHIFT) *
+                       iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) *
                                objs_per_page;
                        if (i == GENRADIX_ARY)
                                goto restart;
@@ -209,7 +198,7 @@ restart:
                n = n->children[i];
        }
 
-       return &n->data[iter->offset & (PAGE_SIZE - 1)];
+       return &n->data[iter->offset & (GENRADIX_NODE_SIZE - 1)];
 }
 EXPORT_SYMBOL(__genradix_iter_peek);
 
@@ -235,7 +224,7 @@ restart:
 
        if (ilog2(iter->offset) >= genradix_depth_shift(level)) {
                iter->offset = genradix_depth_size(level);
-               iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page;
+               iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) * objs_per_page;
 
                iter->offset -= obj_size_plus_page_remainder;
                iter->pos--;
@@ -251,7 +240,7 @@ restart:
                        size_t objs_per_ptr = genradix_depth_size(level);
 
                        iter->offset = round_down(iter->offset, objs_per_ptr);
-                       iter->pos = (iter->offset >> PAGE_SHIFT) * objs_per_page;
+                       iter->pos = (iter->offset >> GENRADIX_NODE_SHIFT) * objs_per_page;
 
                        if (!iter->offset)
                                return NULL;
@@ -267,7 +256,7 @@ restart:
                n = n->children[i];
        }
 
-       return &n->data[iter->offset & (PAGE_SIZE - 1)];
+       return &n->data[iter->offset & (GENRADIX_NODE_SIZE - 1)];
 }
 EXPORT_SYMBOL(__genradix_iter_peek_prev);
 
@@ -289,7 +278,7 @@ int __genradix_prealloc(struct __genradix *radix, size_t size,
 {
        size_t offset;
 
-       for (offset = 0; offset < size; offset += PAGE_SIZE)
+       for (offset = 0; offset < size; offset += GENRADIX_NODE_SIZE)
                if (!__genradix_ptr_alloc(radix, offset, gfp_mask))
                        return -ENOMEM;