radix-tree: make radix_tree_descend() more useful

Now that the shift amount is stored in the node, radix_tree_descend()
can calculate offset itself from index, which removes several lines of
code from each of the tree walkers.

Signed-off-by: Matthew Wilcox <willy@linux.intel.com>
Cc: Konstantin Khlebnikov <koct9i@gmail.com>
Cc: Kirill Shutemov <kirill.shutemov@linux.intel.com>
Cc: Jan Kara <jack@suse.com>
Cc: Neil Brown <neilb@suse.de>
Cc: Ross Zwisler <ross.zwisler@linux.intel.com>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index c7114d233b3874ec1c7935824b888dec77086ea2..8b7d8459bb9dc2ea96269eeb5dd815cf2e169287 100644 (file)
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -94,9 +94,10 @@ static inline unsigned long get_slot_offset(struct radix_tree_node *parent,
        return slot - parent->slots;
 }
 
-static unsigned radix_tree_descend(struct radix_tree_node *parent,
-                               struct radix_tree_node **nodep, unsigned offset)
+static unsigned int radix_tree_descend(struct radix_tree_node *parent,
+                       struct radix_tree_node **nodep, unsigned long index)
 {
+       unsigned int offset = (index >> parent->shift) & RADIX_TREE_MAP_MASK;
        void **entry = rcu_dereference_raw(parent->slots[offset]);
 
 #ifdef CONFIG_RADIX_TREE_MULTIORDER
@@ -536,8 +537,7 @@ int __radix_tree_create(struct radix_tree_root *root, unsigned long index,
 
                /* Go a level down */
                node = entry_to_node(child);
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-               offset = radix_tree_descend(node, &child, offset);
+               offset = radix_tree_descend(node, &child, index);
                slot = &node->slots[offset];
        }
 
@@ -625,13 +625,12 @@ void *__radix_tree_lookup(struct radix_tree_root *root, unsigned long index,
 {
        struct radix_tree_node *node, *parent;
        unsigned long maxindex;
-       unsigned int shift;
        void **slot;
 
  restart:
        parent = NULL;
        slot = (void **)&root->rnode;
-       shift = radix_tree_load_root(root, &node, &maxindex);
+       radix_tree_load_root(root, &node, &maxindex);
        if (index > maxindex)
                return NULL;
 
@@ -641,9 +640,7 @@ void *__radix_tree_lookup(struct radix_tree_root *root, unsigned long index,
                if (node == RADIX_TREE_RETRY)
                        goto restart;
                parent = entry_to_node(node);
-               shift -= RADIX_TREE_MAP_SHIFT;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-               offset = radix_tree_descend(parent, &node, offset);
+               offset = radix_tree_descend(parent, &node, index);
                slot = parent->slots + offset;
        }
 
@@ -713,19 +710,15 @@ void *radix_tree_tag_set(struct radix_tree_root *root,
 {
        struct radix_tree_node *node, *parent;
        unsigned long maxindex;
-       unsigned int shift;
 
-       shift = radix_tree_load_root(root, &node, &maxindex);
+       radix_tree_load_root(root, &node, &maxindex);
        BUG_ON(index > maxindex);
 
        while (radix_tree_is_internal_node(node)) {
                unsigned offset;
 
-               shift -= RADIX_TREE_MAP_SHIFT;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-
                parent = entry_to_node(node);
-               offset = radix_tree_descend(parent, &node, offset);
+               offset = radix_tree_descend(parent, &node, index);
                BUG_ON(!node);
 
                if (!tag_get(parent, tag, offset))
@@ -779,21 +772,17 @@ void *radix_tree_tag_clear(struct radix_tree_root *root,
 {
        struct radix_tree_node *node, *parent;
        unsigned long maxindex;
-       unsigned int shift;
        int uninitialized_var(offset);
 
-       shift = radix_tree_load_root(root, &node, &maxindex);
+       radix_tree_load_root(root, &node, &maxindex);
        if (index > maxindex)
                return NULL;
 
        parent = NULL;
 
        while (radix_tree_is_internal_node(node)) {
-               shift -= RADIX_TREE_MAP_SHIFT;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-
                parent = entry_to_node(node);
-               offset = radix_tree_descend(parent, &node, offset);
+               offset = radix_tree_descend(parent, &node, index);
        }
 
        if (node)
@@ -823,25 +812,21 @@ int radix_tree_tag_get(struct radix_tree_root *root,
 {
        struct radix_tree_node *node, *parent;
        unsigned long maxindex;
-       unsigned int shift;
 
        if (!root_tag_get(root, tag))
                return 0;
 
-       shift = radix_tree_load_root(root, &node, &maxindex);
+       radix_tree_load_root(root, &node, &maxindex);
        if (index > maxindex)
                return 0;
        if (node == NULL)
                return 0;
 
        while (radix_tree_is_internal_node(node)) {
-               int offset;
-
-               shift -= RADIX_TREE_MAP_SHIFT;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
+               unsigned offset;
 
                parent = entry_to_node(node);
-               offset = radix_tree_descend(parent, &node, offset);
+               offset = radix_tree_descend(parent, &node, index);
 
                if (!node)
                        return 0;
@@ -874,7 +859,7 @@ static inline void __set_iter_shift(struct radix_tree_iter *iter,
 void **radix_tree_next_chunk(struct radix_tree_root *root,
                             struct radix_tree_iter *iter, unsigned flags)
 {
-       unsigned shift, tag = flags & RADIX_TREE_ITER_TAG_MASK;
+       unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
        struct radix_tree_node *node, *child;
        unsigned long index, offset, maxindex;
 
@@ -895,7 +880,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
                return NULL;
 
  restart:
-       shift = radix_tree_load_root(root, &child, &maxindex);
+       radix_tree_load_root(root, &child, &maxindex);
        if (index > maxindex)
                return NULL;
        if (!child)
@@ -912,9 +897,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
 
        do {
                node = entry_to_node(child);
-               shift -= RADIX_TREE_MAP_SHIFT;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-               offset = radix_tree_descend(node, &child, offset);
+               offset = radix_tree_descend(node, &child, index);
 
                if ((flags & RADIX_TREE_ITER_TAGGED) ?
                                !tag_get(node, tag, offset) : !child) {
@@ -936,7 +919,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
                                                break;
                                }
                        index &= ~node_maxindex(node);
-                       index += offset << shift;
+                       index += offset << node->shift;
                        /* Overflow after ~0UL */
                        if (!index)
                                return NULL;
@@ -952,7 +935,7 @@ void **radix_tree_next_chunk(struct radix_tree_root *root,
        /* Update the iterator state */
        iter->index = (index &~ node_maxindex(node)) | (offset << node->shift);
        iter->next_index = (index | node_maxindex(node)) + 1;
-       __set_iter_shift(iter, shift);
+       __set_iter_shift(iter, node->shift);
 
        /* Construct iter->tags bit-mask from node->tags[tag] array */
        if (flags & RADIX_TREE_ITER_TAGGED) {
@@ -1010,10 +993,10 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
 {
        struct radix_tree_node *parent, *node, *child;
        unsigned long maxindex;
-       unsigned int shift = radix_tree_load_root(root, &child, &maxindex);
        unsigned long tagged = 0;
        unsigned long index = *first_indexp;
 
+       radix_tree_load_root(root, &child, &maxindex);
        last_index = min(last_index, maxindex);
        if (index > last_index)
                return 0;
@@ -1030,11 +1013,9 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
        }
 
        node = entry_to_node(child);
-       shift -= RADIX_TREE_MAP_SHIFT;
 
        for (;;) {
-               unsigned offset = (index >> shift) & RADIX_TREE_MAP_MASK;
-               offset = radix_tree_descend(node, &child, offset);
+               unsigned offset = radix_tree_descend(node, &child, index);
                if (!child)
                        goto next;
                if (!tag_get(node, iftag, offset))
@@ -1042,7 +1023,6 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
                /* Sibling slots never have tags set on them */
                if (radix_tree_is_internal_node(child)) {
                        node = entry_to_node(child);
-                       shift -= RADIX_TREE_MAP_SHIFT;
                        continue;
                }
 
@@ -1063,12 +1043,12 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
                        tag_set(parent, settag, offset);
                }
  next:
-               /* Go to next item at level determined by 'shift' */
-               index = ((index >> shift) + 1) << shift;
+               /* Go to next entry in node */
+               index = ((index >> node->shift) + 1) << node->shift;
                /* Overflow can happen when last_index is ~0UL... */
                if (index > last_index || !index)
                        break;
-               offset = (index >> shift) & RADIX_TREE_MAP_MASK;
+               offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;
                while (offset == 0) {
                        /*
                         * We've fully scanned this node. Go up. Because
@@ -1076,8 +1056,7 @@ unsigned long radix_tree_range_tag_if_tagged(struct radix_tree_root *root,
                         * we do below cannot wander astray.
                         */
                        node = node->parent;
-                       shift += RADIX_TREE_MAP_SHIFT;
-                       offset = (index >> shift) & RADIX_TREE_MAP_MASK;
+                       offset = (index >> node->shift) & RADIX_TREE_MAP_MASK;
                }
                if (is_sibling_entry(node, node->slots[offset]))
                        goto next;
@@ -1275,13 +1254,10 @@ struct locate_info {
 static unsigned long __locate(struct radix_tree_node *slot, void *item,
                              unsigned long index, struct locate_info *info)
 {
-       unsigned int shift;
        unsigned long i;
 
-       shift = slot->shift + RADIX_TREE_MAP_SHIFT;
-
        do {
-               shift -= RADIX_TREE_MAP_SHIFT;
+               unsigned int shift = slot->shift;
 
                for (i = (index >> shift) & RADIX_TREE_MAP_MASK;
                     i < RADIX_TREE_MAP_SIZE;
@@ -1304,9 +1280,7 @@ static unsigned long __locate(struct radix_tree_node *slot, void *item,
                        slot = node;
                        break;
                }
-               if (i == RADIX_TREE_MAP_SIZE)
-                       break;
-       } while (shift);
+       } while (i < RADIX_TREE_MAP_SIZE);
 
 out:
        if ((index == 0) && (i == RADIX_TREE_MAP_SIZE))