*
* @index: index of current slot
* @next_index: one beyond the last index for this chunk
- * @tags: bit-mask for tag-iterating
* @node: node that contains current slot
*
* This radix tree iterator works in terms of "chunks" of slots. A chunk is a
struct radix_tree_iter {
unsigned long index;
unsigned long next_index;
- unsigned long tags;
struct radix_tree_node *node;
};
void __rcu **slot, void *entry);
void radix_tree_iter_replace(struct radix_tree_root *,
const struct radix_tree_iter *, void __rcu **slot, void *entry);
-void radix_tree_iter_delete(struct radix_tree_root *,
- struct radix_tree_iter *iter, void __rcu **slot);
void *radix_tree_delete_item(struct radix_tree_root *, unsigned long, void *);
void *radix_tree_delete(struct radix_tree_root *, unsigned long);
void radix_tree_init(void);
struct radix_tree_iter *iter, gfp_t gfp,
unsigned long max);
-enum {
- RADIX_TREE_ITER_TAG_MASK = 0x0f, /* tag index in lower nybble */
- RADIX_TREE_ITER_TAGGED = 0x10, /* lookup tagged slots */
- RADIX_TREE_ITER_CONTIG = 0x20, /* stop at first hole */
-};
-
/**
* radix_tree_iter_init - initialize radix tree iterator
*
radix_tree_iter_init(struct radix_tree_iter *iter, unsigned long start)
{
/*
- * Leave iter->tags uninitialized. radix_tree_next_chunk() will fill it
- * in the case of a successful tagged chunk lookup. If the lookup was
- * unsuccessful or non-tagged then nobody cares about ->tags.
- *
* Set index to zero to bypass next_index overflow protection.
* See the comment in radix_tree_next_chunk() for details.
*/
* This function looks up the next chunk in the radix tree starting from
* @iter->next_index. It returns a pointer to the chunk's first slot.
* Also it fills @iter with data about chunk: position in the tree (index),
- * its end (next_index), and constructs a bit mask for tagged iterating (tags).
+ * its end (next_index).
*/
void __rcu **radix_tree_next_chunk(const struct radix_tree_root *,
struct radix_tree_iter *iter, unsigned flags);
-/**
- * radix_tree_iter_lookup - look up an index in the radix tree
- * @root: radix tree root
- * @iter: iterator state
- * @index: key to look up
- *
- * If @index is present in the radix tree, this function returns the slot
- * containing it and updates @iter to describe the entry. If @index is not
- * present, it returns NULL.
- */
-static inline void __rcu **
-radix_tree_iter_lookup(const struct radix_tree_root *root,
- struct radix_tree_iter *iter, unsigned long index)
-{
- radix_tree_iter_init(iter, index);
- return radix_tree_next_chunk(root, iter, RADIX_TREE_ITER_CONTIG);
-}
-
/**
* radix_tree_iter_find - find a present entry
* @root: radix tree root
return radix_tree_next_chunk(root, iter, 0);
}
-/**
- * radix_tree_iter_retry - retry this chunk of the iteration
- * @iter: iterator state
- *
- * If we iterate over a tree protected only by the RCU lock, a race
- * against deletion or creation may result in seeing a slot for which
- * radix_tree_deref_retry() returns true. If so, call this function
- * and continue the iteration.
- */
-static inline __must_check
-void __rcu **radix_tree_iter_retry(struct radix_tree_iter *iter)
-{
- iter->next_index = iter->index;
- iter->tags = 0;
- return NULL;
-}
-
static inline unsigned long
__radix_tree_iter_add(struct radix_tree_iter *iter, unsigned long slots)
{
return iter->index + slots;
}
-/**
- * radix_tree_iter_resume - resume iterating when the chunk may be invalid
- * @slot: pointer to current slot
- * @iter: iterator state
- * Returns: New slot pointer
- *
- * If the iterator needs to release then reacquire a lock, the chunk may
- * have been invalidated by an insertion or deletion. Call this function
- * before releasing the lock to continue the iteration from the next index.
- */
-void __rcu **__must_check radix_tree_iter_resume(void __rcu **slot,
- struct radix_tree_iter *iter);
-
/**
* radix_tree_chunk_size - get current chunk size
*
* Returns: pointer to next slot, or NULL if there no more left
*
* This function updates @iter->index in the case of a successful lookup.
- * For tagged lookup it also eats @iter->tags.
- *
- * There are several cases where 'slot' can be passed in as NULL to this
- * function. These cases result from the use of radix_tree_iter_resume() or
- * radix_tree_iter_retry(). In these cases we don't end up dereferencing
- * 'slot' because either:
- * a) we are doing tagged iteration and iter->tags has been set to 0, or
- * b) we are doing non-tagged iteration, and iter->index and iter->next_index
- * have been set up so that radix_tree_chunk_size() returns 1 or 0.
*/
static __always_inline void __rcu **radix_tree_next_slot(void __rcu **slot,
struct radix_tree_iter *iter, unsigned flags)
{
- if (flags & RADIX_TREE_ITER_TAGGED) {
- iter->tags >>= 1;
- if (unlikely(!iter->tags))
- return NULL;
- if (likely(iter->tags & 1ul)) {
- iter->index = __radix_tree_iter_add(iter, 1);
- slot++;
- goto found;
- }
- if (!(flags & RADIX_TREE_ITER_CONTIG)) {
- unsigned offset = __ffs(iter->tags);
-
- iter->tags >>= offset++;
- iter->index = __radix_tree_iter_add(iter, offset);
- slot += offset;
- goto found;
- }
- } else {
- long count = radix_tree_chunk_size(iter);
+ long count = radix_tree_chunk_size(iter);
- while (--count > 0) {
- slot++;
- iter->index = __radix_tree_iter_add(iter, 1);
+ while (--count > 0) {
+ slot++;
+ iter->index = __radix_tree_iter_add(iter, 1);
- if (likely(*slot))
- goto found;
- if (flags & RADIX_TREE_ITER_CONTIG) {
- /* forbid switching to the next chunk */
- iter->next_index = 0;
- break;
- }
- }
+ if (likely(*slot))
+ goto found;
}
return NULL;
slot || (slot = radix_tree_next_chunk(root, iter, 0)) ; \
slot = radix_tree_next_slot(slot, iter, 0))
-/**
- * radix_tree_for_each_tagged - iterate over tagged slots
- *
- * @slot: the void** variable for pointer to slot
- * @root: the struct radix_tree_root pointer
- * @iter: the struct radix_tree_iter pointer
- * @start: iteration starting index
- * @tag: tag index
- *
- * @slot points to radix tree slot, @iter->index contains its index.
- */
-#define radix_tree_for_each_tagged(slot, root, iter, start, tag) \
- for (slot = radix_tree_iter_init(iter, start) ; \
- slot || (slot = radix_tree_next_chunk(root, iter, \
- RADIX_TREE_ITER_TAGGED | tag)) ; \
- slot = radix_tree_next_slot(slot, iter, \
- RADIX_TREE_ITER_TAGGED | tag))
-
#endif /* _LINUX_RADIX_TREE_H */
}
EXPORT_SYMBOL(radix_tree_tag_get);
-/* Construct iter->tags bit-mask from node->tags[tag] array */
-static void set_iter_tags(struct radix_tree_iter *iter,
- struct radix_tree_node *node, unsigned offset,
- unsigned tag)
-{
- unsigned tag_long = offset / BITS_PER_LONG;
- unsigned tag_bit = offset % BITS_PER_LONG;
-
- if (!node) {
- iter->tags = 1;
- return;
- }
-
- iter->tags = node->tags[tag][tag_long] >> tag_bit;
-
- /* This never happens if RADIX_TREE_TAG_LONGS == 1 */
- if (tag_long < RADIX_TREE_TAG_LONGS - 1) {
- /* Pick tags from next element */
- if (tag_bit)
- iter->tags |= node->tags[tag][tag_long + 1] <<
- (BITS_PER_LONG - tag_bit);
- /* Clip chunk size, here only BITS_PER_LONG tags */
- iter->next_index = __radix_tree_iter_add(iter, BITS_PER_LONG);
- }
-}
-
-void __rcu **radix_tree_iter_resume(void __rcu **slot,
- struct radix_tree_iter *iter)
-{
- slot++;
- iter->index = __radix_tree_iter_add(iter, 1);
- iter->next_index = iter->index;
- iter->tags = 0;
- return NULL;
-}
-EXPORT_SYMBOL(radix_tree_iter_resume);
-
/**
* radix_tree_next_chunk - find next chunk of slots for iteration
*
void __rcu **radix_tree_next_chunk(const struct radix_tree_root *root,
struct radix_tree_iter *iter, unsigned flags)
{
- unsigned tag = flags & RADIX_TREE_ITER_TAG_MASK;
struct radix_tree_node *node, *child;
unsigned long index, offset, maxindex;
- if ((flags & RADIX_TREE_ITER_TAGGED) && !root_tag_get(root, tag))
- return NULL;
-
/*
* Catch next_index overflow after ~0UL. iter->index never overflows
* during iterating; it can be zero only at the beginning.
/* Single-slot tree */
iter->index = index;
iter->next_index = maxindex + 1;
- iter->tags = 1;
iter->node = NULL;
return (void __rcu **)&root->xa_head;
}
node = entry_to_node(child);
offset = radix_tree_descend(node, &child, index);
- if ((flags & RADIX_TREE_ITER_TAGGED) ?
- !tag_get(node, tag, offset) : !child) {
- /* Hole detected */
- if (flags & RADIX_TREE_ITER_CONTIG)
- return NULL;
-
- if (flags & RADIX_TREE_ITER_TAGGED)
- offset = radix_tree_find_next_bit(node, tag,
- offset + 1);
- else
- while (++offset < RADIX_TREE_MAP_SIZE) {
- void *slot = rcu_dereference_raw(
- node->slots[offset]);
- if (slot)
- break;
- }
+ if (!child) {
+ while (++offset < RADIX_TREE_MAP_SIZE) {
+ void *slot = rcu_dereference_raw(
+ node->slots[offset]);
+ if (slot)
+ break;
+ }
index &= ~node_maxindex(node);
index += offset << node->shift;
/* Overflow after ~0UL */
iter->next_index = (index | node_maxindex(node)) + 1;
iter->node = node;
- if (flags & RADIX_TREE_ITER_TAGGED)
- set_iter_tags(iter, node, offset, tag);
-
return node->slots + offset;
}
EXPORT_SYMBOL(radix_tree_next_chunk);
return node && delete_node(root, node);
}
-/**
- * radix_tree_iter_delete - delete the entry at this iterator position
- * @root: radix tree root
- * @iter: iterator state
- * @slot: pointer to slot
- *
- * Delete the entry at the position currently pointed to by the iterator.
- * This may result in the current node being freed; if it is, the iterator
- * is advanced so that it will not reference the freed memory. This
- * function may be called without any locking if there are no other threads
- * which can access this tree.
- */
-void radix_tree_iter_delete(struct radix_tree_root *root,
- struct radix_tree_iter *iter, void __rcu **slot)
-{
- if (__radix_tree_delete(root, iter->node, slot))
- iter->index = iter->next_index;
-}
-EXPORT_SYMBOL(radix_tree_iter_delete);
-
/**
* radix_tree_delete_item - delete an item from a radix tree
* @root: radix tree root
else
iter->next_index = 1;
iter->node = node;
- set_iter_tags(iter, node, offset, IDR_FREE);
return slot;
}
projects / users / willy / xarray.git / commitdiff