min = mas_safe_min(mas, wr_mas->pivots, offset);
max = wr_mas->pivots[offset];
if (unlikely(offset == count))
- goto max;
+ goto max; /* may have been set to zero */
index = mas->index;
if (unlikely(index <= max))
* @l_mas: The left maple state
* @r_mas: The right maple state
*/
-static inline void mas_extend_spanning_null(struct ma_state *l_mas,
- struct ma_state *r_mas)
+static inline void mas_extend_spanning_null(
+ struct ma_state *l_mas, unsigned long r_min, void *content,
+ struct ma_state *r_mas, unsigned long r_max, void *r_content)
{
unsigned char l_slot = l_mas->offset;
unsigned char r_slot = r_mas->offset;
unsigned char cp_r_slot = r_slot;
- unsigned long range_max = mas_safe_pivot(r_mas, r_slot);
- unsigned long range_min = l_mas->min;
- void __rcu **slots = ma_slots(mte_to_node(l_mas->node),
- mte_node_type(l_mas->node));
- void *content = mas_slot_locked(l_mas, slots, l_slot);
-
- if (l_slot)
- range_min = mas_safe_pivot(l_mas, l_slot - 1) + 1;
+ struct maple_node *l_node = mas_mn(l_mas);
+ enum maple_type l_type = mte_node_type(l_mas->node);
+ void __rcu **slots = ma_slots(l_node, l_type);
/* Expand NULL to start of the range. */
if (!content)
- l_mas->index = range_min;
+ l_mas->index = r_min;
- if ((l_mas->index == range_min) &&
- l_slot && !slots[l_slot - 1]) {
- if (l_slot > 1)
- l_mas->index = mas_safe_pivot(l_mas, l_slot - 2) + 1;
- else
+ if ((l_mas->index == r_min) &&
+ (l_slot && !slots[l_slot - 1])) {
+ if (l_slot > 1) {
+ unsigned long *pivots = ma_pivots(l_node, l_type);
+
+ l_mas->index = pivots[l_slot - 2] + 1;
+ } else
l_mas->index = l_mas->min;
l_mas->offset = l_slot - 1;
}
- slots = ma_slots(mte_to_node(r_mas->node),
- mte_node_type(r_mas->node));
- if (!mas_slot_locked(r_mas, slots, r_slot)) {
- if (r_mas->last < range_max)
- r_mas->last = range_max;
+ if (!r_content) {
+ if (r_mas->last < r_max)
+ r_mas->last = r_max;
cp_r_slot++;
}
- if (r_mas->last == range_max &&
- r_mas->last < r_mas->max &&
+ slots = ma_slots(mte_to_node(r_mas->node),
+ mte_node_type(r_mas->node));
+ if ((r_mas->last == r_max) &&
+ (r_mas->last < r_mas->max) &&
!mas_slot_locked(r_mas, slots, r_slot + 1)) {
r_mas->last = mas_safe_pivot(r_mas, r_slot + 1);
cp_r_slot++;
r_mas->offset = cp_r_slot;
}
-/*
- * mas_node_walk() - Walk a maple node to offset of the index.
- * @mas: The maple state
- * @type: The maple node type
- * @*range_min: Pointer to store the minimum range of the offset
- * @*range_max: Pointer to store the maximum range of the offset
- *
- * The offset will be stored in the maple state.
- *
- */
-static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
- enum maple_type type, unsigned long *range_min,
- unsigned long *range_max)
-
-{
- unsigned long *pivots;
- unsigned char count;
- unsigned long prev, max;
- unsigned char offset;
- unsigned long index;
-
- if (unlikely(ma_is_dense(type))) {
- (*range_max) = (*range_min) = mas->index;
- if (unlikely(ma_dead_node(node)))
- return;
-
- mas->offset = mas->index = mas->min;
- return;
- }
-
- pivots = ma_pivots(node, type);
- max = pivots[0];
- if (unlikely(ma_dead_node(node)))
- return;
-
- offset = 0;
- prev = mas->min;
- index = mas->index;
- if (unlikely(index <= max))
- goto offset_zero;
-
- count = mt_pivots[type];
- while (++offset < count) {
- prev = max;
- max = pivots[offset];
- if (unlikely(ma_dead_node(node)))
- return;
-
- if (index <= max)
- goto offset_found;
- else if (unlikely(!max))
- goto mas_max;
- }
-
- prev = max;
-mas_max:
- max = mas->max;
-offset_found:
- prev++;
-offset_zero:
- mas->offset = offset;
- if (ma_is_leaf(type)) {
- *range_max = max;
- *range_min = prev;
- } else {
- mas->max = max;
- mas->min = prev;
- }
-}
-
static inline void *mtree_range_walk(struct ma_state *mas)
{
unsigned long *pivots;
unsigned long prev_max, prev_min;
last = next = mas->node;
- prev_min = min = 0;
- max = ULONG_MAX;
+ prev_min = min = mas->min;
+ max = mas->max;
do {
offset = 0;
last = next;
end = ma_data_end(node, type, pivots, max);
if (pivots[offset] >= mas->index) {
prev_max = max;
+ prev_min = min;
max = pivots[offset];
goto next;
}
offset++;
} while((offset < end) && (pivots[offset] < mas->index));
- prev_max = max;
prev_min = min;
-
- if (offset < end)
+ prev_max = max;
+ if (offset < mt_pivots[type] && pivots[offset])
max = pivots[offset];
if (offset)
min = pivots[offset - 1] + 1;
+
+ if (likely(offset > end) && pivots[offset]) {
next:
- if (likely(offset > end) && pivots[offset])
max = pivots[offset];
+ }
slots = ma_slots(node, type);
next = mt_slot(mas->tree, slots, offset);
return NULL;
}
+static inline void *mas_state_walk(struct ma_state *mas)
+{
+ void *entry;
+
+ entry = mas_start(mas);
+ if (mas_is_none(mas))
+ goto not_found;
+
+ if (mas_is_ptr(mas)) {
+ if (!mas->index)
+ return entry;
+
+ goto not_found;
+ }
+
+ return mtree_range_walk(mas);
+
+not_found:
+ mas->offset = MAPLE_NODE_SLOTS;
+ return NULL;
+}
/*
* mtree_lookup_walk() - Internal quick lookup that does not keep maple state up
* to date.
return NULL;
}
-/*
- * mas_descend_walk(): Locates a value and sets the mas->node and slot
- * accordingly. range_min and range_max are set to the range which the entry is
- * valid.
- * @mas: The maple state
- * @*range_min: A pointer to store the minimum of the range
- * @*range_max: A pointer to store the maximum of the range
- *
- * Check mas->node is still valid on return of any value.
- *
- * Return: true if pointing to a valid node and offset. False otherwise.
- */
-static inline bool mas_descend_walk(struct ma_state *mas,
- unsigned long *range_min, unsigned long *range_max)
-{
- struct maple_enode *next;
- struct maple_node *node;
- enum maple_type type;
-
- next = mas->node;
- while (true) {
- node = mte_to_node(next);
- type = mte_node_type(next);
- mas_node_walk(mas, node, type, range_min, range_max);
- next = mas_slot(mas, ma_slots(node, type), mas->offset);
- if (unlikely(ma_dead_node(node)))
- return false;
-
- if (unlikely(ma_is_leaf(type)))
- return true;
-
- /* Descend. */
- mas->node = next;
- }
- return false;
-}
-
/*
* mas_new_root() - Create a new root node that only contains the entry passed
* in.
*/
static inline int mas_spanning_store(struct ma_state *mas, void *entry)
{
- unsigned long range_min, range_max;
struct maple_big_node b_node;
struct maple_subtree_state mast;
unsigned char height = mas_mt_height(mas);
int node_count = 1 + height * 3;
- void *content;
+ void *content, *r_content;
+ unsigned long r_min, r_max;
/* Holds new left and right sub-tree */
MA_STATE(l_mas, mas->tree, mas->index, mas->index);
r_mas.last++;
r_mas.index = r_mas.last;
- r_mas.offset = 0;
- mas_descend_walk(&r_mas, &range_min, &range_max);
+ r_content = mtree_range_walk(&r_mas);
+ r_max = r_mas.last;
r_mas.last = r_mas.index = mas->last;
/* Set up left side. */
l_mas = *mas;
- l_mas.offset = 0;
- mas_descend_walk(&l_mas, &range_min, &range_max);
- content = mas_get_slot(&l_mas, l_mas.offset);
+ content = mtree_range_walk(&l_mas);
+ r_min = l_mas.index;
+ l_mas.index = mas->index;
+ l_mas.last = mas->last;
if (!entry) {
- mas_extend_spanning_null(&l_mas, &r_mas);
+ mas_extend_spanning_null(&l_mas, r_min, content,
+ &r_mas, r_max, r_content);
mas->index = l_mas.index;
mas->last = l_mas.last = r_mas.index = r_mas.last;
mas->offset = l_mas.offset;
}
-
/* Copy l_mas and store the value in b_node. */
b_node.b_end = mas_store_b_node(&l_mas, &b_node, entry,
mas_data_end(&l_mas), mas_data_end(&l_mas),
return entry;
}
-/*
- * mas_tree_walk() - Walk to @mas->index and set the range values.
- * @mas: The maple state.
- * @*range_min: The minimum range to be set.
- * @*range_max: The maximum range to be set.
- *
- * Ranges are only valid if there is a valid entry at @mas->index.
- *
- * Return: True if a value exists, false otherwise.
- */
-static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
- unsigned long *range_max)
-{
- bool ret;
-
-retry:
- ret = false;
- mas_start(mas);
- if (mas_is_none(mas))
- goto not_found;
-
- if (mas_is_ptr(mas)) {
- *range_min = *range_max = 0;
- if (!mas->index)
- return true;
-
- goto not_found;
- }
-
- ret = mas_descend_walk(mas, range_min, range_max);
- if (unlikely(mte_dead_node(mas->node))) {
- mas->node = MAS_START;
- goto retry;
- }
-
- return ret;
-
-not_found:
- mas->offset = MAPLE_NODE_SLOTS;
- return false;
-}
-
static inline void mas_rewalk(struct ma_state *mas, unsigned long index)
{
retry:
mas_set(mas, index);
- mas_start(mas);
- if (mas_is_none(mas))
- goto not_found;
-
- if (mas_is_ptr(mas)) {
- if (!mas->index)
- return;
-
- goto not_found;
- }
-
- mtree_range_walk(mas);
+ mas_state_walk(mas);
if (mas_is_start(mas))
goto retry;
return;
-not_found:
- mas->offset = MAPLE_NODE_SLOTS;
}
/*
return found;
}
-/*
- * mas_range_load() - Load the entry at an index and get the range.
- * @mas: The maple state
- * @*range_min: Pointer to store the minimum range the entry is valid
- * @*range_max: Pointer to store the maximum range the entry is valid
- *
- * Must hold rcu_read_lock or the write lock.
- * Find where mas->index is located and return the entry.
- * @mas->node will point to the node containing the entry.
- * range_min and range_max will be set accordingly.
- *
- * Return: The entry at mas->index or %NULL
- */
-static inline void *mas_range_load(struct ma_state *mas,
- unsigned long *range_min, unsigned long *range_max)
-{
- void *entry = NULL;
- unsigned long index = mas->index;
-
- if (mas_is_none(mas) || mas_is_paused(mas))
- mas->node = MAS_START;
-retry:
- if (mas_tree_walk(mas, range_min, range_max))
- if (unlikely(mas->node == MAS_ROOT))
- return mas_root(mas);
-
- if (likely(mas->offset != MAPLE_NODE_SLOTS))
- entry = mas_get_slot(mas, mas->offset);
-
- if (mas_dead_node(mas, index))
- goto retry;
-
- return entry;
-}
-
/**
* mas_walk() - Search for @mas->index in the tree.
* @mas - the maple state.
*/
void *mas_walk(struct ma_state *mas)
{
- unsigned long range_min, range_max;
void *entry;
- /* Retries on dead nodes handled by mas_range_load */
- entry = mas_range_load(mas, &range_min, &range_max);
- mas->index = range_min;
- mas->last = range_max;
+retry:
+ entry = mas_state_walk(mas);
+ if (mas_is_start(mas))
+ goto retry;
+
return entry;
}
*/
if (mas->mas_flags & MA_STATE_REBALANCE) {
unsigned char end;
- unsigned long range_min, range_max;
if (mas_is_start(mas))
mas_start(mas);
- mas_descend_walk(mas, &range_min, &range_max);
+ mtree_range_walk(mas);
end = mas_data_end(mas) + 1;
if (end < mt_min_slot_count(mas->node) - 1)
mas_destroy_rebalance(mas, end);
*/
void *mas_erase(struct ma_state *mas)
{
- unsigned long r_max, r_min;
void *entry = NULL;
- entry = mas_range_load(mas, &r_min, &r_max);
-retry:
- mas->node = MAS_START;
- mas->index = r_min;
- mas->last = r_max;
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+
+ /* Retry unnecessary when holding the write lock. */
+ entry = mas_state_walk(mas);
+write_retry:
+ /* Must reset to ensure spanning writes of last slot are detected */
+ mas_reset(mas);
mas_store_entry(mas, NULL);
if (mas_nomem(mas, GFP_KERNEL))
- goto retry;
+ goto write_retry;
return entry;
}
rcu_read_lock();
retry:
- entry = mas_start(&mas);
- if (unlikely(mas_is_none(&mas)))
- goto unlock;
-
- if (unlikely(mas_is_ptr(&mas))) {
- if (index)
- entry = NULL;
-
- goto unlock;
- }
+ entry = mas_state_walk(&mas);
+ if (mas_is_start(&mas))
+ goto retry;
- entry = mtree_range_walk(&mas);
- if (xa_is_zero(entry))
+ if (unlikely(xa_is_zero(entry)))
entry = NULL;
if (entry)
goto unlock;
- if (!entry && unlikely(mas_is_start(&mas)))
- goto retry;
-
while (mas_searchable(&mas) && (mas.index < max)) {
entry = mas_next_entry(&mas, max);
if (likely(entry && !xa_is_zero(entry)))
MT_BUG_ON(mt, (*index) && ((*index) <= copy));
#endif
}
- if (xa_is_zero(entry))
- return NULL;
return entry;
}
if (!mte_is_leaf(mte))
gaps = ma_gaps(mte_to_node(mte), mte_node_type(mte));
-
for (i = 0; i < mt_slot_count(mte); i++) {
p_end = mas_logical_pivot(mas, pivots, i, mte_node_type(mte));
return count;
}
+/*
+ * mas_node_walk() - Walk a maple node to offset of the index.
+ * @mas: The maple state
+ * @type: The maple node type
+ * @*range_min: Pointer to store the minimum range of the offset
+ * @*range_max: Pointer to store the maximum range of the offset
+ *
+ * The offset will be stored in the maple state.
+ *
+ */
+static inline void mas_node_walk(struct ma_state *mas, struct maple_node *node,
+ enum maple_type type, unsigned long *range_min,
+ unsigned long *range_max)
+
+{
+ unsigned long *pivots;
+ unsigned char count;
+ unsigned long prev, max;
+ unsigned char offset;
+ unsigned long index;
+
+ if (unlikely(ma_is_dense(type))) {
+ (*range_max) = (*range_min) = mas->index;
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ mas->offset = mas->index = mas->min;
+ return;
+ }
+
+ pivots = ma_pivots(node, type);
+ max = pivots[0];
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ offset = 0;
+ prev = mas->min;
+ index = mas->index;
+ if (unlikely(index <= max))
+ goto offset_zero;
+
+ count = mt_pivots[type];
+ while (++offset < count) {
+ prev = max;
+ max = pivots[offset];
+ if (unlikely(ma_dead_node(node)))
+ return;
+
+ if (index <= max)
+ goto offset_found;
+ else if (unlikely(!max))
+ goto mas_max;
+ }
+
+ prev = max;
+mas_max:
+ max = mas->max;
+offset_found:
+ prev++;
+offset_zero:
+ mas->offset = offset;
+ if (ma_is_leaf(type)) {
+ *range_max = max;
+ *range_min = prev;
+ } else {
+ mas->max = max;
+ mas->min = prev;
+ }
+}
+
+/*
+ * mas_descend_walk(): Locates a value and sets the mas->node and slot
+ * accordingly. range_min and range_max are set to the range which the entry is
+ * valid.
+ * @mas: The maple state
+ * @*range_min: A pointer to store the minimum of the range
+ * @*range_max: A pointer to store the maximum of the range
+ *
+ * Check mas->node is still valid on return of any value.
+ *
+ * Return: true if pointing to a valid node and offset. False otherwise.
+ */
+static inline bool mas_descend_walk(struct ma_state *mas,
+ unsigned long *range_min, unsigned long *range_max)
+{
+ struct maple_enode *next;
+ struct maple_node *node;
+ enum maple_type type;
+
+ next = mas->node;
+ while (true) {
+ node = mte_to_node(next);
+ type = mte_node_type(next);
+ mas_node_walk(mas, node, type, range_min, range_max);
+ next = mas_slot(mas, ma_slots(node, type), mas->offset);
+ if (unlikely(ma_dead_node(node)))
+ return false;
+
+ if (unlikely(ma_is_leaf(type)))
+ return true;
+
+ /* Descend. */
+ mas->node = next;
+ }
+ return false;
+}
+
+/*
+ * mas_tree_walk() - Walk to @mas->index and set the range values.
+ * @mas: The maple state.
+ * @*range_min: The minimum range to be set.
+ * @*range_max: The maximum range to be set.
+ *
+ * Ranges are only valid if there is a valid entry at @mas->index.
+ *
+ * Return: True if a value exists, false otherwise.
+ */
+static inline bool mas_tree_walk(struct ma_state *mas, unsigned long *range_min,
+ unsigned long *range_max)
+{
+ bool ret;
+
+retry:
+ ret = false;
+ mas_start(mas);
+ if (mas_is_none(mas))
+ goto not_found;
+
+ if (mas_is_ptr(mas)) {
+ *range_min = *range_max = 0;
+ if (!mas->index)
+ return true;
+
+ goto not_found;
+ }
+
+ ret = mas_descend_walk(mas, range_min, range_max);
+ if (unlikely(mte_dead_node(mas->node))) {
+ mas->node = MAS_START;
+ goto retry;
+ }
+
+ return ret;
+
+not_found:
+ mas->offset = MAPLE_NODE_SLOTS;
+ return false;
+}
+
+static inline void *mas_range_load(struct ma_state *mas,
+ unsigned long *range_min, unsigned long *range_max)
+
+{
+ void *entry = NULL;
+ unsigned long index = mas->index;
+
+ if (mas_is_none(mas) || mas_is_paused(mas))
+ mas->node = MAS_START;
+retry:
+ if (mas_tree_walk(mas, range_min, range_max))
+ if (unlikely(mas->node == MAS_ROOT))
+ return mas_root(mas);
+
+ if (likely(mas->offset != MAPLE_NODE_SLOTS))
+ entry = mas_get_slot(mas, mas->offset);
+
+ if (mas_dead_node(mas, index))
+ goto retry;
+
+ return entry;
+}
static noinline void check_erase2_testset(struct maple_tree *mt,
unsigned long *set, unsigned long size)
{
projects / users / jedix / linux-maple.git / commitdiff