entry = _mte_get_rcu_slot(mn, slot, type);
if (mt_will_coalesce(entry)) {
if (piv == prev_piv) {
- printk("will coalesce %d\n", slot);
(*coalesce)++;
}
//counted_null = true;
} else if (!entry) {
if (counted_null) {
- printk("has value\n");
(*coalesce)++;
}
counted_null = true;
enum maple_type mas_type = mte_node_type(mas->node);
unsigned char ret = 0, half = mt_slots[mas_type] / 2;
unsigned char src_slot = 0, slot = 0;
- unsigned long piv, prev_piv = mas->min, written_piv = mas->min;
+ unsigned long piv = 0, prev_piv = mas->min, written_piv = mas->min;
bool attempt_insert = false;
+ bool appending = false;
MA_STATE(cp, mas->tree, mas->index, mas->last);
if (ma_is_leaf(mas_type))
attempt_insert = true;
- printk("%s %u\n", __func__, half);
mas_dup_state(&cp, left);
do {
void *existing = NULL;
- piv = mas_get_safe_pivot(mas, src_slot);
- existing = mte_get_rcu_slot(mas->node, src_slot);
- printk("%p[%u] (%lu)\n", mas_mn(mas), src_slot, piv);
+ if (entry_cnt <= 0) {
+ appending = true;
+ existing = NULL;
+ } else {
+ piv = mas_get_safe_pivot(mas, src_slot);
+ existing = mte_get_rcu_slot(mas->node, src_slot);
+ }
- if (mt_will_coalesce(existing)) {
- printk("coalesce\n");
- goto skip_src_slot;
- }
+ if (!appending) {
+ if (mt_will_coalesce(existing)) {
+ goto skip_src_slot;
+ }
- if (prev_piv > piv) {// insert overwrote this pivot.
- printk("prev_piv > piv\n");
- goto skip_src_slot;
- }
+ if (prev_piv > piv) {// insert overwrote this pivot.
+ goto skip_src_slot;
+ }
- if (src_slot && piv == prev_piv) {
- printk("piv == prev_piv..\n");
- goto skip_src_slot;
+ if (src_slot && piv == prev_piv) {
+ goto skip_src_slot;
+ }
}
// Insert happens in leaf
if (attempt_insert && (cp.index <= piv)) {
- printk("%lu %lu\n", prev_piv, piv);
if (written_piv != cp.index - 1) {
// Insert a previous entry..
mte_set_rcu_slot(cp.node, slot, existing);
slot++;
}
- printk("inserting new data at %p[%u]\n", cp.node, slot);
mte_set_rcu_slot(cp.node, slot, entry);
mte_set_pivot(cp.node, slot, cp.last);
slot++;
- if (piv > cp.last) {
+ if (!appending && (piv > cp.last)) {
mte_set_rcu_slot(cp.node, slot, existing);
mte_set_pivot(cp.node, slot, piv);
} else
attempt_insert = false;
} else {
- printk("Copy %u -> %p[%u]\n", src_slot, mas_mn(&cp), slot);
mte_set_pivot(cp.node, slot, piv);
if (mas_type == maple_arange_64)
mte_cp_gap(cp.node, slot, mas->node, src_slot);
if ((slot >= half) && (cp.node != right->node)) {
ret = slot;
left->max = piv;
- printk("Set %p max %lu\n", mas_mn(left), piv);
right->min = piv + 1;
- printk("Switch dst %d\n", entry_cnt);
mas_dup_state(&cp, right);
slot = 0;
} else {
MA_CP(cp, mas->node, left, 0, split);
- printk("cp %p[0-%u] to %p\n", mas_mn(mas), split, mte_to_node(left));
mas_copy(mas, &cp);
cp.src_start = split + 1;
MA_STATE(left, mas->tree, mas->index, mas->last);
MA_STATE(right , mas->tree, mas->index, mas->last);
+ printk("%s: %p\n", __func__, mas_mn(mas));
+ //mt_dump(mas->tree);
if (mte_is_root(mas->node)) {
old_parent = full;
if (mt_is_alloc(mas->tree))
else
ptype = maple_range_64;
p_slot = 0;
- printk("root\n");
} else {
unsigned long last_pivot;
unsigned char coalesce;
p_slot = mte_parent_slot(mas->node);
mas_dup_state(&parent, mas);
- printk("parent slot %u\n", p_slot);
mas_encoded_parent(&parent);
old_parent = parent.node;
ptype = mte_node_type(parent.node);
p_end = mas_data_end(&parent, ptype, &last_pivot, &coalesce);
if (p_end - coalesce >= mt_slots[ptype] - 1) {
- printk("Split the parent\n");
/* Must split the parent */
split = mas_split(&parent, p_slot, active,
p_end - coalesce + 1, entry);
// split the data into left & right and do the insert.
split = mas_do_split(mas, &left, &right, entry_cnt, entry);
- // KEEP REWRITING THIS DOWN WITH left/right/parent states.
- //
// Copy the parents information
if (!mte_is_root(full)) {
MA_CP(cp, old_parent, new_parent, 0, p_slot - 1);
cp.src_start = p_slot + 1;
cp.src_end = p_end;
cp.dst_start += skip;
- printk("Copy anything larger than %lu\n", right.max);
_mas_copy(&parent, &cp, right.max);
}
}
if (right.node)
mte_link(right.node, new_parent, p_slot + 1,
right.max, ptype);
-#if 0
- if (mt_is_alloc(mas->tree)) {
- if (right->node) {
- unsigned long min = mas->min;
- mas->node = right;
- mas_gap_link(mas, new_parent, p_slot + 1, mas->max);
- mas->min = min;
- }
- mas->node = left;
- mas_gap_link(mas, new_parent, p_slot, left.max);
- }
-#endif
// Copy grand parent to the parent, including slot encoding.
mte_to_node(new_parent)->parent = mte_to_node(old_parent)->parent;
// Update encoded slots in children
p_slot += 2;
}
- // If we are going to insert into the parent..?
- if (mas->index > mt_node_max(mas->node) &&
- mas->index > mt_node_max(mas->node) + mas->min) {
- // FIXME: Do the insert
- mas->node = new_parent;
- split = p_slot;
- }
-
// Free the full node.
if (old_parent != full) {
if (!active)
end = mas_data_end(mas, mte_node_type(mas->node), &last_piv, &coalesce);
req_slots = end - coalesce + 1;
- printk("Start with %d, req_slots %d %lu\n", end, req_slots, last_piv);
if (max == mas->max && min == mas->min)
return req_slots; // overwriting a single entry.
if (min < mas->index) {
- printk("min inc\n");
req_slots++; //slot needed at start.
}
if (max > mas->last) {
- printk("max inc\n");
req_slots++; // slot needed at end.
}else {
unsigned char end_slot = end;
break;
end_slot--;
}
- printk("Can go down slots?\n");
req_slots -= (end_slot - slot);
}
/* Check for splitting */
new_end = _mas_add_slot_cnt(mas, slot, min, max);
- printk("into %p old_end %u new %u %u\n", mas_mn(mas), old_end, new_end, slot);
if (mas->index > min)
null_entry = true;
if (new_end > slot_cnt) {
/* Not enough space in the node */
- printk("Split %u > %u, %lu %lu\n", new_end, slot_cnt, mas->min, mas->max);
unsigned char split = mas_split(mas, slot, active, new_end,
entry);
if (mas_is_err(mas))
return 0;
- if (mte_is_leaf(mas->node))
- return old_end - new_end;
-
- if (split <= slot)
- slot = slot - split;
-
- mas_set_slot(mas, slot);
- return _mas_add(mas, entry, overwrite, active);
+ return old_end - new_end;
}
prev_enode = mas->node;
enum maple_type mt;
unsigned char child_slot = 0;
- printk("not a leaf\n");
// Create a new node.
mas_node_cnt(mas, 1);
if (mas_is_err(mas))
leaf_state.node = leaf;
leaf_state.min = mas->index;
leaf_state.max = max;
- printk("leaf %lu %lu\n", mas->index, max);
if (null_entry)
child_slot = 1;
mas_set_slot(&leaf_state, child_slot);
slot += 2;
} else {
if (overwrite && slot) {
- printk("Setting %p[%u] %lu\n", mas_mn(mas), slot - 1, mas->index-1);
mte_set_pivot(mas->node, slot-1, mas->index - 1);
}
* _mas_next() - Finds the next entry, sets index to the start of the range.
*
*/
-static inline void *_mas_next(struct ma_state *mas, unsigned long max,
+static inline void *_mas_next(struct ma_state *mas, unsigned long limit,
unsigned long *range_start)
{
void *entry = NULL;
if (mas->node && !mas_searchable(mas))
return NULL;
- if (!mas->node || mas_is_start(mas)) // First run.
+ if (!mas->node || mas_is_start(mas)) {// First run.
+ *range_start = 0;
+ mas_start(mas);
entry = mas_range_load(mas, range_start, &range_max);
+ }
if (entry)
return entry;
- return __mas_next(mas, max, range_start);
+ return __mas_next(mas, limit, range_start);
}
/*
MA_CP(cp, this_enode, NULL, 0, end);
l_slot_cnt = ma_hard_data(end, coalesce);
- printk("l_slot_cnt %u %u %u\n", end, coalesce, l_slot_cnt);
if (mte_is_root(this_enode))
return mas_coalesce_node(mas, end, coalesce, true);
mas_descend(&p_state);
end = mas_data_end(&p_state,
mte_node_type(p_state.node), &l_piv, &coalesce);
- printk("%p end %u\n", mas_mn(&p_state), end);
return mas_rebalance(&p_state, end, coalesce);
}
// No left or right, rebalance the parent.
mas_descend(&r_state);
r_enode = r_state.node;
- printk("Using right %p\n", mas_mn(&r_state));
r_end = mas_data_end(&r_state, mte_node_type(r_enode), &l_piv, &r_coalesce);
r_slot_cnt = ma_hard_data(r_end, r_coalesce);
- printk("r_end %u %u\n", r_end, r_slot_cnt);
if (r_end < r_coalesce) {
// This node needs to be a skip entry.
all_slots = l_slot_cnt;
}
// Add 1 for each slot 0.
all_slots = r_slot_cnt + 2 + l_slot_cnt;
- printk("Total slot count %u\n", all_slots);
- printk("Using right %p\n", mas_mn(&r_state));
node_cnt = 1;
if (all_slots > mt_slot_count(this_enode) - 1)
node_cnt++;
- printk("Total slot count %u nodes %u\n", all_slots, node_cnt);
mas_node_cnt(mas, node_cnt);
if (mas_is_err(mas))
return 0;
// Coalesce this_enode into a new node.
cp.dst_end = l_slot_cnt;
- printk("copy from %p[%u-%u] to %p[%u-%u]\n", cp.src, cp.src_start, cp.src_end,
- cp.dst, cp.dst_start, cp.dst_end);
mas_copy(mas, &cp); // cp.dst now has coalesced this_enode.
- printk("dst_start = %u src_start %u\n", cp.dst_start, cp.src_start);
// check if left ends in NULL, right start in NULL..
- printk("Check %p[%u] %p\n", cp.dst, cp.dst_start - 1, mte_get_rcu_slot(cp.dst, cp.dst_start - 1));
if (!mte_get_rcu_slot(cp.dst, cp.dst_start - 1))
l_null_end = true;
- printk("Check %p[%u] %p\n", r_enode, 0, mte_get_rcu_slot(r_enode, 0));
if (mt_will_coalesce(mte_get_rcu_slot(r_enode, 0)) ||
!mte_get_rcu_slot(r_enode, 0))
r_null_start = true;
if (l_null_end && r_null_start)
{
- printk("r_enode %p[0] = %p\n", mas_mn(&r_state), mte_get_rcu_slot(r_enode, 0));
all_slots--;
- // Can remove a lost.
} else if (!l_null_end && !r_null_start) {
// Need an null if the pivots don't line up here.
mte_set_pivot(cp.dst, cp.dst_start++, r_state.min);
else
cp.dst_end = (all_slots + 1)/ 2; // Take 1/2 the entries.
- printk("copy from %p[%u-%u] to %p[%u-%u]\n", cp.src, cp.src_start, cp.src_end,
- cp.dst, cp.dst_start, cp.dst_end);
-
mas_copy(&r_state, &cp); // cp.dst is now complete, place it in the tree.
mte_to_node(cp.dst)->parent = mte_to_node(this_enode)->parent;
new_type = mte_node_type(cp.dst);
mas_set_slot(mas, 0);
}
done:
+ printk("Set min %lu\n", min);
mas_set_slot(mas, i);
*range_max = max;
*range_min = min;
static inline bool mas_search_cont(struct ma_state *mas, unsigned long index,
unsigned long max, void *entry)
{
+ if (mas_is_start(mas))
+ return true;
+
if (index >= max)
return false;
EXPORT_SYMBOL(mt_find);
static inline void ma_inactive_insert(struct ma_state *mas, void *entry);
+
+
+/* Private
+ * mas_replace_tree() - Build a new tree and replace the entire structure.
+ *
+ */
static inline int mas_replace_tree(struct ma_state *mas, void *new_entry)
{
- long l_node_cnt = 0, r_node_cnt = 0;
- unsigned int r_slot_cnt = 0, slot_cnt = 0;
- long node_cnt = 0, nodes = 0;
void *entry;
- struct maple_enode *r_node = NULL, *last = NULL;
- unsigned char r_slot = 0, slot;
+ unsigned long piv;
+ unsigned char coalesce;
+ unsigned int slot_cnt = 0;
+ long node_cnt = 0, leaves= 0;
+ struct maple_enode *last = NULL;
+ enum maple_type p_type = mas_parent_enum(mas, mas->node);
// Create a new tree.
DEFINE_MTREE(new_tree);
MA_STATE(new_mas, &new_tree, 0, 0);
+ MA_STATE(r_mas, mas->tree, mas->last + 1, mas->last + 1);
+ MA_STATE(l_mas, mas->tree, 0, 0);
+ // Count the slots that will be used in the node we landed.
+ slot_cnt = 3 + mas_get_slot(mas); // 3 is the max a new entry can create.
- MA_STATE(r_mas, mas->tree, mas->last + 1, mas->last + 1);
- slot_cnt = 1 + mas_get_slot(mas);
+ // Count the nodes that are currently used to the left.
mas_set_slot(mas, mte_parent_slot(mas->node));
- while (mas->node != MAS_NONE) {
+ while (!mas_is_none(mas)) {
last = mas->node;
mas_prev_node(mas, 0);
- l_node_cnt++;
+ leaves++;
}
-
+ // Set mas->node to a valid node.
mas->node = last;
- _mas_walk(&r_mas);
- if (mas_get_slot(&r_mas) != MAPLE_NODE_SLOTS) {
- unsigned long piv;
- unsigned char coalesce;
- r_slot_cnt += mas_data_end(&r_mas, mte_node_type(r_mas.node),
- &piv, &coalesce);
- if (piv != ULONG_MAX) {
- unsigned char p_slot = mte_parent_slot(r_mas.node);
-
- r_node = r_mas.node;
- r_slot = mas_get_slot(&r_mas);
- r_slot_cnt -= r_slot;
- slot_cnt += r_slot_cnt;
- mas_set_slot(&r_mas, p_slot);
- while (r_mas.node != MAS_NONE) {
- last = r_mas.node;
- mas_next_node(&r_mas, ULONG_MAX);
- r_node_cnt++;
- }
- r_mas.node = last;
- }
- if (r_mas.max < mas->last)
- slot_cnt++;
+ // Walk down to the right side of the tree.
+ _mas_walk(&r_mas);
+ // Add the slots to the right of where the search landed.
+ if (mas_get_slot(&r_mas) == MAPLE_NODE_SLOTS) {
+ r_mas.node = MAS_NONE;
+ goto skip_r_count;
+ }
+ slot_cnt -= mas_get_slot(&r_mas);
+ slot_cnt += mas_data_end(&r_mas, mte_node_type(r_mas.node), &piv,
+ &coalesce);
+
+ // Count the nodes to the right.
+ mas_set_slot(&r_mas, mte_parent_slot(r_mas.node));
+ while (!mas_is_none(&r_mas)) {
+ last = r_mas.node;
+ mas_next_node(&r_mas, ULONG_MAX);
+ leaves++;
}
+ r_mas.node = last;
+skip_r_count:
+ // Calculate all the nodes needed for a new tree.
if (slot_cnt > mt_slot_count(mas->node))
- nodes++;
+ leaves++;
- nodes = l_node_cnt + r_node_cnt + 2;
node_cnt = 1; // Root node.
- while (nodes) {
- node_cnt += nodes;
- nodes /= 4;
+ while (leaves) { // add the number of nodes at each level.
+ node_cnt += leaves;
+ leaves /= mt_slots[p_type];
}
-
- // FIXME: When splitting & reusing we need an extra node.
- mas_node_cnt(mas, node_cnt + 1);
+ mas_node_cnt(mas, node_cnt);
if (mas_is_err(mas))
return 0;
+ /* Move allocations from mas to new_mas.
+ * NOTE: This is necessary as mas will pass back errors and will retry
+ * the allocation, so it has to be done in mas and has to be moved for
+ * below.
+ */
new_mas.alloc = mas->alloc;
mas->alloc = NULL;
// Copy left side
- mas_set_slot(mas, 0);
- new_mas.last = mas_first_entry(mas, mas->index);
- slot = mas_get_slot(mas);
- do {
- new_mas.index = mas->min;
- new_mas.last = mas->min;
- while (slot < mt_slot_count(mas->node)) {
- if (new_mas.index == mas->index) {
- break;
- }
- new_mas.last = mas_get_safe_pivot(mas, slot);
- if (!new_mas.last && slot)
- break;
-
- if (new_mas.last > mas->index)
- new_mas.last = mas->index - 1;
-
- entry = mte_get_rcu_slot(mas->node, slot);
- if (entry) {
- int cnt = mas_get_alloc_cnt(&new_mas);
- ma_inactive_insert(&new_mas, entry);
- if (mas_is_err(&new_mas))
- BUG_ON(1);
- if (cnt < mas_get_alloc_cnt(&new_mas) - 1)
- BUG_ON(1);
- }
-
- if (new_mas.last == mas->index)
- break;
-
- new_mas.index = new_mas.last + 1;
-
- slot++;
- }
- mas_set_slot(mas, mte_parent_slot(mas->node));
- mas_next_node(mas, mas->index);
- slot = 0;
- } while (!mas_is_none(mas) && mas->min < mas->index);
+ mas_for_each(&l_mas, entry, mas->index - 1) {
+ new_mas.index = l_mas.index;
+ new_mas.last = mas_get_safe_pivot(&l_mas, mas_get_slot(&l_mas));
+ printk("Insert %lu->%lu\n", new_mas.index, new_mas.last);
+ ma_inactive_insert(&new_mas, entry);
+ if (mas_is_err(&new_mas))
+ BUG_ON(1);
+ mt_dump(new_mas.tree);
+ }
// Insert the new value.
new_mas.index = mas->index;
* - new_mas.last ends in a null
* - new_mas.last has a sequentially next value (48)
*/
- if (!r_node) // No entry beyond new_mas.last
+ if (mas_is_none(&r_mas)) // No entry beyond new_mas.last
goto skip_right;
- new_mas.index = new_mas.last + 1;
- new_mas.last = mte_get_pivot(r_node, r_slot);
-
- mas_set_slot(mas, r_slot);
- mas->node = r_node;
- mas->min = new_mas.index;
- do {
- new_mas.index = mas->min;
- new_mas.last = mas->min;
- slot = mas_get_slot(mas);
- while (slot++ < mt_slot_count(mas->node)) {
- new_mas.last = mas_get_safe_pivot(mas, slot);
- if (!new_mas.last)
- break;
-
- entry = mte_get_rcu_slot(mas->node, slot);
- if (entry) {
- ma_inactive_insert(&new_mas, entry);
- if (mas_is_err(&new_mas))
- BUG_ON(1);
- }
+ mas_for_each(&l_mas, entry, ULONG_MAX) {
+ if (l_mas.index < r_mas.index)
+ continue;
- new_mas.index = new_mas.last + 1;
- }
- mas_next_node(mas, mas->index);
- mas_set_slot(mas, 0);
- } while (mas->node != MAS_NONE);
+ new_mas.index = l_mas.index;
+ new_mas.last = mas_get_safe_pivot(&l_mas, mas_get_slot(&l_mas));
+ ma_inactive_insert(&new_mas, entry);
+ if (mas_is_err(&new_mas))
+ BUG_ON(1);
+ }
skip_right:
- r_node = mas->tree->ma_root;
+ last = mas->tree->ma_root;
mas->node = new_tree.ma_root;
_mas_replace(mas, false, false);
mas->node = 0;
mas->alloc = new_mas.alloc;
- mte_destroy_walk(r_node);
+ mte_destroy_walk(last);
- // Copy right side
return node_cnt;
}
static inline bool mas_rewind_node(struct ma_state *mas);
projects / users / jedix / linux-maple.git / commitdiff