[maple_sparse_64] = MAPLE_SPARSE64_SLOTS / 2,
[maple_leaf_16] = MAPLE_RANGE16_SLOTS / 2,
[maple_leaf_32] = MAPLE_RANGE32_SLOTS / 2,
- [maple_leaf_64] = MAPLE_RANGE64_SLOTS / 2,
+ [maple_leaf_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
[maple_range_16] = MAPLE_RANGE16_SLOTS / 2,
[maple_range_32] = MAPLE_RANGE32_SLOTS / 2,
- [maple_range_64] = MAPLE_RANGE64_SLOTS / 2,
+ [maple_range_64] = (MAPLE_RANGE64_SLOTS / 2) - 2,
[maple_arange_64] = (MAPLE_ARANGE64_SLOTS + 1) / 2,
};
#define mt_min_slot_cnt(x) mt_min_slots[mte_node_type(x)]
}
mte_set_pivot(mas->node, slot, val);
}
-static inline struct maple_enode *_mte_get_rcu_slot(
- const struct maple_enode *mn, unsigned char slot,
+static inline struct maple_enode *ma_get_rcu_slot(
+ const struct maple_node *mn, unsigned char slot,
enum maple_type type)
{
switch (type) {
case maple_range_64:
case maple_leaf_64:
- return rcu_dereference(mte_to_node(mn)->mr64.slot[slot]);
+ return rcu_dereference(mn->mr64.slot[slot]);
default:
case maple_dense:
- return rcu_dereference(mte_to_node(mn)->slot[slot]);
+ return rcu_dereference(mn->slot[slot]);
case maple_arange_64:
- return rcu_dereference(mte_to_node(mn)->ma64.slot[slot]);
+ return rcu_dereference(mn->ma64.slot[slot]);
case maple_sparse_6:
- return rcu_dereference(mte_to_node(mn)->ms6.slot[slot]);
+ return rcu_dereference(mn->ms6.slot[slot]);
case maple_sparse_9:
- return rcu_dereference(mte_to_node(mn)->ms9.slot[slot]);
+ return rcu_dereference(mn->ms9.slot[slot]);
case maple_sparse_16:
- return rcu_dereference(mte_to_node(mn)->ms16.slot[slot]);
+ return rcu_dereference(mn->ms16.slot[slot]);
case maple_sparse_21:
- return rcu_dereference(mte_to_node(mn)->ms21.slot[slot]);
+ return rcu_dereference(mn->ms21.slot[slot]);
case maple_sparse_32:
- return rcu_dereference(mte_to_node(mn)->ms32.slot[slot]);
+ return rcu_dereference(mn->ms32.slot[slot]);
case maple_sparse_64:
- return rcu_dereference(mte_to_node(mn)->ms64.slot[slot]);
+ return rcu_dereference(mn->ms64.slot[slot]);
case maple_range_16:
case maple_leaf_16:
- return rcu_dereference(mte_to_node(mn)->mr16.slot[slot]);
+ return rcu_dereference(mn->mr16.slot[slot]);
case maple_range_32:
case maple_leaf_32:
- return rcu_dereference(mte_to_node(mn)->mr32.slot[slot]);
+ return rcu_dereference(mn->mr32.slot[slot]);
}
}
+static inline struct maple_enode *_mte_get_rcu_slot(
+ const struct maple_enode *mn, unsigned char slot,
+ enum maple_type type)
+{
+ return ma_get_rcu_slot(mte_to_node(mn), slot, type);
+}
static inline struct maple_enode *mte_get_rcu_slot(const struct maple_enode *mn,
unsigned char slot)
{
static inline void mas_ascend(struct ma_state *mas)
{
- struct maple_enode *p_enode;
- struct maple_node *p_node;
- unsigned char p_slot;
- enum maple_type p_type;
+ struct maple_enode *p_enode; // parent enode.
+ struct maple_enode *a_enode = mas->node; // ancestor enode.
+ struct maple_node *a_node = mas_mn(mas); // ancestor node.
+ unsigned char a_slot = 0;
+ enum maple_type a_type = mte_node_type(mas->node);
+ unsigned long max = 0, min = ULONG_MAX;
+ bool set_max = false, set_min = false;
+
+ //mt_dump(mas->tree);
+ //printk("ascending from %p\n", mas_mn(mas));
+ p_enode = mt_mk_node(mte_parent(mas->node),
+ mas_parent_enum(mas, mas->node));
- p_type = mas_parent_enum(mas, mas->node);
- p_node = mte_parent(mas->node);
- p_slot = mte_parent_slot(mas->node);
- p_enode = mt_mk_node(p_node, p_type);
- if (_ma_is_root(p_node)) {
- mas->min = 0;
- mas->max = mt_max[p_type];
- } else if (!p_slot || p_slot >= mt_pivots[p_type]) {
- mas->node = p_enode;
- mas_ascend(mas);
- mas_set_slot(mas, p_slot);
- mas_descend(mas);
- } else {
- struct maple_enode *gp_enode = mt_mk_node(mte_parent(p_enode),
- mas_parent_enum(mas, p_enode));
- unsigned char gp_slot = mte_parent_slot(p_enode);
- mas->node = gp_enode;
- mas_set_slot(mas, gp_slot);
- mas_update_limits(mas, gp_slot, mas_parent_enum(mas, p_enode));
+
+ if (_ma_is_root(a_node))
+ goto no_parent;
+
+ a_type = mas_parent_enum(mas, mas->node);
+ a_enode = p_enode;
+
+ if (mte_is_root(a_enode)) {
+ a_node = mte_to_node(a_enode);
+ //printk("parent is root\n");
+ goto no_parent;
+ }
+
+ mas->node = p_enode;
+ascend:
+ a_type = mas_parent_enum(mas, mas->node);
+ a_node = mte_parent(mas->node);
+ a_slot = mte_parent_slot(mas->node);
+ a_enode = mt_mk_node(a_node, a_type);
+
+ //printk("a_slot = %u %s %s %u\n", a_slot, set_min ? "yes" : "no",
+ // set_max ? "yes" : "no", mt_pivots[a_type]);
+ if (!set_min && a_slot) {
+ set_min = true;
+ min = mte_get_pivot(a_enode, a_slot - 1) + 1;
+ //printk("Using min from %p[%u] %lu\n", a_node, a_slot-1, min);
+ }
+
+ if (!set_max && a_slot < mt_pivots[a_type]) {
+ set_max = true;
+ //printk("Getting pivot %u from %p\n", a_slot, a_enode);
+ max = mte_get_pivot(a_enode, a_slot);
+ //printk("Using max from %p[%u] %lu\n", a_node, a_slot, max);
}
+
+no_parent:
+ if (_ma_is_root(a_node)) {
+ //printk("%s: hit root %p\n", __func__, a_node);
+ if (!set_min)
+ min = 0;
+ if (!set_max)
+ max = mt_max[a_type];
+ }
+
+ if (!max || min == ULONG_MAX) {
+ mas->node = a_enode;
+ goto ascend;
+ }
+
+ mas->max = max;
+ mas->min = min;
mas->node = p_enode;
+ //printk("%p %lu-%lu\n", mas_mn(mas), mas->min, mas->max);
+ return;
}
/* mas_get_prev_pivot() - Return the previous pivot.
void *entry;
piv = _mas_get_safe_pivot(mas, slot, type);
- printk("%p[%u] piv %lu\n", mas_mn(mas), slot, piv);
if (!piv && slot) { // Past the end of data.
slot--;
piv = prev_piv;
(*coalesce) = 0;
break;
}
- printk("data end with counted nulls %p\n", mas_mn(mas));
- printk("%u %u %u\n", slot, (*coalesce), counted_null);
+// printk("data end with counted nulls %p\n", mas_mn(mas));
+// printk("%u %u %u\n", slot, (*coalesce), counted_null);
(*coalesce) = (*coalesce) - counted_null + 1;
slot -= counted_null;
}
}
if (piv == mas->max) {
- printk("max hit %lu\n", mas->max);
+// printk("max hit %lu\n", mas->max);
break;
}
prev_piv = piv;
}
- printk("last piv %lu slot %u coalesce %u\n", piv, slot, *coalesce);
+// printk("last piv %lu slot %u coalesce %u\n", piv, slot, *coalesce);
*last_piv = piv;
return slot;
}
unsigned long piv)
{
left->max = piv;
- printk("going to use %lu as max\n", left->max);
+// printk("going to use %lu as max\n", left->max);
right->min = piv + 1;
mas_dup_state(cp, right);
return mas_mn(right);
enum maple_type type, unsigned char slot,
unsigned long piv, void *entry, bool flush)
{
- //printk("set %p[%u] -> %p\n", mn, slot, entry);
+ printk("%s: set %p[%u] -> %p\n", __func__, mn, slot, entry);
ma_set_rcu_slot(mn, slot, type, entry);
if (flush)
wmb(); // data needs to exist before pivot for readers
- if (slot < mt_pivots[type])
+ if (slot < mt_pivots[type]) {
+ printk("%s: set %p[%u] piv %lu\n", __func__, mn, slot, piv);
ma_set_pivot(mn, slot, type, piv);
+ }
}
static inline unsigned char mas_cp_calc_split(struct ma_state *mas,
* mas_ma_cp() - copy mas->node to mn (or left->node and right->node for
* splits).
*
+ * @mas - src
+ * @p_slot - parent slot - used to update gaps on split.
+ * @left - destination 1 mas
+ * @right - destination 2 mas - if not null, treat this as a split call.
+ * @mn - destination 1 node - if null, taken from @left->node - exists for
+ * inactive (unallocated) nodes.
+ * @mn_type - destination 1 node type - same as @mn, taken from @left->node.
+ * @entry - entry to insert at mas.index/mas.last
+ * @dst_start- destination start slot. If non-zero, implies append mode.
+ *
* Trying to calculate a split is a rather difficult task. One has to
* remember:
- * 1. Keep gaps at the end of a node
+ * 1. Keep gaps at the end of a node - might not be done right now?
* 2. Coalescing may affect the middle position of data
* 3. Use the target slot and the new count to better get a 50/50 split.
*
static inline unsigned char mas_ma_cp(struct ma_state *mas,
unsigned char p_slot, struct ma_state *left,
struct ma_state *right, struct maple_node *mn,
- enum maple_type mn_type, int entry_cnt, void *entry,
- bool append)
+ enum maple_type mn_type, unsigned long mn_min,
+ int entry_cnt, void *entry, unsigned char dst_start)
{
enum maple_type mas_type = mte_node_type(mas->node);
struct maple_node *parent = mte_to_node(mas->node);
unsigned long max_gap = 0;
bool attempt_insert = false;
bool appending = false;
+ bool append = false;
void *existing = NULL;
bool null_run = false;
+ bool prev_null = false;
bool update_gaps = mt_is_alloc(mas->tree);
unsigned long piv[3] = {
mas->index - 1,
MA_STATE(cp, mas->tree, mas->index, mas->last);
+ if (dst_start)
+ append = true;
+
if (ma_is_leaf(mas_type))
attempt_insert = true;
if (right) {
split = mas_cp_calc_split(mas, mas_type, append);
- // printk("Using split of %d\n", split);
+ printk("Using split of %d\n", split);
}
if (left) {
mn_type = mte_node_type(cp.node);
p_slot = mte_parent_slot(left->node);
parent = mte_parent(left->node);
+ printk("parent %p of %p\n", parent, mas->node);
mas_type = mas_parent_enum(mas, left->node);
- // printk("parent is %p\n", parent);
}
if (!entry_cnt)
piv[2] = mas->max;
if (append) { // start at a given slot, so append there.
- // printk("append\n");
+ printk("append\n");
//mt_dump(mas->tree);
mas_dup_state(&cp, mas);
src_slot = entry_cnt;
- existing = mte_get_rcu_slot(mas->node, src_slot);
- written_piv = mas_get_safe_pivot(mas, src_slot);
- if (!written_piv) // empty node.
- written_piv = mas->min;
-
- // printk("src_slot %u written %lu\n", src_slot, written_piv);
- slot = entry_cnt;
- if (existing) {
+ slot = dst_start;
+ if (mn == mas_mn(&cp)) // src and dst are the same.
+ slot = entry_cnt;
+
+ existing = ma_get_rcu_slot(mn, slot, mn_type);
+ written_piv = mn_min;
+ if (slot)
+ written_piv = ma_get_pivot(mn, slot, mn_type);
+
+ printk("slot %u written %lu src_slot %u\n", slot, written_piv, src_slot);
+ if (mn == mas_mn(&cp)) { // src and dst are the same.
+ printk("src == dst\n");
+ if (existing || (!slot && written_piv < cp.index)) {
+ src_slot++;
+ slot++;
+ existing = NULL;
+ }
+ entry_cnt = 0;
+ } else if (existing && written_piv != cp.index) {
slot++;
+ entry_cnt = -1;
+ printk("Existing\n");
+ } else {
src_slot++;
- // printk("there is an existing\n");
+ entry_cnt = 0;
}
- entry_cnt = 0;
+
+ printk("slot %u written %lu src_slot %u\n", slot, written_piv, src_slot);
+
appending = true;
prev_piv = written_piv;
attempt_insert = true;
max_gap = mas_leaf_max_gap(mas);
}
+ printk("slot is %u\n", slot);
do {
int i = 2; // Default to just writing the pivot.
int j = 3; // Loop limit.
- // printk("src slot %u entry_cnt %d slot %u\n", src_slot, entry_cnt, slot);
+ printk("src slot %u entry_cnt %d slot %u\n", src_slot, entry_cnt, slot);
if (entry_cnt < 0) {
- // printk("Setting to appending mode\n");
+ printk("Setting to appending mode\n");
appending = true;
existing = NULL;
piv[2] = mas->max;
if (!piv[2] && src_slot)
piv[2] = mas->max;
existing = mte_get_rcu_slot(mas->node, src_slot);
- // printk("getting %u (%lu->%p)\n", src_slot, piv[2],
- // existing);
+ printk("getting %p[%u] (%lu->%p)\n", mas_mn(mas),
+ src_slot, piv[2], existing);
}
if (!appending) {
if (mt_will_coalesce(existing)) {
- // printk("coalesce\n");
+ printk("coalesce\n");
+ if (prev_null) {
+ if (slot && slot - 1 < mt_pivots[mas_type]) {
+ printk("Overwrite piv %u\n", slot - 1);
+ ma_set_pivot(mn, slot - 1, mas_type,
+ piv[2]);
+ written_piv = piv[2];
+ }
+ if (update_gaps) {
+ mas_ma_update_gap(mas, mas_type, mn, mn_type,
+ piv[2], written_piv, src_slot,
+ slot, &max_gap, NULL);
+ }
+ }
goto skip_src_slot;
}
goto skip_src_slot;
}
- if (append || (attempt_insert && (cp.index <= piv[2]))) {
+ if (ma_is_leaf(mn_type) &&
+ (append || (attempt_insert && cp.index <= piv[2]))) {
i = 0;
- // printk("Written piv %lu\n", written_piv);
- if (written_piv == cp.index - 1)
+ printk("Written piv %lu\n", written_piv);
+ if (written_piv == cp.index - 1) {
i = 1; // previous pivot matches exactly.
+ }
else if (!src_slot && written_piv == cp.index)
- i = 1; // mas->min matches exactly.
+ i = 1;
- // printk("piv2 %lu <= last %lu\n", piv[2], cp.last);
+ printk("piv2 %lu <= last %lu\n", piv[2], cp.last);
if (appending)
j--;
else if (!appending && (piv[2] <= cp.last))
j--; // skip last pivot.
- // printk("piv array %u-%u\n", i, j);
+ printk("piv array %u-%u\n", i, j);
attempt_insert = false;
}
if (!loop_entry && !ma_is_leaf(mn_type))
{
- // printk("BUG Here\n");
+ printk("BUG Here\n");
}
- // printk("write array %d %lu to %u entry_cnt %d\n", i, piv[i], slot, entry_cnt);
+ printk("write array %d %lu to %u entry_cnt %d\n", i, piv[i], slot, entry_cnt);
if (i == 1)
loop_entry = entry;
if (append)
wmb = true;
- if (mt_is_empty(loop_entry)) {
+ if (mt_is_empty(loop_entry) ||
+ xa_is_retry(loop_entry)) {
if (null_run)
slot--;
slot, &max_gap, loop_entry);
}
+ if (!loop_entry)
+ prev_null = true;
+ else
+ prev_null = false;
+
written_piv = piv[i];
slot++;
// If this is a split operation, right will exist
// and the target may need to be switched.
if ((right) && (cp.node != right->node) &&
(slot > split)) {
- ret = slot - 1;
+ ret = slot;
if (update_gaps) {
- // printk("Write gap %p[%u] -> %lu\n", parent, p_slot, max_gap);
+ printk("Write gap %p[%u] -> %lu\n", parent, p_slot, max_gap);
parent->ma64.gap[p_slot] = max_gap;
}
prev_piv = piv[2];
skip_src_slot:
src_slot++;
- } while (entry_cnt-- > 0 || attempt_insert);
+ } while ((entry_cnt-- > 0) || attempt_insert);
if (right && update_gaps)
parent->ma64.gap[p_slot] = max_gap;
- else
- ret = src_slot - 1;
+ else if (!right)
+ ret = slot - 1;
- // printk("Done\n");
+ printk("Done\n");
return ret;
}
// Get the leaf slot.
mas_set_slot(mas, 0);
mas_first_node(mas, limit);
+ if (mas_is_none(mas))
+ return limit;
return mas_get_safe_pivot(mas,
mas_get_slot(mas));
}
void *entry;
printk("%u to %u\n", sloc, dloc);
- if (prev_piv >= mas->max)
+ if (prev_piv >= mas->max) {
+ printk("%d %u %lu >= %lu\n", __LINE__, sloc, prev_piv, mas->max);
break;
+ }
- if (sloc < pivot_cnt)
+ if (sloc < pivot_cnt) {
piv = mte_get_pivot(cp->src, sloc);
- else
+ printk("%d\n", __LINE__);
+ }
+ else {
+ printk("%d\n", __LINE__);
piv = mas->max;
+ }
if (sloc && !piv)
break;
entry = mte_get_rcu_slot(cp->src, sloc);
if (mt_will_coalesce(entry)) {
+ printk("%d\n", __LINE__);
printk("slot %u will coalesce\n", sloc);
if (!sloc) {
printk("HERE!\n");
goto next_src_slot;
}
}
+ else
+ goto next_src_slot;
}
else
goto next_src_slot;
// Last entry.
if (dloc && (piv == mas->max || !piv)) {
+ printk("%d\n", __LINE__);
if (!mte_get_rcu_slot(cp->dst, dloc -1) &&
mt_is_empty(entry)) {
mte_set_pivot(cp->dst, dloc -1, 0);
+ printk("%d\n", __LINE__);
break;
}
}
- if (piv < cp->start_piv)
+ if (piv < cp->start_piv) {
+ printk("start_piv ski\n");
goto next_src_slot;
+ }
- if (sloc && piv == prev_piv)
+ if (sloc && piv == prev_piv) {
+ printk("skip slot\n");
goto next_src_slot;
+ }
if (dloc < pivot_cnt) {
- printk("piv %p[%u] -> %lu\n", cp->dst, dloc, piv);
+ printk("dloc\n");
mte_set_pivot(cp->dst, dloc, piv);
}
if (dtype == maple_arange_64)
mte_cp_gap(cp->dst, dloc, cp->src, sloc);
- printk("%p[%u] -> %p[%u]\n", cp->src, sloc, cp->dst, dloc);
mte_cp_rcu_slot(cp->dst, dloc++, cp->src, sloc);
prev_piv = piv;
next_src_slot:
break;
child = _mte_get_rcu_slot(parent, slot, type);
- if (child)
+ if (!mt_is_empty(child))
mte_set_parent(child, parent, slot);
}
}
MA_STATE(left, mas->tree, mas->index, mas->last);
MA_STATE(right , mas->tree, mas->index, mas->last);
- printk("%s\n", __func__);
+ printk("%s %p\n", __func__, mas_mn(mas));
type = mte_node_type(mas->node);
if (mte_is_root(mas->node)) {
old_parent = full;
mas_dup_state(&parent, mas);
mas_ascend(&parent);
old_parent = parent.node;
+ printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas),
+ mas_mn(&parent), parent.max);
ptype = mte_node_type(parent.node);
p_end = mas_data_end(&parent, ptype, &last_pivot, &coalesce);
if (mas_is_err(mas))
return 0;
+ printk("Parent split returned\n");
+ mt_dump(mas->tree);
mas_dup_state(&parent, mas);
if (split < p_slot)
p_slot -= split;
mas_set_slot(&parent, p_slot);
}
ptype = mte_node_type(parent.node);
+ printk("end of node %p\n", mas_mn(&parent));
p_end = mas_data_end(&parent, ptype, &last_pivot, &coalesce);
mas_dup_state(mas, &parent);
mas_set_slot(mas, p_slot);
+ printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas),
+ mas_mn(&parent), parent.max);
mas_descend(mas);
+ printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas),
+ mas_mn(&parent), parent.max);
}
+ printk("%s mas %p parent %p max %lu\n", __func__, mas_mn(mas),
+ mas_mn(&parent), parent.max);
+
mas_node_cnt(mas, 3);
if (mas_is_err(mas))
return 0;
mte_set_parent(left.node, new_parent, p_slot);
mte_set_parent(right.node, new_parent, p_slot+1);
// split the data into left & right and do the insert.
- split = mas_ma_cp(mas, p_slot, &left, &right, NULL, type, entry_cnt,
- entry, false);
- right.max = mas->max;
+ split = mas_ma_cp(mas, p_slot, &left, &right, NULL, type, 0, entry_cnt,
+ entry, 0);
// Copy the parents information
if (!mte_is_root(full)) {
if (right.node)
skip++;
- // Copy from p_slot + @skip to the end.
+ // Copy from p_slot + skip to the end.
if (p_slot < mt_slots[ptype] - 2) {
cp.src_start = p_slot + 1;
cp.src_end = p_end;
cp.dst_start += skip;
+ printk("copy %p[%u-%u] up to %lu\n", mte_to_node(cp.src), cp.src_start,cp.src_end, right.max);
_mas_copy(&parent, &cp, right.max);
}
}
+ right.max = mas->max;
// left will be placed in p_slot
mte_link(left.node, new_parent, p_slot, left.max, ptype);
// Check if it's safe to use the slot without a pivot.
// max of the slot == max of the node.
// max of the tree is being set.
- if ((max == mas->max && mas->max != ULONG_MAX) ||
+ if ((mas->last == mas->max && mas->max != ULONG_MAX) ||
(mas->max == ULONG_MAX && mas->last == ULONG_MAX))
return req_slots;
enum maple_type mas_type = mte_node_type(mas->node);
struct maple_node space;
struct maple_node* mn = NULL;
+ unsigned long mn_min = mas->min;
int ret = 0;
MA_STATE(cp, mas->tree, mas->index, mas->last);
if (append) {
printk("Appending.\n");
- mn = mas_mn(mas);
+ mn = mas_mn(mas); // Appending means writing to the source.
} else if (active) {
cp.node = mt_mk_node(ma_mnode_ptr(mas_next_alloc(mas)),
mas_type);
cp.node = NULL;
}
- ret = mas_ma_cp(mas, 0, &cp, NULL, mn, mas_type, entry_cnt, entry,
- append);
+ ret = mas_ma_cp(mas, 0, &cp, NULL, mn, mas_type, mn_min, entry_cnt, entry,
+ append ? entry_cnt : 0);
mn->parent = mas_mn(mas)->parent;
// FIXME: Propagate gaps.
goto update_gap;
}
+
+ if (!ma_is_leaf(this_type)) {
+
+ BUG_ON(1);
+ // Allocation failed, so rebuild the tree.
+ }
+
+ printk("%s: here %d\n", __func__, __LINE__);
old_end = mas_data_end(mas, this_type, &last_piv, &coalesce);
if (slot > slot_cnt) // search returned MAPLE_NODE_SLOTS
slot = old_end + 1;
_mas_get_range(mas, slot, &min, &max);
+ if (mas_get_slot(mas) > slot_cnt)
+ max = mas->max;
+ printk("Landed %p[%u] max %lu mas_max %lu\n", mas_mn(mas), mas_get_slot(mas), max,
+ mas->max);
if (slot <= old_end)
contents = mte_get_rcu_slot(mas->node, slot);
+ printk("%s: here %d\n", __func__, __LINE__);
// Check early failures.
if (!overwrite) {
if (mas->last > max) { // spans range.
+ printk("%s: here %d %lu %lu\n", __func__, __LINE__, mas->last, max);
mas_set_err(mas, -ERANGE);
return 0;
}
if (!mt_is_empty(contents)) {
+ printk("%s: here %d\n", __func__, __LINE__);
mas_set_err(mas, -EBUSY);
return 0;
}
}
+ printk("%s: here %d\n", __func__, __LINE__);
// At this point, the we can perform the add.
// spans node means we will replace the tree.
+ printk("mas: l %lu max %lu\n", mas->last, mas->max);
if (mas->last > mas->max)
return mas_replace_tree(mas, entry);
+ if (!mte_is_leaf(mas->node)) {
+ // An allocation failed previously during a rebalance. There
+ // is no way to know how broken things are, so try to rebuild
+ // the tree.
+ mas_reset(mas);
+ mas_first_node(mas, ULONG_MAX);
+ printk("Using first node %p\n", mas_mn(mas));
+ return mas_replace_tree(mas, entry);
+ }
+
// Fits neatly into a slot.
if (mas->index == min && mas->last == max) {
mte_set_rcu_slot(mas->node, slot, entry);
return old_end - new_end;
}
- if(!mte_is_leaf(mas->node)) {
- printk("trying to add %lu-%lu to %p\n", mas->index, mas->last, mas_mn(mas));
- }
- BUG_ON(!mte_is_leaf(mas->node));
-
if (active) {
printk("%s %p\n", __func__, mas);
printk("Active and needs a node\n");
printk("slot %u old_end %u\n", slot, old_end);
if (slot > old_end && !coalesce)
append = true;
- // FIXME: append (slot > old_end) may cause coalescing which wouldn't
- // be
+
__mas_add(mas, entry, old_end, active, append);
do {
void *entry = NULL;
if (slot)
- mas->min = mas_get_safe_pivot(mas, slot - 1);
+ mas->min = mas_get_safe_pivot(mas, slot - 1) + 1;
mas->max = mas_get_safe_pivot(mas, slot);
entry = mte_get_rcu_slot(mas->node, slot);
if (xa_is_skip(entry)) {
if (mt_is_empty(mas->node))
goto no_entry;
- if (mte_is_leaf(mas->node))
+ if (mte_is_leaf(mas->node)) {
goto done;
+ }
slot = 0;
} while (1);
break;
mn = mte_get_rcu_slot(mas->node, slot);
- if (!mn) {
+ if (mt_is_empty(mn)) {
prev_piv = pivot;
continue;
}
if (mte_is_root(mas->node))
goto no_entry;
+ mas_set_slot(mas, mte_parent_slot(mas->node));
}
no_entry:
void *entry = mte_get_rcu_slot(mas->node, slot - 1);
if (mte_is_leaf(mas->node)) {
mas->index = range_start - 1;
+ mas->index = mte_get_pivot(mas->node, slot - 1);
return entry;
}
mas->max = prev_max;
mas_coalesce_pivots(mas, p_slot, p_type, false);
}
-/** Private
- *
- * mas_rebalance() - Rebalance mas->node by acquiring other pivots from the
- * node to the right or rebalance the left node.
- *
- *
- *
- * Try to allocate 1-2 nodes, depending on if the right node will be
- * completely consumed.
- *
- * returns 0 on failure or the number of slots that were not filled.
- *
- */
-/*
- *
- * FIXME: Test spanning multiple levels of the tree.
- *
- */
-static inline void mas_coalesce(struct ma_state *mas);
-#if 0
-static inline int mas_rebalance(struct ma_state *mas, unsigned char end,
- unsigned char coalesce)
-{
- struct maple_enode *r_enode, *this_enode = mas->node;
- unsigned long ret = 0;
- unsigned char this_p_slot, r_p_slot; // Parent slots (this one and right)
- unsigned char all_slots; // Total slots needed for this and right node.
- unsigned char l_slot_cnt, r_slot_cnt; // left and right slot counts
- unsigned long l_piv, r_piv = 0; // left and right pivots
- unsigned char r_end, r_coalesce; // right node end and values that can be coalesced.
- unsigned char node_cnt = 0; // Number of nodes to allocate
- unsigned char r_take;
- enum maple_type new_type;
- bool p_coalesce = false; // coalesce parent.
- bool l_null_end = false, r_null_start = false;
- struct maple_node new_node;
- struct ma_state *src = mas;
- int i = 0;
-
- MA_STATE(p_state, mas->tree, mas->index, mas->last);
- MA_STATE(r_state, mas->tree, mas->index, mas->last);
- MA_STATE(cp_state, mas->tree, mas->index, mas->last);
- MA_CP(cp, this_enode, NULL, 0, end);
-
-
- printk("%s: %p\n", __func__, mas_mn(mas));
- mt_dump(mas->tree);
- l_slot_cnt = ma_hard_data(end, coalesce);
- if (mte_is_root(this_enode))
- return mas_coalesce_node(mas, end, coalesce, true);
-
- this_p_slot = mte_parent_slot(this_enode);
- mas_dup_state(&p_state, mas); // set the parent node, etc.
- mas_ascend(&p_state); // Actually make it the parent.
-
- // Get the next entry.
- mas_set_slot(&p_state, this_p_slot + 1);
- if (!mas_next_nentry(&p_state, ULONG_MAX, &r_piv)) {
- // this_enode is the right-most node of the parent.
- // Reset parent info.
- mas_dup_state(&p_state, mas);
- mas_ascend(&p_state);
- mas_set_slot(&p_state, this_p_slot);
- if (mas_prev_nentry(&p_state, 0, &r_piv)) {
- // If there is a node to the left, rebalance the left
- mas_descend(&p_state);
- end = mas_data_end(&p_state,
- mte_node_type(p_state.node), &l_piv, &coalesce);
- return mas_rebalance(&p_state, end, coalesce);
- }
- // No left or right, rebalance the parent.
- // But first, remove this entry if it's empty.
- if (end < coalesce) {
- printk("%d empty\n", __LINE__);
- mas_coalesce_empty(&p_state, p_state.node, this_p_slot,
- mte_node_type(p_state.node));
- }
- mas_coalesce(&p_state);
- if (mas_is_err(&p_state)) {
- mas->node = p_state.node;
- return 0;
- }
- return 1;
- }
-
- // If we reached here, then the node to the right exists.
- // set the ma_state information and save a copy for this slot.
- mas_dup_state(&r_state, &p_state);
- mas_descend(&r_state);
- r_enode = r_state.node;
-
- r_end = mas_data_end(&r_state, mte_node_type(r_enode), &l_piv, &r_coalesce);
- r_slot_cnt = ma_hard_data(r_end, r_coalesce);
- printk("r_slot_cnt %u\n", r_slot_cnt);
- if (r_end < r_coalesce && r_state.max != ULONG_MAX) {
- printk("skip entry?\n");
- // This node needs to be a skip entry.
- all_slots = l_slot_cnt;
- new_type = mte_node_type(this_enode);
- l_piv = r_state.max;
- goto right_empty;
- }
- // Add 1 for each slot 0.
- all_slots = r_slot_cnt + 2 + l_slot_cnt;
- r_take = r_slot_cnt;
- node_cnt = 1;
- if (all_slots > mt_slot_count(this_enode) - 1) {
- node_cnt++;
- r_take = mt_slot_count(this_enode) / 2;
- }
-
- if (r_take > r_slot_cnt)
- r_take = r_slot_cnt;
-
- printk("%d l_slot_cnt %d\n", __LINE__, l_slot_cnt);
- // check if left ends in NULL
- if (l_slot_cnt && !mte_get_rcu_slot(this_enode, l_slot_cnt))
- l_null_end = true;
-
- printk("r_end and r_coalesce %u %u\n", r_end, r_coalesce);
- if ((r_end == r_coalesce) && l_null_end) { // no space needed..
- printk("ends in null at %p[%u] => %ld\n", this_enode, l_slot_cnt, r_state.max);
- mte_set_pivot(this_enode, l_slot_cnt, r_state.max);
- do { // FIXME: What about non-end nodes?
- mas_ascend(mas);
- mte_set_pivot(mas->node, this_p_slot, r_state.max);
- this_p_slot = mte_parent_slot(mas->node);
- } while (!mte_is_root(mas->node));
- return 0;
- }
-
-
- if (end < coalesce) { // Left is empty, just copy right.
- l_slot_cnt = -1;
- ret = r_slot_cnt;
- goto left_empty;
- }
-
- mas_node_cnt(mas, node_cnt);
- if (mas_is_err(mas))
- return 0;
- printk("%d: nodes %u\n", __LINE__, node_cnt);
-
- mas_dup_state(&cp_state, mas);
- // for each entry from right to copy,
- cp_state.index = r_state.min;
- cp_state.node = mt_mk_node(mas_next_alloc(mas),
- mte_node_type(this_enode));
-
-
- l_slot_cnt = end;
-
- for (i = 0; i <= r_take; i++) {
- // set cp_state->index, cp_state->last, entry
- void *entry = mte_get_rcu_slot(r_enode, i);
- if (!entry || mt_will_coalesce(entry)) {
- cp_state.index = mte_get_pivot(r_enode, i)+1;
- continue;
- }
- cp_state.last = mte_get_pivot(r_enode, i);
- // copy node on first run, append after.
- //
- // call mas_ma_cp(&cp_state, 0, &l_state, type, l_slot_cnt + 1?,
- // entry, i == 0 ? false : true)
- printk("\tma_cp with %lu-%lu -> %p\n", cp_state.index, cp_state.last,
- entry);
- l_slot_cnt = mas_ma_cp(mas, 0, &cp_state, NULL, NULL,
- mte_node_type(this_enode), l_slot_cnt, entry,
- i == 0 ? false : true);
- cp_state.index = cp_state.last + 1;
- cp_state.max = cp_state.last; // update cp state max.
- cp.src_start = i;
- }
- //
-
- mte_to_node(cp_state.node)->parent = mte_to_node(this_enode)->parent;
- mas->node = cp_state.node;
- printk("mas node is %p\n", mas_mn(mas));
- new_type = mte_node_type(cp_state.node);
- mas_replace(mas);
- mas->max = cp_state.max;
- ret = l_slot_cnt;
- mte_set_pivot(p_state.node, this_p_slot, mas->max);
-
-right_empty:
- printk("Right empty goto\n");
- if (node_cnt < 2) {
- printk("Right is empty, skip\n");
- // Right was entirely consumed.
- void *entry = XA_SKIP_ENTRY;
-
- ret = mt_slots[new_type] - all_slots;
- if (!ret)
- ret = 1;
-
- r_p_slot = mte_parent_slot(r_enode);
- mte_set_rcu_slot(p_state.node, r_p_slot, entry);
- if (r_p_slot < mt_pivot_count(mas->node))
- mte_set_pivot(p_state.node, r_p_slot, l_piv);
- mte_free(r_enode);
- p_coalesce = true;
- goto right_done;
- }
-left_empty:
-
- cp.src = r_state.node;
- cp.src_end = r_end;
- cp.dst = NULL;
- cp.dst_start = l_slot_cnt + 1;
- cp.dst_end = r_end;
- printk("Copy %p to ?[%u-%u]\n", cp.src, cp.dst_start, cp.dst_end);
- mas_copy(&r_state, &cp); // cp.dst is coalesced remainder of r_enode.
- mte_to_node(cp.dst)->parent = mte_to_node(r_enode)->parent;
- r_state.node = cp.dst;
- r_p_slot = mte_parent_slot(r_enode);
- mas_replace(&r_state);
- if (r_p_slot < mt_pivot_count(p_state.node) - 1)
- mte_set_pivot(p_state.node, r_p_slot, r_state.max);
-
-
-right_done:
- // Set the parent slots to l_piv for all skip slots..
- while (r_p_slot-- > this_p_slot)
- mte_set_pivot(p_state.node, r_p_slot, l_piv);
-
- /* If there is a freed node, then mas->node must point to the parent
- * which contained the freed node so it can also be checked for
- * coalescing.
- */
- if (p_coalesce)
- mas_coalesce(&p_state);
- return ret;
-}
-#else
-static inline int mas_rebalance(struct ma_state *mas, unsigned char end,
- unsigned char coalesce)
-{
- struct maple_enode *r_enode, *this_enode = mas->node;
- unsigned long ret = 0;
- unsigned char this_p_slot, r_p_slot; // Parent slots (this one and right)
- unsigned char all_slots; // Total slots needed for this and right node.
- unsigned char l_slot_cnt, r_slot_cnt; // left and right slot counts
- unsigned long l_piv, r_piv = 0; // left and right pivots
- unsigned char r_end, r_coalesce; // right node end and values that can be coalesced.
- unsigned char node_cnt; // Number of nodes to allocate
- enum maple_type new_type;
- bool p_coalesce = false; // coalesce parent.
- bool l_null_end = false, r_null_start = false;
-
- MA_STATE(p_state, mas->tree, mas->index, mas->last);
- MA_STATE(r_state, mas->tree, mas->index, mas->last);
- MA_CP(cp, this_enode, NULL, 0, end);
-
- printk("%s: %p\n", __func__, mas_mn(mas));
- l_slot_cnt = ma_hard_data(end, coalesce);
- if (mte_is_root(this_enode))
- return mas_coalesce_node(mas, end, coalesce, true);
-
- this_p_slot = mte_parent_slot(this_enode);
- mas_dup_state(&p_state, mas); // set the parent node, etc.
- mas_ascend(&p_state); // Actually make it the parent.
-
- // Get the next entry.
- mas_set_slot(&p_state, this_p_slot + 1);
- if (!mas_next_nentry(&p_state, ULONG_MAX, &r_piv)) {
- // this_enode is the right-most node of the parent.
- // Reset parent info.
- mas_dup_state(&p_state, mas);
- mas_ascend(&p_state);
- mas_set_slot(&p_state, this_p_slot);
- if (mas_prev_nentry(&p_state, 0, &r_piv)) {
- // If there is a node to the left, rebalance the left
- mas_descend(&p_state);
- end = mas_data_end(&p_state,
- mte_node_type(p_state.node), &l_piv, &coalesce);
- return mas_rebalance(&p_state, end, coalesce);
- }
- // No left or right, rebalance the parent.
- // But first, remove this entry if it's empty.
- if (end < coalesce) {
- printk("%d empty %d %d\n", __LINE__, end, coalesce);
- // FIXME: deleted?
- // deleted.. AND FREE!
- mas_coalesce_empty(&p_state, p_state.node, this_p_slot,
- mte_node_type(p_state.node));
- }
- mas_coalesce(&p_state);
- if (mas_is_err(&p_state)) {
- mas->node = p_state.node;
- return 0;
- }
- return 1;
- }
-
- printk("left and right\n");
- // If we reached here, then the node to the right exists.
- // set the ma_state information and save a copy for this slot.
- mas_dup_state(&r_state, &p_state);
- mas_descend(&r_state);
- r_enode = r_state.node;
-
- r_end = mas_data_end(&r_state, mte_node_type(r_enode), &l_piv, &r_coalesce);
- r_slot_cnt = ma_hard_data(r_end, r_coalesce);
- printk("r_slot_cnt %u\n", r_slot_cnt);
- if (r_end < r_coalesce && r_state.max != ULONG_MAX) {
- printk("skip entry?\n");
- // This node needs to be a deleted entry.
- all_slots = l_slot_cnt;
- new_type = mte_node_type(this_enode);
- l_piv = r_state.max;
- goto right_empty;
- }
- // Add 1 for each slot 0.
- all_slots = r_slot_cnt + 2 + l_slot_cnt;
-
- node_cnt = 1;
- if (all_slots > mt_slot_count(this_enode) - 1)
- node_cnt++;
-
- printk("%d l_slot_cnt %d %u %u\n", __LINE__, l_slot_cnt, end, coalesce);
- // check if left ends in NULL
- if (l_slot_cnt && !mte_get_rcu_slot(this_enode, l_slot_cnt))
- l_null_end = true;
-
- printk("r_end and r_coalesce %u %u\n", r_end, r_coalesce);
- if ((r_end == r_coalesce) && l_null_end) { // no space needed..
- printk("ends in null at %p[%u] => %ld\n", this_enode, l_slot_cnt, r_state.max);
- mte_set_pivot(this_enode, l_slot_cnt, r_state.max);
- do {
- unsigned old_piv, old_mas_max = mas->max;
- mas_ascend(mas);
- old_piv = mte_get_pivot(mas->node, this_p_slot);
- if (old_piv != old_mas_max)
- break;
- mte_set_pivot(mas->node, this_p_slot, r_state.max);
- this_p_slot = mte_parent_slot(mas->node);
- } while (!mte_is_root(mas->node));
- return 1;
- }
- mas_node_cnt(mas, node_cnt);
- if (mas_is_err(mas)) {
- // Special case. If we are failing to allocate but the tree
- // should be collapsed back, then do the right thing (tm).
- if ((r_end == r_coalesce) &&
- (r_state.max == ULONG_MAX) &&
- (end >= coalesce)) {
- mte_set_pivot(this_enode, l_slot_cnt, r_state.max);
- do {
- unsigned old_piv, old_mas_max = mas->max;
- mas_ascend(mas);
- old_piv = mte_get_pivot(mas->node, this_p_slot);
- if (old_piv != old_mas_max)
- break;
- mte_set_pivot(mas->node, this_p_slot, r_state.max);
- this_p_slot = mte_parent_slot(mas->node);
- } while (!mte_is_root(mas->node));
- r_p_slot = mte_parent_slot(r_enode);
- mas_coalesce_empty(&r_state, p_state.node, r_p_slot,
- mte_node_type(p_state.node));
- mt_dump(mas->tree);
- }
- return 0;
- }
- printk("here %d\n", __LINE__);
-
- // Coalesce this_enode into a new node.
- cp.dst_end = l_slot_cnt;
- printk("%d Copy %p[%u-%u] to %p[%u-%u]\n", __LINE__, mas_mn(mas), cp.src_start, cp.src_end, "??", cp.dst_start, cp.dst_end);
- mas_copy(mas, &cp); // cp.dst now has coalesced this_enode.
-
-
- // check if right start in NULL..
- if (mt_will_coalesce(mte_get_rcu_slot(r_enode, 0)) ||
- !mte_get_rcu_slot(r_enode, 0))
- r_null_start = true;
-
- printk("r first is %p\n", mte_get_rcu_slot(r_enode, 0));
- if (l_null_end && r_null_start)
- {
- all_slots--;
- } else if (!l_null_end && !r_null_start &&
- r_state.min != mas->max + 1) {
- // Need a null if the pivots don't line up here.
- mte_set_pivot(cp.dst, cp.dst_start++, r_state.min);
- all_slots++;
- }
-
- // Copy from the right node.
- cp.src = r_enode;
- cp.src_start = 0;
- cp.dst_end = mt_slot_count(cp.dst) - 1;
- if (all_slots <= mt_slot_count(mas->node) - 1)
- cp.src_end = r_end;
- else
- cp.dst_end = (all_slots + 1)/ 2; // Take 1/2 the entries.
- printk("Copy %p[%u-%u] to %p[%u-%u]\n", mas_mn(&r_state), cp.src_start, cp.src_end, mte_to_node(cp.dst), cp.dst_start, cp.dst_end);
-
- _mas_copy(&r_state, &cp,
- mte_get_pivot(cp.dst, cp.dst_start-1)); // cp.dst is now complete, place it in the tree.
- mte_to_node(cp.dst)->parent = mte_to_node(this_enode)->parent;
- new_type = mte_node_type(cp.dst);
- mas->node = cp.dst;
- mas_replace(mas);
- mas->max = r_state.max;
- if (all_slots > mt_slots[new_type] - 1) {
- mas_data_end(mas, mte_node_type(mas->node), &l_piv, &coalesce);
- mas->max = l_piv;
- }
- l_piv = mas->max;
- ret = mt_slot_count(mas->node) - cp.dst_start + 1;
- mte_set_pivot(p_state.node, this_p_slot, l_piv);
-
-right_empty:
- if (all_slots <= mt_slots[new_type] - 1) {
- // Right was entirely consumed.
- void *entry = XA_SKIP_ENTRY;
-
- ret = mt_slots[new_type] - all_slots;
- if (!ret)
- ret = 1;
-
- r_p_slot = mte_parent_slot(r_enode);
- mte_set_rcu_slot(p_state.node, r_p_slot, entry);
- if (r_p_slot < mt_pivot_count(mas->node))
- mte_set_pivot(p_state.node, r_p_slot, l_piv);
- mte_free(r_enode);
- p_coalesce = true;
- goto right_done;
- }
-
- cp.src_end = r_end;
- cp.dst = NULL;
- cp.dst_start = 0;
- cp.dst_end = r_end;
- mas_copy(&r_state, &cp); // cp.dst is coalesced remainder of r_enode.
- mte_to_node(cp.dst)->parent = mte_to_node(r_enode)->parent;
- r_state.node = cp.dst;
- r_p_slot = mte_parent_slot(r_enode);
- mas_replace(&r_state);
- if (r_p_slot < mt_pivot_count(p_state.node) - 1)
- mte_set_pivot(p_state.node, r_p_slot, r_state.max);
-
-
-right_done:
- // Set the parent slots to l_piv for all skip slots..
- while (r_p_slot-- > this_p_slot)
- mte_set_pivot(p_state.node, r_p_slot, l_piv);
-
- /* If there is a freed node, then mas->node must point to the parent
- * which contained the freed node so it can also be checked for
- * coalescing.
- */
- if (p_coalesce)
- mas_coalesce(&p_state);
- return ret;
-}
-#endif
-
/** Private
*
*/
mas_replace(mas);
}
+/* Append from src to mas */
+static inline int mas_cp_append(struct ma_state *mas, struct ma_state *src,
+ struct ma_state *p_mas, unsigned char dst_cnt,
+ unsigned char count, bool retry)
+{
+ enum maple_type mas_type = mte_node_type(mas->node);
+ int slot = 0;
+ printk("count = %u\n", count);
+ MA_STATE(dst, mas->tree, mas->index, mas->last);
+
+ mas_dup_state(&dst, mas);
+
+ dst.index = src->min;
+ dst.max = src->max;
+ while (slot <= count ) {
+ void *entry = mte_get_rcu_slot(src->node, slot);
+ dst.last = mas_get_safe_pivot(src, slot);
+ printk("range %lu-%lu\n", dst.index, dst.last);
+ if (mt_is_empty(entry) && dst.last != ULONG_MAX)
+ goto next;
+ src->index = dst.index;
+ src->last = dst.last;
+
+ printk("Appending %p[%u] to %p[%u]\n", mas_mn(src), slot,
+ mas_mn(&dst), dst_cnt);
+
+ dst_cnt = mas_ma_cp(src, 0, &dst, NULL, mas_mn(&dst), mas_type,
+ dst.min, slot,
+ entry, dst_cnt);
+next:
+ dst.index = dst.last+1;
+ printk("update dst to %lu\n", dst.index);
+ slot++;
+ }
+ printk("Just left while with %u\n", slot);
+ if (retry) {
+ mte_set_pivot(p_mas->node, mte_parent_slot(mas->node),
+ mas_get_safe_pivot(src, slot));
+ wmb();
+ do {
+ mte_set_rcu_slot(src->node, slot, XA_RETRY_ENTRY); // relocated.
+ } while(slot-- > 0);
+ }
+ return dst_cnt;
+ // FIXME: Check mas->node's gap.
+}
/** Private
- * mas_coalesce() - Check prev/next node for end/start NULLs and adjust the
- * gaps and parent pivot accordingly.
- *
- *
- * Attempt to move things left with a rebalance. Upon failure, check if there
- * is a contiguous gap from the end of this node to the start of the next.
- *
- * Notes:
- * - Entries move Left, appending data at the end of the leaf
- * - Holes move Right.
- * - Either way, parent pivots need to be changed.
- * - empty nodes will be replaced by skip entries on allocation failure.
+ * mas_coalesce() -
*
+ * coalesce completely consumes the right node into this node.
*
*/
-static inline void mas_coalesce(struct ma_state *mas)
-{
- unsigned char end, p_slot, coalesce;
- struct maple_enode *this_enode, *eparent;
- enum maple_type this_type, p_type;
- bool check_parent = false;
- unsigned long piv, this_piv;
- int h_data;
- void *entry;
-
-start:
- mt_dump(mas->tree);
- if (mte_is_root(mas->node))
- return mas_coalesce_root(mas);
- this_enode = mas->node;
- this_type = mte_node_type(this_enode);
- end = mas_data_end(mas, this_type, &this_piv, &coalesce);
- printk("data end of %p = %u\n", mas_mn(mas), end);
- p_type = mas_parent_enum(mas, this_enode);
- p_slot = mte_parent_slot(this_enode);
- eparent = mt_mk_node(mte_parent(this_enode), p_type);
-
-
- /* If there is any space to save, try to reallocate */
- h_data = ma_hard_data(end, coalesce);
- if (h_data < mt_min_slots[this_type] - 1) {
- printk("rebalance?\n");
- if (mas_rebalance(mas, end, coalesce))
- goto done;
-
- if (mas_is_err(mas)) {
- if (end >= coalesce)
- return;
- if (mas->max == ULONG_MAX)
- return;
- // Single entry and allocation failed, free this_enode
- printk("failure path? %u %u\n", end, coalesce);
- // FIXME: Deleted?
- mas->node = this_enode;
- mas_coalesce_empty(mas, eparent, p_slot, p_type);
- check_parent = true;
- goto done;
- }
+static inline bool mas_coalesce(struct ma_state *mas, unsigned char l_end_slot,
+ unsigned char l_coalesce, enum maple_type l_type,
+ struct ma_state *r_mas, unsigned char r_end_slot,
+ struct ma_state *p_mas, unsigned long total_slots,
+ struct ma_cp *cp)
+{
+ struct maple_node *mn;
+ bool ret = false;
+ printk("%s: %p + %p\n", __func__, mas_mn(mas), mas_mn(r_mas));
+
+ MA_STATE(dst, mas->tree, mas->index, mas->last);
+ // it is possible that all of the right node can be appended to the
+ // left
+ if (total_slots + 1 - l_end_slot < mt_slots[l_type] - l_end_slot) {
+ printk("append to %p[%u] from %p\n", mas_mn(mas), l_end_slot,
+ mas_mn(r_mas));
+// mas->max = r_mas->max;
+ mas_cp_append(mas, r_mas, p_mas, l_end_slot, r_end_slot, false);
+ mte_set_rcu_slot(p_mas->node, mte_parent_slot(r_mas->node),
+ XA_SKIP_ENTRY);
+ mte_free(r_mas->node);
+ printk("coalesce complete\n");
+ goto done;
}
- /* Group the gaps together. Acquire any data from the next node, if
- * necessary
- */
-
- entry = mte_get_rcu_slot(this_enode, end);
- if (!mt_is_empty(entry))
- goto check_start;
-
-
- // Check if next node starts with null.
- mas_next_slot(mas, ULONG_MAX);
- if (mas->node == MAS_NONE)
- goto check_start;
-
- if (mt_is_empty(mas->node) ||
- mt_is_empty(mte_get_rcu_slot(mas->node, 0))) {
- unsigned char slot = end;
-
- if (check_parent)
- slot = 0;
+ mas_node_cnt(mas, 1); // Try to allocate.
+ if (mas_is_err(mas)) {
+ printk("%s: Allocation failure\n", __func__);
+ // Allocation failed, we could try to append as much
+ // as possible here?
+ return false;
+ }
- while (--slot > 0) {
- if (!mt_is_empty(mte_get_rcu_slot(this_enode, slot)))
- break;
- }
- mas->node = eparent;
- mas_update_limits(mas, p_slot, p_type);
- mas->node = this_enode;
+ printk("New node\n");
+ mn = mas_next_alloc(mas);
+ mas_dup_state(&dst, mas);
+ mn->parent = mas_mn(mas)->parent;
+ dst.node = mt_mk_node(mn, l_type);
+// cp->dst = mas->node;
+// mas_copy(mas, cp);
+ l_end_slot = mas_cp_append(&dst, mas, p_mas, 0, l_end_slot, false);
+ printk("l_node_end is now %u\n", l_end_slot);
+ if (!l_end_slot && !mte_get_pivot(dst.node, 0)) {
+ mte_set_pivot(dst.node, 0, mas->max);
+ printk("set first pivot to %lu\n", mas->max);
+ l_end_slot++;
+ }
+ mas->node = dst.node;
+ // FIXME: Broken.
+ mas_cp_append(mas, r_mas, p_mas, l_end_slot, r_end_slot, true);
+ ret = true;
+done:
+ if (!mte_is_leaf(mas->node))
+ mte_adopt_children(mas->node);
- if (!slot) {
- // Empty node...
- // deleted.. AND FREE!
- mas_coalesce_empty(mas, eparent, p_slot, p_type);
- check_parent = true;
- piv = mas->min;
- } else {
- piv = mas_get_safe_pivot(mas, slot);
- }
+ mte_set_pivot(p_mas->node, mte_parent_slot(mas->node), r_mas->max);
+ return ret;
+}
- if (p_slot <= mt_pivot_count(eparent))
- mte_set_pivot(eparent, p_slot, piv);
- // Walk up checking for the old pivot and set to piv.
+/** Private
+ * mas_rebalance() -
+ *
+ * rebalance moves data from the node to the right to this node if the
+ * low watermark of data is not met. It also calls coalesce if the right data
+ * can fully be moved to the left.
+ *
+ */
+static inline void mas_rebalance(struct ma_state *mas)
+{
+ struct maple_enode *this_enode;
+ unsigned char l_end_slot, l_coalesce, r_end_slot, r_coalesce;
+ unsigned char p_slot; // parent slot.
+ unsigned char total_slots, copy_count;
+ unsigned long l_end_piv, r_end_piv;
+ enum maple_type l_type, r_type;
+ bool try_anyways = false;
+ bool free;
+
+ MA_STATE(r_mas, mas->tree, mas->index, mas->last); // right state
+ MA_STATE(p_mas, mas->tree, mas->index, mas->last); // parent state
+ MA_CP(cp, mas->node, NULL, 0, 0);
+ mt_dump(mas->tree);
+start:
+ printk("%s: Starting with %p\n", __func__, mas_mn(mas));
+ free = false;
+ this_enode = mas->node;
+ l_type = mte_node_type(mas->node);
+ if (mte_is_root(mas->node))
+ return mas_coalesce_root(mas); // height reduction and such.
- if (!slot)
+ mas_dup_state(&p_mas, mas);
+ mas_ascend(&p_mas);
+ p_slot = mte_parent_slot(mas->node);
+ l_end_slot = mas_data_end(mas, l_type, &l_end_piv, &l_coalesce);
+ printk("hard data is %u\n", ma_hard_data(l_end_slot, l_coalesce));
+ if (!try_anyways &&
+ (ma_hard_data(l_end_slot, l_coalesce) >= mt_min_slots[l_type])) {
+ printk("Done this\n");
+ return; // Everything's perfectly all right now.
+ }
+#if 0
+ else if (ma_hard_data(l_end_slot, l_coalesce == 1)) {
+ printk("Empty node %p\n", mas_mn(mas));
+ mte_set_rcu_slot(p_mas.node, p_slot, XA_DELETED_ENTRY);
+ mte_free(mas->node);
+ goto done;
+ }
+#endif
+ try_anyways = false;
+
+ // Make sure there is a right node.
+ mas_dup_state(&r_mas, &p_mas);
+ mas_set_slot(&r_mas, p_slot + 1);
+ if (!mas_next_nentry(&r_mas, ULONG_MAX, &r_end_piv)) {
+ // Right-most node coalescing.
+ printk("Right most\n");
+ mas_dup_state(&r_mas, &p_mas);
+ printk("reset to node is %p\n", mas_mn(&r_mas));
+ mas_set_slot(&r_mas, p_slot);
+ if (!mas_prev_nentry(&r_mas, 0, &r_end_piv)) {
+ // Single entry in the parent.
+ printk("Single entry?\n");
+ if (l_end_slot < l_coalesce) { // Single entry is empty.
+ free = true;
+ } else if (l_end_slot == l_coalesce && mas->max == ULONG_MAX) {
+ // Possible single entry of null for
+ // ULONG_MAX.
+ printk("ulong max check needed\n");
+ BUG_ON(1);
+ free = true;
+ }
+ if (free) {
+ mte_set_rcu_slot(p_mas.node, p_slot,
+ XA_DELETED_ENTRY);
+ }
goto done;
+ }
+ mas_descend(&r_mas);
+ printk("prev slot is %p\n", mas_mn(&r_mas));
+ mas_dup_state(mas, &r_mas);
+ printk("Forcing next one, coalesce prev.\n");
+ try_anyways = true; // Force a rebalance/coalesce of these nodes.
+ goto start; // restart with new left.
+ }
+ mas_descend(&r_mas);
+ printk("Right node is %p\n", mas_mn(&r_mas));
+
+ // We have a left and a right, check if they can be coalesced.
+ r_type = mte_node_type(r_mas.node); // not for racing.
+ r_end_slot = mas_data_end(&r_mas, r_type, &r_end_piv, &r_coalesce);
+
+ // end_slot values don't count slot 0, so add one.
+ total_slots = l_end_slot + 1 - l_coalesce;
+ total_slots += r_end_slot + 1 - r_coalesce;
+ if (l_end_piv + 1 != r_mas.min)
+ total_slots++; // will need a null entry between the two.
+
+ mas_dup_state(&p_mas, mas);
+ mas_ascend(&p_mas);
+ cp.src_end = l_end_slot;
+ if (total_slots <= mt_slots[l_type]) {
+ // Totally consume the right node.
+ free = mas_coalesce(mas, l_end_slot, l_coalesce, l_type,
+ &r_mas, r_end_slot, &p_mas, total_slots, &cp);
+ if (mas_is_err(mas))
+ return;
- // Indicate value has been moved.
- while (++slot <= end)
- mte_set_rcu_slot(this_enode, slot, XA_RETRY_ENTRY);
+ goto done;
}
-check_start:
- mas->node = this_enode;
- entry = mte_get_rcu_slot(this_enode, 0);
- if (mt_is_empty(entry)) {
- unsigned char prev_end;
- // Check the previous node.
- mas_prev_slot(mas, 0);
- if (mas_is_none(mas))
- goto done;
+ // Rebalance between mas and r_mas nodes.
+ p_slot = mte_parent_slot(mas->node);
- if (!mt_is_empty(mas->node)) {
- prev_end = mas_data_end(mas, mte_node_type(mas->node),
- &piv, &coalesce);
+ mas_node_cnt(mas, 1); // Try to allocate.
+ if (mas_is_err(mas)) {
+ printk("%s: Allocation failure\n", __func__);
+ // Allocation failed, we could try to append as much
+ // as possible here?
+ return;
+ }
+ printk("new node\n");
- if (!mt_is_empty(mte_get_rcu_slot(mas->node, prev_end)))
- goto done;
- } else {
- piv = mte_get_pivot(this_enode, 0);
- }
+ free = true; // free this_enode after the operation.
+ mas->node = mt_mk_node(mas_next_alloc(mas), l_type);
+ mas_mn(mas)->parent = mte_to_node(this_enode)->parent;
+ cp.dst = mas->node;
+ mas_copy(mas, &cp);
- if (p_slot)
- mte_set_pivot(eparent, p_slot - 1, piv);
+ // Put 1/2 of the contents into the left if not all of them can fit.
+ copy_count = (total_slots / 2) - (l_end_slot + 1 - l_coalesce);
- // Walk up and set all the pivots to piv
- }
-done:
- mas->node = this_enode;
- if (check_parent) {
- check_parent = false;
- mas_ascend(mas);
- printk(" -> max is %lu\n", mas->max);
- mte_free(this_enode);
- printk("going to start with %p\n", mas_mn(mas));
- goto start;
+ mas_cp_append(mas, &r_mas, &p_mas, l_end_slot, copy_count, true);
+ if (total_slots <= mt_slots[l_type]) {
+ printk("Use skip entry\n");
+ mte_set_rcu_slot(p_mas.node, mte_parent_slot(r_mas.node),
+ XA_SKIP_ENTRY);
+ mte_free(r_mas.node);
}
- if (mt_is_alloc(mas->tree))
- mas_update_gap(mas);
-
- return;
+done:
+ if (!mte_is_leaf(mas->node))
+ mte_adopt_children(mas->node);
+ if (free)
+ mas_replace(mas);
+ mt_dump(mas->tree);
+ // Check parent for rebalancing.
+ mas_dup_state(mas, &p_mas);
+ goto start;
}
-
static inline bool _mas_rev_awalk(struct ma_state *mas, unsigned long size)
{
enum maple_type type;
default:
for (i = mas_get_slot(mas); i < mt_slots[type]; i++) {
pivot = _mas_get_safe_pivot(mas, i, type);
+
if (i != 0 && pivot == 0) {
i = MAPLE_NODE_SLOTS;
goto done;
}
+ if (min > pivot) // coalescing value was in the last slot.
+ min = pivot;
+
if (mas->index <= pivot) {
max = pivot;
break;
}
-
min = pivot + 1;
}
}
EXPORT_SYMBOL(mt_find);
+static inline int mas_build_replacement(struct ma_state *mas, void *new_entry,
+ long node_cnt)
+
+{
+ struct maple_enode *last = NULL;
+ unsigned long new_index, new_last;
+ unsigned long r_index, r_last;
+ void *entry;
+
+ DEFINE_MTREE(new_tree);
+ MA_STATE(new_mas, &new_tree, 0, 0);
+
+
+ if (!node_cnt)
+ return 0;
+ node_cnt += 3; // Room for an extra split.
+
+ //printk("Using node count %d for %lu-%lu\n", node_cnt, mas->index, mas->last);
+ mas_node_cnt(mas, node_cnt);
+ if (mas_is_err(mas))
+ return 0;
+
+ new_index = mas->index;
+ new_last = mas->last;
+
+ /* Move allocations from mas to new_mas.
+ * NOTE: This is necessary as mas will pass back errors and will retry
+ * the allocation, so it has to be done in mas and has to be moved for
+ * below.
+ */
+ new_mas.alloc = mas->alloc;
+ mas->alloc = NULL;
+
+ //printk("\tCopy left\n");
+ // Copy left side
+ mas_reset(mas);
+ mas->index = 0;
+ mas->last = 0;
+ mas_for_each(mas, entry, new_index - 1) {
+ //printk("\tLeft for each\n");
+ new_mas.index = mas->index;
+ new_mas.last = mas_get_safe_pivot(mas, mas_get_slot(mas));
+ if (entry == XA_DELETED_ENTRY)
+ BUG_ON(1);
+ ma_inactive_insert(&new_mas, entry);
+ if (mas_is_err(&new_mas)) {
+ goto error;
+ }
+ //printk("node count is %u\n", mas_get_alloc_cnt(&new_mas));
+ //mt_dump(new_mas.tree);
+ }
+
+ //printk("Done\n");
+ //printk("new_mas node %p\n", mas_mn(&new_mas));
+ // Insert the new value.
+ new_mas.index = new_index;
+ new_mas.last = new_last;
+ //printk("\tinsert new entry.. %p %lu %lu\n", mas_mn(&new_mas), new_mas.index, new_mas.last);
+ ma_inactive_insert(&new_mas, new_entry);
+ if (mas_is_err(&new_mas))
+ goto error;
+
+
+ /*
+ * We need to run through a few things:
+ * - new_mas.last goes beyond anything right now (no entries)
+ * - new_mas.last cuts a range
+ * - new_mas.last ends in a null
+ * - new_mas.last has a sequentially next value (48)
+ */
+
+ mas_reset(mas);
+ mas->index = new_last+1;
+ mas->last = new_last+1;
+ _mas_range_walk(mas, &r_index, &r_last);
+
+ if (mas_get_slot(mas) == MAPLE_NODE_SLOTS)
+ goto skip_right;
+
+
+ if (mte_is_leaf(mas->node)) {
+ entry = mte_get_rcu_slot(mas->node, mas_get_slot(mas));
+ if (!mt_is_empty(entry))
+ {
+ new_mas.index = r_index;
+ new_mas.last = r_last;
+ printk("!!Inserting %lu-%lu -> %p\n", r_index, r_last, entry);
+ ma_inactive_insert(&new_mas, entry);
+ if (mas_is_err(&new_mas))
+ goto error;
+ }
+ }
+
+ printk("\tCopy right from %p\n", mas->node);
+ //mt_dump(mas->tree);
+ mas_for_each(mas, entry, ULONG_MAX) {
+ if (mas->index < new_index)
+ continue;
+
+ new_mas.index = mas->index;
+ new_mas.last = mas_get_safe_pivot(mas, mas_get_slot(mas));
+ // printk("\tInsert %lu-%lu => %p\n", new_mas.index, new_mas.last, entry);
+ ma_inactive_insert(&new_mas, entry);
+ if (mas_is_err(&new_mas))
+ goto error;
+ //mt_dump(new_mas.tree);
+ }
+
+skip_right:
+
+ mt_dump(mas->tree);
+ printk("Done rebuild due to %lu-%lu\n", new_index, new_last);
+ mt_dump(new_mas.tree);
+ last = mas->tree->ma_root;
+ mas->node = new_tree.ma_root;
+ _mas_replace(mas, false, false);
+ mas->node = MAS_START;
+ mas->alloc = new_mas.alloc;
+ mte_destroy_walk(last);
+
+ return node_cnt;
+
+error:
+ printk("Error on adding %lu\n",new_mas.index);
+ if (new_mas.tree)
+ mte_destroy_walk(new_mas.tree->ma_root);
+ mt_dump(mas->tree);
+ BUG_ON(1);
+ return 0;
+}
+
/* Private
* mas_replace_tree() - Build a new tree and replace the entire structure.
*
*/
static inline int mas_replace_tree(struct ma_state *mas, void *new_entry)
{
- void *entry;
unsigned long piv;
unsigned char coalesce;
unsigned int slot_cnt = 0;
enum maple_type p_type = mas_parent_enum(mas, mas->node);
// Create a new tree.
- DEFINE_MTREE(new_tree);
- MA_STATE(new_mas, &new_tree, 0, 0);
MA_STATE(r_mas, mas->tree, mas->last + 1, mas->last + 1);
- MA_STATE(l_mas, mas->tree, 0, 0);
-
+ printk("%s\n", __func__);
+ mt_dump(mas->tree);
// Count the slots that will be used in the node we landed.
slot_cnt = 3 + mas_get_slot(mas); // 3 is the max a new entry can create.
// Count the nodes that are currently used to the left.
mas_set_slot(mas, mte_parent_slot(mas->node));
+ printk("Starting left at %p[%u]\n", mas_mn(mas), mas_get_slot(mas));
while (!mas_is_none(mas)) {
last = mas->node;
mas_prev_node(mas, 0);
+ printk("prev node %p[%u]\n", mas_mn(mas), mas_get_slot(mas));
leaves++;
}
// Set mas->node to a valid node.
// Walk down to the right side of the tree.
_mas_walk(&r_mas);
+ printk("Starting right at %p[%u]\n", mas_mn(&r_mas), mas_get_slot(&r_mas));
// Add the slots to the right of where the search landed.
if (mas_get_slot(&r_mas) == MAPLE_NODE_SLOTS) {
r_mas.node = MAS_NONE;
+ slot_cnt++; //entry for oo
goto skip_r_count;
}
slot_cnt -= mas_get_slot(&r_mas);
mas_next_node(&r_mas, ULONG_MAX);
leaves++;
}
- r_mas.node = last;
skip_r_count:
// Calculate all the nodes needed for a new tree.
leaves++;
printk("%d leaves\n", leaves);
- node_cnt = 2; // Root node.
+ node_cnt = 1; // Root node. and room to split.
while (leaves) { // add the number of nodes at each level.
node_cnt += leaves;
leaves /= mt_slots[p_type];
}
-
- printk("Using node count %d for %lu-%lu\n", node_cnt, mas->index, mas->last);
- mas_node_cnt(mas, node_cnt);
- if (mas_is_err(mas))
- return 0;
-
- /* Move allocations from mas to new_mas.
- * NOTE: This is necessary as mas will pass back errors and will retry
- * the allocation, so it has to be done in mas and has to be moved for
- * below.
- */
- new_mas.alloc = mas->alloc;
- mas->alloc = NULL;
-
- // Copy left side
- mas_reset(&l_mas);
- mas_for_each(&l_mas, entry, mas->index - 1) {
- new_mas.index = l_mas.index;
- new_mas.last = l_mas.index;
- ma_inactive_insert(&new_mas, entry);
- if (mas_is_err(&new_mas)) {
- mt_dump(new_mas.tree);
- BUG_ON(1);
- }
- }
-
- // Insert the new value.
- new_mas.index = mas->index;
- new_mas.last = mas->last;
- ma_inactive_insert(&new_mas, new_entry);
- if (mas_is_err(&new_mas))
- BUG_ON(1);
-
-
- /*
- * We need to run through a few things:
- * - new_mas.last goes beyond anything right now (no entries)
- * - new_mas.last cuts a range
- * - new_mas.last ends in a null
- * - new_mas.last has a sequentially next value (48)
- */
- if (mas_is_none(&r_mas)) // No entry beyond new_mas.last
- goto skip_right;
-
- printk("node %p\n", new_mas.node);
- mas_for_each(&l_mas, entry, ULONG_MAX) {
- if (l_mas.index < r_mas.index)
- continue;
-
- new_mas.index = l_mas.index;
- new_mas.last = mas_get_safe_pivot(&l_mas, mas_get_slot(&l_mas));
- ma_inactive_insert(&new_mas, entry);
- if (mas_is_err(&new_mas))
- BUG_ON(1);
- }
-
-skip_right:
-
- last = mas->tree->ma_root;
- mas->node = new_tree.ma_root;
- _mas_replace(mas, false, false);
- mas->node = MAS_START;
- mas->alloc = new_mas.alloc;
- mte_destroy_walk(last);
-
- return node_cnt;
+ return mas_build_replacement(mas, new_entry, node_cnt);
}
+
static inline bool mas_rewind_node(struct ma_state *mas);
static inline void mas_rev_awalk(struct ma_state *mas, unsigned long size)
{
type = mte_node_type(mas->node);
entry = mte_get_rcu_slot(mas->node, slot);
+ printk("%s: %lu return %p[%u] -> %p now to zero.\n", __func__,
+ mas->index, mas_mn(mas), slot, entry);
mte_update_rcu_slot(mas->node, slot, XA_DELETED_ENTRY);
// dense nodes only need to set a single value.
- if (!ma_is_dense(type))
- mas_coalesce_pivots(mas, slot, type, false);
+// if (!ma_is_dense(type))
+// mas_coalesce_pivots(mas, slot, type, false);
- mas_coalesce(mas);
+ mas_rebalance(mas);
+ printk("%s: %lu return %p[%u] -> %p\n", __func__,
+ mas->index, mas_mn(mas), slot, entry);
return entry;
}
MA_STATE(mas, mt, index, index);
+ printk("%s: erase %lu\n", __func__, index);
mtree_lock(mt);
entry = mas_erase(&mas);
mtree_unlock(mt);
gap = mte_get_gap(mte, i);
if (mt_is_empty(mte_get_rcu_slot(mas->node, i)))
BUG_ON(gap != p_end - p_start + 1);
- else
+ else {
+ printk("gap %lu p_end %lu p_start %lu\n", gap, p_end, p_start);
+ printk("Gap %p[%u] != %lu\n", mas_mn(mas), i, p_end - p_start+1);
BUG_ON(gap >= p_end - p_start + 1);
+ }
}
if (gap > max_gap)
unsigned long piv = mas_get_safe_pivot(mas, i);
if (!piv)
break;
+ printk("Checking %p[%u] %lu %lu\n", mas_mn(mas), i, mas->min, mas->max);
+ if (piv > mas->max) {
+ printk("piv fails %lu\n", piv);
+ mt_dump(mas->tree);
+ }
BUG_ON(piv < mas->min);
BUG_ON(piv > mas->max);
}
projects / users / jedix / linux-maple.git / commitdiff