#define NSEC_PER_SEC 1000000000L
-static long long benchmark_iter(struct radix_tree_root *root, bool tagged)
+static long long benchmark_iter(struct xarray *xa, bool marked)
{
volatile unsigned long sink = 0;
- struct radix_tree_iter iter;
struct timespec start, finish;
long long nsec;
int l, loops = 1;
- void **slot;
#ifdef BENCHMARK
again:
#endif
clock_gettime(CLOCK_MONOTONIC, &start);
for (l = 0; l < loops; l++) {
- if (tagged) {
- radix_tree_for_each_tagged(slot, root, &iter, 0, 0)
- sink ^= (unsigned long)slot;
+ XA_STATE(xas, xa, 0);
+ void *entry;
+
+ xas_set(&xas, 0);
+ if (marked) {
+ xas_for_each_marked(&xas, entry, ULONG_MAX, XA_MARK_0)
+ sink ^= (unsigned long)entry;
} else {
- radix_tree_for_each_slot(slot, root, &iter, 0)
- sink ^= (unsigned long)slot;
+ xas_for_each(&xas, entry, ULONG_MAX)
+ sink ^= (unsigned long)entry;
}
}
clock_gettime(CLOCK_MONOTONIC, &finish);
return nsec;
}
-static void benchmark_insert(struct radix_tree_root *root,
+static void benchmark_insert(struct xarray *xa,
unsigned long size, unsigned long step)
{
struct timespec start, finish;
clock_gettime(CLOCK_MONOTONIC, &start);
for (index = 0 ; index < size ; index += step)
- item_insert(root, index);
+ item_insert(xa, index);
clock_gettime(CLOCK_MONOTONIC, &finish);
size, step, nsec);
}
-static void benchmark_tagging(struct radix_tree_root *root,
+static void benchmark_marking(struct xarray *xa,
unsigned long size, unsigned long step)
{
struct timespec start, finish;
clock_gettime(CLOCK_MONOTONIC, &start);
- for (index = 0 ; index < size ; index += step)
- radix_tree_tag_set(root, index, 0);
+ for (index = 0; index < size; index += step)
+ xa_set_mark(xa, index, XA_MARK_0);
clock_gettime(CLOCK_MONOTONIC, &finish);
nsec = (finish.tv_sec - start.tv_sec) * NSEC_PER_SEC +
(finish.tv_nsec - start.tv_nsec);
- printv(2, "Size: %8ld, step: %8ld, tagging: %17lld ns\n",
+ printv(2, "Size: %8ld, step: %8ld, marking: %17lld ns\n",
size, step, nsec);
}
-static void benchmark_delete(struct radix_tree_root *root,
+static void benchmark_delete(struct xarray *xa,
unsigned long size, unsigned long step)
{
struct timespec start, finish;
clock_gettime(CLOCK_MONOTONIC, &start);
- for (index = 0 ; index < size ; index += step)
- item_delete(root, index);
+ for (index = 0; index < size; index += step)
+ item_delete(xa, index);
clock_gettime(CLOCK_MONOTONIC, &finish);
static void benchmark_size(unsigned long size, unsigned long step)
{
- RADIX_TREE(tree, GFP_KERNEL);
- long long normal, tagged;
+ DEFINE_XARRAY(array);
+ long long normal, marked;
- benchmark_insert(&tree, size, step);
- benchmark_tagging(&tree, size, step);
+ benchmark_insert(&array, size, step);
+ benchmark_marking(&array, size, step);
- tagged = benchmark_iter(&tree, true);
- normal = benchmark_iter(&tree, false);
+ marked = benchmark_iter(&array, true);
+ normal = benchmark_iter(&array, false);
- printv(2, "Size: %8ld, step: %8ld, tagged iteration: %8lld ns\n",
- size, step, tagged);
+ printv(2, "Size: %8ld, step: %8ld, marked iteration: %8lld ns\n",
+ size, step, marked);
printv(2, "Size: %8ld, step: %8ld, normal iteration: %8lld ns\n",
size, step, normal);
- benchmark_delete(&tree, size, step);
+ benchmark_delete(&array, size, step);
- item_kill_tree(&tree);
+ assert(xa_empty(&array));
rcu_barrier();
}
int c, s;
printv(1, "starting benchmarks\n");
- printv(1, "RADIX_TREE_MAP_SHIFT = %d\n", RADIX_TREE_MAP_SHIFT);
+ printv(1, "XA_CHUNK_SHIFT = %d\n", XA_CHUNK_SHIFT);
for (c = 0; size[c]; c++)
for (s = 0; step[s]; s++)
assert(idr_alloc(&idr, DUMMY_PTR, 0, 0, GFP_KERNEL) == 0);
idr_remove(&idr, 1);
- for (i = 1; i < RADIX_TREE_MAP_SIZE; i++)
+ for (i = 1; i < XA_CHUNK_SIZE; i++)
assert(idr_alloc(&idr, DUMMY_PTR, 0, 0, GFP_KERNEL) == i);
idr_remove(&idr, 1 << 30);
idr_destroy(&idr);
rcu_register_thread();
while (!test_complete) {
- tag_tagged_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG);
+ mark_marked_items(&array, 0, MAX_IDX, 10, TAG, NEW_TAG);
}
rcu_unregister_thread();
return NULL;
void *kmem_cache_alloc(struct kmem_cache *cachep, int flags)
{
- struct radix_tree_node *node;
+ struct xa_node *node;
if (!(flags & __GFP_DIRECT_RECLAIM))
return NULL;
memset(objp, POISON_FREE, cachep->size);
free(objp);
} else {
- struct radix_tree_node *node = objp;
+ struct xa_node *node = objp;
cachep->nr_objs++;
node->parent = cachep->objs;
cachep->objs = node;
void (*func)(struct rcu_head *head))
{
if (kmalloc_verbose)
- printf("Delaying free of %p to slab\n", (char *)head -
- offsetof(struct radix_tree_node, rcu_head));
+ printf("Delaying free of %p to slab\n",
+ container_of(head, struct xa_node, rcu_head));
call_rcu(head, func);
}
void __gang_check(unsigned long middle, long down, long up, int chunk, int hop)
{
long idx;
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
middle = 1 << 30;
for (idx = -down; idx < up; idx++)
- item_insert(&tree, middle + idx);
+ item_insert(&xa, middle + idx);
- item_check_absent(&tree, middle - down - 1);
+ item_check_absent(&xa, middle - down - 1);
for (idx = -down; idx < up; idx++)
- item_check_present(&tree, middle + idx);
- item_check_absent(&tree, middle + up);
+ item_check_present(&xa, middle + idx);
+ item_check_absent(&xa, middle + up);
if (chunk > 0) {
- item_gang_check_present(&tree, middle - down, up + down,
+ item_gang_check_present(&xa, middle - down, up + down,
chunk, hop);
- item_full_scan(&tree, middle - down, down + up, chunk);
+ item_full_scan(&xa, middle - down, down + up, chunk);
}
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
}
void gang_check(void)
void add_and_check(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
- item_insert(&tree, 44);
- item_check_present(&tree, 44);
- item_check_absent(&tree, 43);
- item_kill_tree(&tree);
+ item_insert(&xa, 44);
+ item_check_present(&xa, 44);
+ item_check_absent(&xa, 43);
+ item_kill_tree(&xa);
}
void dynamic_height_check(void)
{
int i;
- RADIX_TREE(tree, GFP_KERNEL);
- tree_verify_min_height(&tree, 0);
+ DEFINE_XARRAY(xa);
+ tree_verify_min_height(&xa, 0);
- item_insert(&tree, 42);
- tree_verify_min_height(&tree, 42);
+ item_insert(&xa, 42);
+ tree_verify_min_height(&xa, 42);
- item_insert(&tree, 1000000);
- tree_verify_min_height(&tree, 1000000);
+ item_insert(&xa, 1000000);
+ tree_verify_min_height(&xa, 1000000);
- assert(item_delete(&tree, 1000000));
- tree_verify_min_height(&tree, 42);
+ assert(item_delete(&xa, 1000000));
+ tree_verify_min_height(&xa, 42);
- assert(item_delete(&tree, 42));
- tree_verify_min_height(&tree, 0);
+ assert(item_delete(&xa, 42));
+ tree_verify_min_height(&xa, 0);
for (i = 0; i < 1000; i++) {
- item_insert(&tree, i);
- tree_verify_min_height(&tree, i);
+ item_insert(&xa, i);
+ tree_verify_min_height(&xa, i);
}
i--;
for (;;) {
- assert(item_delete(&tree, i));
+ assert(item_delete(&xa, i));
if (i == 0) {
- tree_verify_min_height(&tree, 0);
+ tree_verify_min_height(&xa, 0);
break;
}
i--;
- tree_verify_min_height(&tree, i);
+ tree_verify_min_height(&xa, i);
}
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
}
-void check_copied_tags(struct radix_tree_root *tree, unsigned long start, unsigned long end, unsigned long *idx, int count, int fromtag, int totag)
+void check_copied_tags(struct xarray *xa, unsigned long start, unsigned long end, unsigned long *idx, int count, int fromtag, int totag)
{
int i;
/* if (i % 1000 == 0)
putchar('.'); */
if (idx[i] < start || idx[i] > end) {
- if (item_tag_get(tree, idx[i], totag)) {
+ if (xa_get_mark(xa, idx[i], totag)) {
printv(2, "%lu-%lu: %lu, tags %d-%d\n", start,
- end, idx[i], item_tag_get(tree, idx[i],
+ end, idx[i], xa_get_mark(xa, idx[i],
fromtag),
- item_tag_get(tree, idx[i], totag));
+ xa_get_mark(xa, idx[i], totag));
}
- assert(!item_tag_get(tree, idx[i], totag));
+ assert(!xa_get_mark(xa, idx[i], totag));
continue;
}
- if (item_tag_get(tree, idx[i], fromtag) ^
- item_tag_get(tree, idx[i], totag)) {
+ if (xa_get_mark(xa, idx[i], fromtag) ^
+ xa_get_mark(xa, idx[i], totag)) {
printv(2, "%lu-%lu: %lu, tags %d-%d\n", start, end,
- idx[i], item_tag_get(tree, idx[i], fromtag),
- item_tag_get(tree, idx[i], totag));
+ idx[i], xa_get_mark(xa, idx[i], fromtag),
+ xa_get_mark(xa, idx[i], totag));
}
- assert(!(item_tag_get(tree, idx[i], fromtag) ^
- item_tag_get(tree, idx[i], totag)));
+ assert(!(xa_get_mark(xa, idx[i], fromtag) ^
+ xa_get_mark(xa, idx[i], totag)));
}
}
void copy_tag_check(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
unsigned long idx[ITEMS];
unsigned long start, end, count = 0, tagged, cur, tmp;
int i;
-// printf("generating radix tree indices...\n");
+// printf("generating xarray indices...\n");
start = rand();
end = rand();
if (start > end && (rand() % 10)) {
* with high probability to check for off by one errors */
cur = rand();
if (cur & 1) {
- item_insert(&tree, start);
+ item_insert(&xa, start);
if (cur & 2) {
if (start <= end)
count++;
- item_tag_set(&tree, start, 0);
+ xa_set_mark(&xa, start, XA_MARK_0);
}
}
if (cur & 4) {
- item_insert(&tree, start-1);
+ item_insert(&xa, start-1);
if (cur & 8)
- item_tag_set(&tree, start-1, 0);
+ xa_set_mark(&xa, start-1, XA_MARK_0);
}
if (cur & 16) {
- item_insert(&tree, end);
+ item_insert(&xa, end);
if (cur & 32) {
if (start <= end)
count++;
- item_tag_set(&tree, end, 0);
+ xa_set_mark(&xa, end, XA_MARK_0);
}
}
if (cur & 64) {
- item_insert(&tree, end+1);
+ item_insert(&xa, end+1);
if (cur & 128)
- item_tag_set(&tree, end+1, 0);
+ xa_set_mark(&xa, end+1, XA_MARK_0);
}
for (i = 0; i < ITEMS; i++) {
do {
idx[i] = rand();
- } while (item_lookup(&tree, idx[i]));
+ } while (item_lookup(&xa, idx[i]));
- item_insert(&tree, idx[i]);
+ item_insert(&xa, idx[i]);
if (rand() & 1) {
- item_tag_set(&tree, idx[i], 0);
+ xa_set_mark(&xa, idx[i], XA_MARK_0);
if (idx[i] >= start && idx[i] <= end)
count++;
}
}
// printf("\ncopying tags...\n");
- tagged = tag_tagged_items(&tree, start, end, ITEMS, XA_MARK_0, XA_MARK_1);
+ tagged = mark_marked_items(&xa, start, end, ITEMS, XA_MARK_0, XA_MARK_1);
// printf("checking copied tags\n");
assert(tagged == count);
- check_copied_tags(&tree, start, end, idx, ITEMS, 0, 1);
+ check_copied_tags(&xa, start, end, idx, ITEMS, 0, 1);
/* Copy tags in several rounds */
// printf("\ncopying tags...\n");
tmp = rand() % (count / 10 + 2);
- tagged = tag_tagged_items(&tree, start, end, tmp, XA_MARK_0, XA_MARK_2);
+ tagged = mark_marked_items(&xa, start, end, tmp, XA_MARK_0, XA_MARK_2);
assert(tagged == count);
// printf("%lu %lu %lu\n", tagged, tmp, count);
// printf("checking copied tags\n");
- check_copied_tags(&tree, start, end, idx, ITEMS, 0, 2);
- verify_tag_consistency(&tree, 0);
- verify_tag_consistency(&tree, 1);
- verify_tag_consistency(&tree, 2);
+ check_copied_tags(&xa, start, end, idx, ITEMS, 0, 2);
+ verify_mark_consistency(&xa, XA_MARK_0);
+ verify_mark_consistency(&xa, XA_MARK_1);
+ verify_mark_consistency(&xa, XA_MARK_2);
// printf("\n");
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
}
static void single_thread_tests(bool long_run)
assert((xas.xa_index | mask) == (index[i] | mask));
assert(xas.xa_node->shift == shift);
- assert(!radix_tree_is_internal_node(item));
+ assert(!xa_is_node(item));
assert((item->index | mask) == (index[i] | mask));
assert(item->order == order[i]);
i++;
}
}
- assert(tag_tagged_items(xa, 0, ULONG_MAX, TAG_ENTRIES, XA_MARK_1,
+ assert(mark_marked_items(xa, 0, ULONG_MAX, TAG_ENTRIES, XA_MARK_1,
XA_MARK_2) == TAG_ENTRIES);
for (j = 0; j < 256; j++) {
}
}
- assert(tag_tagged_items(xa, 1, ULONG_MAX, MT_NUM_ENTRIES * 2, XA_MARK_1,
+ assert(mark_marked_items(xa, 1, ULONG_MAX, MT_NUM_ENTRIES * 2, XA_MARK_1,
XA_MARK_0) == TAG_ENTRIES);
i = 0;
xas_set(&xas, 0);
static void *creator_func(void *ptr)
{
/* 'order' is set up to ensure we have sibling entries */
- unsigned int order = RADIX_TREE_MAP_SHIFT - 1;
- struct radix_tree_root *tree = ptr;
+ unsigned int order = XA_CHUNK_SHIFT - 1;
+ struct xarray *xa = ptr;
int i;
for (i = 0; i < 10000; i++) {
- item_insert_order(tree, 0, order);
- item_delete_rcu(tree, 0);
+ item_insert_order(xa, 0, order);
+ item_delete_rcu(xa, 0);
}
stop_iteration = true;
#include "regression.h"
-static RADIX_TREE(mt_tree, GFP_KERNEL);
+static DEFINE_XARRAY(mt_tree);
struct page {
pthread_mutex_t lock;
struct page *p;
p = page_alloc(0);
- xa_lock(&mt_tree);
- radix_tree_insert(&mt_tree, 0, p);
- xa_unlock(&mt_tree);
+ xa_store(&mt_tree, 0, p, GFP_KERNEL);
p = page_alloc(1);
- xa_lock(&mt_tree);
- radix_tree_insert(&mt_tree, 1, p);
- xa_unlock(&mt_tree);
+ xa_store(&mt_tree, 1, p, GFP_KERNEL);
xa_lock(&mt_tree);
- p = radix_tree_delete(&mt_tree, 1);
+ p = __xa_erase(&mt_tree, 1);
pthread_mutex_lock(&p->lock);
p->count--;
pthread_mutex_unlock(&p->lock);
page_free(p);
xa_lock(&mt_tree);
- p = radix_tree_delete(&mt_tree, 0);
+ p = __xa_erase(&mt_tree, 0);
pthread_mutex_lock(&p->lock);
p->count--;
pthread_mutex_unlock(&p->lock);
*
* 0. The xarray contains XA_CHUNK_SIZE items. And a certain entry is marked
* with PAGECACHE_TAG_DIRTY.
- * 1. tag_tagged_items(, start, end, , PAGECACHE_TAG_DIRTY,
+ * 1. mark_marked_items(, start, end, , PAGECACHE_TAG_DIRTY,
* PAGECACHE_TAG_TOWRITE) is called to set PAGECACHE_TAG_TOWRITE on
* entries which have PAGECACHE_TAG_DIRTY. However, there is no entry with
* PAGECACHE_TAG_DIRTY within the range from start to end. As a result,
* 5. xa_extract(, PAGECACHE_TAG_TOWRITE) finds slot 0 has the tag set
* and tries to get some items, but it spins forever.
*
- * The fix is to change tag_tagged_items() to not mark the root
+ * The fix is to change mark_marked_items() to not mark the root
* if it doesn't set any marks within the specified range.
*
* Running:
/* 1. */
start = 0;
end = max_slots - 2;
- tag_tagged_items(&mt_tree, start, end, 1,
+ mark_marked_items(&mt_tree, start, end, 1,
PAGECACHE_TAG_DIRTY, PAGECACHE_TAG_TOWRITE);
/* 2. */
static void
-__simple_checks(struct radix_tree_root *tree, unsigned long index, int tag)
+__simple_checks(struct xarray *xa, unsigned long index, xa_mark_t mark)
{
unsigned long first = 0;
int ret;
- item_check_absent(tree, index);
- assert(item_tag_get(tree, index, tag) == 0);
+ item_check_absent(xa, index);
+ assert(xa_get_mark(xa, index, mark) == 0);
- item_insert(tree, index);
- assert(item_tag_get(tree, index, tag) == 0);
- item_tag_set(tree, index, tag);
- ret = item_tag_get(tree, index, tag);
+ item_insert(xa, index);
+ assert(xa_get_mark(xa, index, mark) == 0);
+ xa_set_mark(xa, index, mark);
+ ret = xa_get_mark(xa, index, mark);
assert(ret != 0);
- ret = tag_tagged_items(tree, first, ~0UL, 10, tag, !tag);
+ ret = mark_marked_items(xa, first, ~0UL, 10, mark, !mark);
assert(ret == 1);
- ret = item_tag_get(tree, index, !tag);
+ ret = xa_get_mark(xa, index, !mark);
assert(ret != 0);
- ret = item_delete(tree, index);
+ ret = item_delete(xa, index);
assert(ret != 0);
- item_insert(tree, index);
- ret = item_tag_get(tree, index, tag);
+ item_insert(xa, index);
+ ret = xa_get_mark(xa, index, mark);
assert(ret == 0);
- ret = item_delete(tree, index);
+ ret = item_delete(xa, index);
assert(ret != 0);
- ret = item_delete(tree, index);
+ ret = item_delete(xa, index);
assert(ret == 0);
}
void simple_checks(void)
{
unsigned long index;
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
for (index = 0; index < 10000; index++) {
- __simple_checks(&tree, index, 0);
- __simple_checks(&tree, index, 1);
+ __simple_checks(&xa, index, 0);
+ __simple_checks(&xa, index, 1);
}
- verify_tag_consistency(&tree, 0);
- verify_tag_consistency(&tree, 1);
+ verify_mark_consistency(&xa, XA_MARK_0);
+ verify_mark_consistency(&xa, XA_MARK_1);
printv(2, "before item_kill_tree: %d allocated\n", nr_allocated);
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
rcu_barrier();
printv(2, "after item_kill_tree: %d allocated\n", nr_allocated);
}
/*
- * Check that tags propagate correctly when extending a tree.
+ * Check that marks propagate correctly when extending a tree.
*/
static void extend_checks(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
-
- item_insert(&tree, 43);
- assert(item_tag_get(&tree, 43, 0) == 0);
- item_tag_set(&tree, 43, 0);
- assert(item_tag_get(&tree, 43, 0) == 1);
- item_insert(&tree, 1000000);
- assert(item_tag_get(&tree, 43, 0) == 1);
-
- item_insert(&tree, 0);
- item_tag_set(&tree, 0, 0);
- item_delete(&tree, 1000000);
- assert(item_tag_get(&tree, 43, 0) != 0);
- item_delete(&tree, 43);
- assert(item_tag_get(&tree, 43, 0) == 0); /* crash */
- assert(item_tag_get(&tree, 0, 0) == 1);
-
- verify_tag_consistency(&tree, 0);
-
- item_kill_tree(&tree);
+ DEFINE_XARRAY(xa);
+
+ item_insert(&xa, 43);
+ assert(xa_get_mark(&xa, 43, XA_MARK_0) == 0);
+ xa_set_mark(&xa, 43, XA_MARK_0);
+ assert(xa_get_mark(&xa, 43, XA_MARK_0) == 1);
+ item_insert(&xa, 1000000);
+ assert(xa_get_mark(&xa, 43, XA_MARK_0) == 1);
+
+ item_insert(&xa, 0);
+ xa_set_mark(&xa, 0, XA_MARK_0);
+ item_delete(&xa, 1000000);
+ assert(xa_get_mark(&xa, 43, XA_MARK_0) != 0);
+ item_delete(&xa, 43);
+ assert(xa_get_mark(&xa, 43, XA_MARK_0) == 0); /* crash */
+ assert(xa_get_mark(&xa, 0, XA_MARK_0) == 1);
+
+ verify_mark_consistency(&xa, XA_MARK_0);
+
+ item_kill_tree(&xa);
}
/*
- * Check that tags propagate correctly when contracting a tree.
+ * Check that marks propagate correctly when contracting a tree.
*/
static void contract_checks(void)
{
struct item *item;
int tmp;
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
- tmp = 1<<RADIX_TREE_MAP_SHIFT;
- item_insert(&tree, tmp);
- item_insert(&tree, tmp+1);
- item_tag_set(&tree, tmp, 0);
- item_tag_set(&tree, tmp, 1);
- item_tag_set(&tree, tmp+1, 0);
- item_delete(&tree, tmp+1);
- item_tag_clear(&tree, tmp, 1);
+ tmp = 1 << XA_CHUNK_SHIFT;
+ item_insert(&xa, tmp);
+ item_insert(&xa, tmp+1);
+ xa_set_mark(&xa, tmp, XA_MARK_0);
+ xa_set_mark(&xa, tmp, XA_MARK_1);
+ xa_set_mark(&xa, tmp+1, XA_MARK_0);
+ item_delete(&xa, tmp+1);
+ xa_clear_mark(&xa, tmp, XA_MARK_1);
- assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 0) == 1);
- assert(radix_tree_gang_lookup_tag(&tree, (void **)&item, 0, 1, 1) == 0);
+ assert(xa_extract(&xa, (void **)&item, 0, ULONG_MAX, 1, XA_MARK_0) == 1);
+ assert(xa_extract(&xa, (void **)&item, 0, ULONG_MAX, 1, XA_MARK_1) == 0);
- assert(item_tag_get(&tree, tmp, 0) == 1);
- assert(item_tag_get(&tree, tmp, 1) == 0);
+ assert(xa_get_mark(&xa, tmp, XA_MARK_0) == 1);
+ assert(xa_get_mark(&xa, tmp, XA_MARK_1) == 0);
- verify_tag_consistency(&tree, 0);
- item_kill_tree(&tree);
+ verify_mark_consistency(&xa, XA_MARK_0);
+ item_kill_tree(&xa);
}
/*
#define N 127
#define BATCH 33
-static void gang_check(struct radix_tree_root *tree,
- char *thrash_state, int tag)
+static void gang_check(struct xarray *xa, char *thrash_state, xa_mark_t mark)
{
struct item *items[BATCH];
int nr_found;
unsigned long index = 0;
unsigned long last_index = 0;
- while ((nr_found = radix_tree_gang_lookup_tag(tree, (void **)items,
- index, BATCH, tag))) {
+ while ((nr_found = xa_extract(xa, (void **)items,
+ index, ULONG_MAX, BATCH, mark))) {
int i;
for (i = 0; i < nr_found; i++) {
}
}
-static void do_thrash(struct radix_tree_root *tree, char *thrash_state, int tag)
+static void do_thrash(struct xarray *xa, char *thrash_state, xa_mark_t mark)
{
int insert_chunk;
int delete_chunk;
index = rand() % THRASH_SIZE;
if (thrash_state[index] != NODE_ABSENT)
continue;
- item_check_absent(tree, index);
- item_insert(tree, index);
+ item_check_absent(xa, index);
+ item_insert(xa, index);
assert(thrash_state[index] != NODE_PRESENT);
thrash_state[index] = NODE_PRESENT;
nr_inserted++;
index = rand() % THRASH_SIZE;
if (thrash_state[index] == NODE_ABSENT)
continue;
- item_check_present(tree, index);
- if (item_tag_get(tree, index, tag)) {
+ item_check_present(xa, index);
+ if (xa_get_mark(xa, index, mark)) {
assert(thrash_state[index] == NODE_TAGGED);
total_tagged--;
} else {
assert(thrash_state[index] == NODE_PRESENT);
}
- item_delete(tree, index);
+ item_delete(xa, index);
assert(thrash_state[index] != NODE_ABSENT);
thrash_state[index] = NODE_ABSENT;
nr_deleted++;
for (i = 0; i < tag_chunk; i++) {
index = rand() % THRASH_SIZE;
if (thrash_state[index] != NODE_PRESENT) {
- if (item_lookup(tree, index))
- assert(item_tag_get(tree, index, tag));
+ if (item_lookup(xa, index))
+ assert(xa_get_mark(xa, index, mark));
continue;
}
- item_tag_set(tree, index, tag);
- item_tag_set(tree, index, tag);
+ xa_set_mark(xa, index, mark);
+ xa_set_mark(xa, index, mark);
assert(thrash_state[index] != NODE_TAGGED);
thrash_state[index] = NODE_TAGGED;
nr_tagged++;
index = rand() % THRASH_SIZE;
if (thrash_state[index] != NODE_TAGGED)
continue;
- item_check_present(tree, index);
- assert(item_tag_get(tree, index, tag));
- item_tag_clear(tree, index, tag);
- item_tag_clear(tree, index, tag);
+ item_check_present(xa, index);
+ assert(xa_get_mark(xa, index, mark));
+ xa_clear_mark(xa, index, mark);
+ xa_clear_mark(xa, index, mark);
assert(thrash_state[index] != NODE_PRESENT);
thrash_state[index] = NODE_PRESENT;
nr_untagged++;
for (index = 0; index < THRASH_SIZE; index++) {
switch (thrash_state[index]) {
case NODE_ABSENT:
- item_check_absent(tree, index);
+ item_check_absent(xa, index);
break;
case NODE_PRESENT:
- item_check_present(tree, index);
- assert(!item_tag_get(tree, index, tag));
+ item_check_present(xa, index);
+ assert(!xa_get_mark(xa, index, mark));
actual_total_present++;
break;
case NODE_TAGGED:
- item_check_present(tree, index);
- assert(item_tag_get(tree, index, tag));
+ item_check_present(xa, index);
+ assert(xa_get_mark(xa, index, mark));
actual_total_present++;
actual_total_tagged++;
break;
}
}
- gang_check(tree, thrash_state, tag);
+ gang_check(xa, thrash_state, mark);
printv(2, "%d(%d) %d(%d) %d(%d) %d(%d) / "
"%d(%d) present, %d(%d) tagged\n",
static void thrash_tags(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
char *thrash_state;
thrash_state = malloc(THRASH_SIZE);
memset(thrash_state, 0, THRASH_SIZE);
- do_thrash(&tree, thrash_state, 0);
+ do_thrash(&xa, thrash_state, 0);
- verify_tag_consistency(&tree, 0);
- item_kill_tree(&tree);
+ verify_mark_consistency(&xa, XA_MARK_0);
+ item_kill_tree(&xa);
free(thrash_state);
}
static void leak_check(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
- item_insert(&tree, 1000000);
- item_delete(&tree, 1000000);
- item_kill_tree(&tree);
+ item_insert(&xa, 1000000);
+ item_delete(&xa, 1000000);
+ item_kill_tree(&xa);
}
static void __leak_check(void)
{
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated);
- item_insert(&tree, 1000000);
+ item_insert(&xa, 1000000);
printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated);
- item_delete(&tree, 1000000);
+ item_delete(&xa, 1000000);
printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated);
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
printv(2, "%d: nr_allocated=%d\n", __LINE__, nr_allocated);
}
static void single_check(void)
{
struct item *items[BATCH];
- RADIX_TREE(tree, GFP_KERNEL);
+ DEFINE_XARRAY(xa);
int ret;
unsigned long first = 0;
- item_insert(&tree, 0);
- item_tag_set(&tree, 0, 0);
- ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0);
+ item_insert(&xa, 0);
+ xa_set_mark(&xa, 0, XA_MARK_0);
+ ret = xa_extract(&xa, (void **)items, 0, ULONG_MAX, BATCH, XA_MARK_0);
assert(ret == 1);
- ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 1, BATCH, 0);
+ ret = xa_extract(&xa, (void **)items, 1, ULONG_MAX, BATCH, XA_MARK_0);
assert(ret == 0);
- verify_tag_consistency(&tree, 0);
- verify_tag_consistency(&tree, 1);
- ret = tag_tagged_items(&tree, first, 10, 10, XA_MARK_0, XA_MARK_1);
+ verify_mark_consistency(&xa, XA_MARK_0);
+ verify_mark_consistency(&xa, XA_MARK_1);
+ ret = mark_marked_items(&xa, first, 10, 10, XA_MARK_0, XA_MARK_1);
assert(ret == 1);
- ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 1);
+ ret = xa_extract(&xa, (void **)items, 0, ULONG_MAX, BATCH, XA_MARK_1);
assert(ret == 1);
- item_tag_clear(&tree, 0, 0);
- ret = radix_tree_gang_lookup_tag(&tree, (void **)items, 0, BATCH, 0);
+ xa_clear_mark(&xa, 0, XA_MARK_0);
+ ret = xa_extract(&xa, (void **)items, 0, ULONG_MAX, BATCH, XA_MARK_0);
assert(ret == 0);
- item_kill_tree(&tree);
+ item_kill_tree(&xa);
}
void tag_check(void)
#include "test.h"
-struct item *
-item_tag_set(struct radix_tree_root *root, unsigned long index, int tag)
-{
- return radix_tree_tag_set(root, index, tag);
-}
-
-struct item *
-item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag)
-{
- return radix_tree_tag_clear(root, index, tag);
-}
-
-int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag)
-{
- return radix_tree_tag_get(root, index, tag);
-}
-
struct item *item_create(unsigned long index, unsigned int order)
{
struct item *ret = malloc(sizeof(*ret));
return ret;
}
-int item_insert(struct radix_tree_root *root, unsigned long index)
+int item_insert(struct xarray *xa, unsigned long index)
{
struct item *item = item_create(index, 0);
- int err = radix_tree_insert(root, item->index, item);
+ int err = xa_insert(xa, item->index, item, GFP_KERNEL);
if (err)
free(item);
return err;
void item_sanity(struct item *item, unsigned long index)
{
unsigned long mask;
- assert(!radix_tree_is_internal_node(item));
+ assert(!xa_is_internal(item));
assert(item->order < BITS_PER_LONG);
mask = (1UL << item->order) - 1;
assert((item->index | mask) == (index | mask));
free(item);
}
-int item_delete(struct radix_tree_root *root, unsigned long index)
+int item_delete(struct xarray *xa, unsigned long index)
{
- struct item *item = radix_tree_delete(root, index);
+ struct item *item = xa_erase(xa, index);
if (!item)
return 0;
return 0;
}
-void item_check_present(struct radix_tree_root *root, unsigned long index)
+void item_check_present(struct xarray *xa, unsigned long index)
{
struct item *item;
- item = radix_tree_lookup(root, index);
+ item = xa_load(xa, index);
assert(item != NULL);
item_sanity(item, index);
}
-struct item *item_lookup(struct radix_tree_root *root, unsigned long index)
+struct item *item_lookup(struct xarray *xa, unsigned long index)
{
- return radix_tree_lookup(root, index);
+ return xa_load(xa, index);
}
-void item_check_absent(struct radix_tree_root *root, unsigned long index)
+void item_check_absent(struct xarray *xa, unsigned long index)
{
struct item *item;
- item = radix_tree_lookup(root, index);
+ item = xa_load(xa, index);
assert(item == NULL);
}
/*
* Scan only the passed (start, start+nr] for present items
*/
-void item_gang_check_present(struct radix_tree_root *root,
+void item_gang_check_present(struct xarray *xa,
unsigned long start, unsigned long nr,
int chunk, int hop)
{
if (nr_to_find > (nr - into))
nr_to_find = nr - into;
- nfound = radix_tree_gang_lookup(root, (void **)items,
- start + into, nr_to_find);
+ nfound = xa_extract(xa, (void **)items, start + into,
+ ULONG_MAX, nr_to_find, XA_PRESENT);
assert(nfound == nr_to_find);
for (i = 0; i < nfound; i++)
assert(items[i]->index == start + into + i);
/*
* Scan the entire tree, only expecting present items (start, start+nr]
*/
-void item_full_scan(struct radix_tree_root *root, unsigned long start,
+void item_full_scan(struct xarray *xa, unsigned long start,
unsigned long nr, int chunk)
{
struct item *items[chunk];
// printf("%s(0x%08lx, 0x%08lx, %d)\n", __FUNCTION__, start, nr, chunk);
- while ((nfound = radix_tree_gang_lookup(root, (void **)items, into,
- chunk))) {
+ while ((nfound = xa_extract(xa, (void **)items, into, ULONG_MAX,
+ chunk, XA_PRESENT))) {
// printf("At 0x%08lx, nfound=%d\n", into, nfound);
for (i = 0; i < nfound; i++) {
assert(items[i]->index == this_index);
}
if (chunk)
assert(this_index == start + nr);
- nfound = radix_tree_gang_lookup(root, (void **)items,
- this_index, chunk);
+ nfound = xa_extract(xa, (void **)items, this_index, ULONG_MAX,
+ chunk, XA_PRESENT);
assert(nfound == 0);
}
/* Use the same pattern as tag_pages_for_writeback() in mm/page-writeback.c */
-int tag_tagged_items(struct xarray *xa, unsigned long start, unsigned long end,
- unsigned batch, xa_mark_t iftag, xa_mark_t thentag)
+int mark_marked_items(struct xarray *xa, unsigned long start, unsigned long end,
+ unsigned batch, xa_mark_t ifmark, xa_mark_t thenmark)
{
XA_STATE(xas, xa, start);
- unsigned int tagged = 0;
+ unsigned int marked = 0;
struct item *item;
if (batch == 0)
batch = 1;
xas_lock_irq(&xas);
- xas_for_each_marked(&xas, item, end, iftag) {
- xas_set_mark(&xas, thentag);
- if (++tagged % batch)
+ xas_for_each_marked(&xas, item, end, ifmark) {
+ xas_set_mark(&xas, thenmark);
+ if (++marked % batch)
continue;
xas_pause(&xas);
}
xas_unlock_irq(&xas);
- return tagged;
+ return marked;
}
-static int verify_node(struct radix_tree_node *slot, unsigned int tag,
- int tagged)
+static int verify_node(struct xa_node *node, xa_mark_t mark, int marked)
{
int anyset = 0;
int i;
int j;
- slot = entry_to_node(slot);
-
/* Verify consistency at this level */
- for (i = 0; i < RADIX_TREE_TAG_LONGS; i++) {
- if (slot->tags[tag][i]) {
+ for (i = 0; i < XA_MARK_LONGS; i++) {
+ if (node->marks[mark][i]) {
anyset = 1;
break;
}
}
- if (tagged != anyset) {
- printf("tag: %u, shift %u, tagged: %d, anyset: %d\n",
- tag, slot->shift, tagged, anyset);
- for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
- printf("tag %d: ", j);
- for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
- printf("%016lx ", slot->tags[j][i]);
+ if (marked != anyset) {
+ printf("mark: %u, shift %u, marked: %d, anyset: %d\n",
+ mark, node->shift, marked, anyset);
+ for (j = 0; j < XA_MAX_MARKS; j++) {
+ printf("mark %d: ", j);
+ for (i = 0; i < XA_MARK_LONGS; i++)
+ printf("%016lx ", node->marks[j][i]);
printf("\n");
}
return 1;
}
- assert(tagged == anyset);
+ assert(marked == anyset);
/* Go for next level */
- if (slot->shift > 0) {
- for (i = 0; i < RADIX_TREE_MAP_SIZE; i++)
- if (slot->slots[i])
- if (verify_node(slot->slots[i], tag,
- !!test_bit(i, slot->tags[tag]))) {
+ if (node->shift > 0) {
+ for (i = 0; i < ARRAY_SIZE(node->slots); i++)
+ if (xa_is_node(node->slots[i]))
+ if (verify_node(xa_to_node(node->slots[i]), mark,
+ !!test_bit(i, node->marks[mark]))) {
printf("Failure at off %d\n", i);
- for (j = 0; j < RADIX_TREE_MAX_TAGS; j++) {
- printf("tag %d: ", j);
- for (i = 0; i < RADIX_TREE_TAG_LONGS; i++)
- printf("%016lx ", slot->tags[j][i]);
+ for (j = 0; j < XA_MAX_MARKS; j++) {
+ printf("mark %d: ", j);
+ for (i = 0; i < XA_MARK_LONGS; i++)
+ printf("%016lx ", node->marks[j][i]);
printf("\n");
}
return 1;
return 0;
}
-void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag)
+void verify_mark_consistency(struct xarray *xa, xa_mark_t mark)
{
- struct radix_tree_node *node = root->xa_head;
- if (!radix_tree_is_internal_node(node))
+ void *entry = xa->xa_head;
+ if (!xa_is_node(entry))
return;
- verify_node(node, tag, !!root_tag_get(root, tag));
+ verify_node(xa_to_node(entry), mark, !!xa_marked(xa, mark));
}
void item_kill_tree(struct xarray *xa)
assert(xa_empty(xa));
}
-void tree_verify_min_height(struct radix_tree_root *root, int maxindex)
+void tree_verify_min_height(struct xarray *xa, unsigned long maxindex)
{
- unsigned shift;
- struct radix_tree_node *node = root->xa_head;
- if (!radix_tree_is_internal_node(node)) {
+ struct xa_node *node = xa->xa_head;
+ if (!xa_is_node(node)) {
assert(maxindex == 0);
return;
}
- node = entry_to_node(node);
- assert(maxindex <= node_maxindex(node));
-
- shift = node->shift;
- if (shift > 0)
- assert(maxindex > shift_maxindex(shift - RADIX_TREE_MAP_SHIFT));
- else
- assert(maxindex > 0);
+ node = xa_to_node(node);
+ assert(maxindex < (XA_CHUNK_SIZE << node->shift));
+ assert(maxindex >= (1 << node->shift));
}
};
struct item *item_create(unsigned long index, unsigned int order);
-int item_insert(struct radix_tree_root *root, unsigned long index);
+int item_insert(struct xarray *, unsigned long index);
void item_sanity(struct item *item, unsigned long index);
void item_free(struct item *item, unsigned long index);
-int item_delete(struct radix_tree_root *root, unsigned long index);
-int item_delete_rcu(struct xarray *xa, unsigned long index);
-struct item *item_lookup(struct radix_tree_root *root, unsigned long index);
+int item_delete(struct xarray *, unsigned long index);
+int item_delete_rcu(struct xarray *, unsigned long index);
+struct item *item_lookup(struct xarray *, unsigned long index);
-void item_check_present(struct radix_tree_root *root, unsigned long index);
-void item_check_absent(struct radix_tree_root *root, unsigned long index);
-void item_gang_check_present(struct radix_tree_root *root,
+void item_check_present(struct xarray *, unsigned long index);
+void item_check_absent(struct xarray *, unsigned long index);
+void item_gang_check_present(struct xarray *,
unsigned long start, unsigned long nr,
int chunk, int hop);
-void item_full_scan(struct radix_tree_root *root, unsigned long start,
+void item_full_scan(struct xarray *, unsigned long start,
unsigned long nr, int chunk);
-void item_kill_tree(struct radix_tree_root *root);
+void item_kill_tree(struct xarray *);
-int tag_tagged_items(struct xarray *, unsigned long start, unsigned long end,
- unsigned batch, xa_mark_t iftag, xa_mark_t thentag);
+int mark_marked_items(struct xarray *, unsigned long start, unsigned long end,
+ unsigned batch, xa_mark_t ifmark, xa_mark_t thenmark);
void xarray_tests(void);
void tag_check(void);
void idr_checks(void);
void ida_tests(void);
-struct item *
-item_tag_set(struct radix_tree_root *root, unsigned long index, int tag);
-struct item *
-item_tag_clear(struct radix_tree_root *root, unsigned long index, int tag);
-int item_tag_get(struct radix_tree_root *root, unsigned long index, int tag);
-void tree_verify_min_height(struct radix_tree_root *root, int maxindex);
-void verify_tag_consistency(struct radix_tree_root *root, unsigned int tag);
+void tree_verify_min_height(struct xarray *, unsigned long maxindex);
+void verify_mark_consistency(struct xarray *, xa_mark_t mark);
extern int nr_allocated;
/* Normally private parts of lib/radix-tree.c */
-struct radix_tree_node *entry_to_node(void *ptr);
-void radix_tree_dump(struct radix_tree_root *root);
-int root_tag_get(struct radix_tree_root *root, unsigned int tag);
-unsigned long node_maxindex(struct radix_tree_node *);
-unsigned long shift_maxindex(unsigned int shift);
int radix_tree_cpu_dead(unsigned int cpu);
struct radix_tree_preload {
unsigned nr;
- struct radix_tree_node *nodes;
+ struct xa_node *nodes;
};
extern struct radix_tree_preload radix_tree_preloads;
projects / users / willy / xarray.git / commitdiff