Skip to content

Commit

Permalink
xarray: Step through an XArray
Browse files Browse the repository at this point in the history 
The xas_next and xas_prev functions move the xas index by one position,
and adjust the rest of the iterator state to match it.  This is more
efficient than calling xas_set() as it keeps the iterator at the leaves
of the tree instead of walking the iterator from the root each time.

Signed-off-by: Matthew Wilcox <[email protected]>
  • Loading branch information
Matthew Wilcox committed Oct 21, 2018
1 parent 687149f commit 64d3e9a
Show file tree
Hide file tree
Showing 3 changed files with 256 additions and 0 deletions.
67 changes: 67 additions & 0 deletions include/linux/xarray.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -828,6 +828,12 @@ static inline bool xas_not_node(struct xa_node *node)
return ((unsigned long)node & 3) || !node;
}

/* True if the node represents RESTART or an error */
static inline bool xas_frozen(struct xa_node *node)
{
return (unsigned long)node & 2;
}

/* True if the node represents head-of-tree, RESTART or BOUNDS */
static inline bool xas_top(struct xa_node *node)
{
Expand Down Expand Up @@ -1082,4 +1088,65 @@ enum {
for (entry = xas_find_marked(xas, max, mark); entry; \
entry = xas_next_marked(xas, max, mark))

void *__xas_next(struct xa_state *);
void *__xas_prev(struct xa_state *);

/**
* xas_prev() - Move iterator to previous index.
* @xas: XArray operation state.
*
* If the @xas was in an error state, it will remain in an error state
* and this function will return %NULL. If the @xas has never been walked,
* it will have the effect of calling xas_load(). Otherwise one will be
* subtracted from the index and the state will be walked to the correct
* location in the array for the next operation.
*
* If the iterator was referencing index 0, this function wraps
* around to %ULONG_MAX.
*
* Return: The entry at the new index. This may be %NULL or an internal
* entry.
*/
static inline void *xas_prev(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;

if (unlikely(xas_not_node(node) || node->shift ||
xas->xa_offset == 0))
return __xas_prev(xas);

xas->xa_index--;
xas->xa_offset--;
return xa_entry(xas->xa, node, xas->xa_offset);
}

/**
* xas_next() - Move state to next index.
* @xas: XArray operation state.
*
* If the @xas was in an error state, it will remain in an error state
* and this function will return %NULL. If the @xas has never been walked,
* it will have the effect of calling xas_load(). Otherwise one will be
* added to the index and the state will be walked to the correct
* location in the array for the next operation.
*
* If the iterator was referencing index %ULONG_MAX, this function wraps
* around to 0.
*
* Return: The entry at the new index. This may be %NULL or an internal
* entry.
*/
static inline void *xas_next(struct xa_state *xas)
{
struct xa_node *node = xas->xa_node;

if (unlikely(xas_not_node(node) || node->shift ||
xas->xa_offset == XA_CHUNK_MASK))
return __xas_next(xas);

xas->xa_index++;
xas->xa_offset++;
return xa_entry(xas->xa, node, xas->xa_offset);
}

#endif /* _LINUX_XARRAY_H */
115 changes: 115 additions & 0 deletions lib/test_xarray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,120 @@ static noinline void check_find(struct xarray *xa)
check_multi_find_2(xa);
}

static noinline void check_move_small(struct xarray *xa, unsigned long idx)
{
XA_STATE(xas, xa, 0);
unsigned long i;

xa_store_index(xa, 0, GFP_KERNEL);
xa_store_index(xa, idx, GFP_KERNEL);

rcu_read_lock();
for (i = 0; i < idx * 4; i++) {
void *entry = xas_next(&xas);
if (i <= idx)
XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
XA_BUG_ON(xa, xas.xa_index != i);
if (i == 0 || i == idx)
XA_BUG_ON(xa, entry != xa_mk_value(i));
else
XA_BUG_ON(xa, entry != NULL);
}
xas_next(&xas);
XA_BUG_ON(xa, xas.xa_index != i);

do {
void *entry = xas_prev(&xas);
i--;
if (i <= idx)
XA_BUG_ON(xa, xas.xa_node == XAS_RESTART);
XA_BUG_ON(xa, xas.xa_index != i);
if (i == 0 || i == idx)
XA_BUG_ON(xa, entry != xa_mk_value(i));
else
XA_BUG_ON(xa, entry != NULL);
} while (i > 0);

xas_set(&xas, ULONG_MAX);
XA_BUG_ON(xa, xas_next(&xas) != NULL);
XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
XA_BUG_ON(xa, xas_next(&xas) != xa_mk_value(0));
XA_BUG_ON(xa, xas.xa_index != 0);
XA_BUG_ON(xa, xas_prev(&xas) != NULL);
XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);
rcu_read_unlock();

xa_erase_index(xa, 0);
xa_erase_index(xa, idx);
XA_BUG_ON(xa, !xa_empty(xa));
}

static noinline void check_move(struct xarray *xa)
{
XA_STATE(xas, xa, (1 << 16) - 1);
unsigned long i;

for (i = 0; i < (1 << 16); i++)
XA_BUG_ON(xa, xa_store_index(xa, i, GFP_KERNEL) != NULL);

rcu_read_lock();
do {
void *entry = xas_prev(&xas);
i--;
XA_BUG_ON(xa, entry != xa_mk_value(i));
XA_BUG_ON(xa, i != xas.xa_index);
} while (i != 0);

XA_BUG_ON(xa, xas_prev(&xas) != NULL);
XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);

do {
void *entry = xas_next(&xas);
XA_BUG_ON(xa, entry != xa_mk_value(i));
XA_BUG_ON(xa, i != xas.xa_index);
i++;
} while (i < (1 << 16));
rcu_read_unlock();

for (i = (1 << 8); i < (1 << 15); i++)
xa_erase_index(xa, i);

i = xas.xa_index;

rcu_read_lock();
do {
void *entry = xas_prev(&xas);
i--;
if ((i < (1 << 8)) || (i >= (1 << 15)))
XA_BUG_ON(xa, entry != xa_mk_value(i));
else
XA_BUG_ON(xa, entry != NULL);
XA_BUG_ON(xa, i != xas.xa_index);
} while (i != 0);

XA_BUG_ON(xa, xas_prev(&xas) != NULL);
XA_BUG_ON(xa, xas.xa_index != ULONG_MAX);

do {
void *entry = xas_next(&xas);
if ((i < (1 << 8)) || (i >= (1 << 15)))
XA_BUG_ON(xa, entry != xa_mk_value(i));
else
XA_BUG_ON(xa, entry != NULL);
XA_BUG_ON(xa, i != xas.xa_index);
i++;
} while (i < (1 << 16));
rcu_read_unlock();

xa_destroy(xa);

for (i = 0; i < 16; i++)
check_move_small(xa, 1UL << i);

for (i = 2; i < 16; i++)
check_move_small(xa, (1UL << i) - 1);
}

static noinline void check_destroy(struct xarray *xa)
{
unsigned long index;
Expand Down Expand Up @@ -512,6 +626,7 @@ static int xarray_checks(void)
check_multi_store(&array);
check_find(&array);
check_destroy(&array);
check_move(&array);

printk("XArray: %u of %u tests passed\n", tests_passed, tests_run);
return (tests_run == tests_passed) ? 0 : -EINVAL;
Expand Down
74 changes: 74 additions & 0 deletions lib/xarray.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,80 @@ void xas_pause(struct xa_state *xas)
}
EXPORT_SYMBOL_GPL(xas_pause);

/*
* __xas_prev() - Find the previous entry in the XArray.
* @xas: XArray operation state.
*
* Helper function for xas_prev() which handles all the complex cases
* out of line.
*/
void *__xas_prev(struct xa_state *xas)
{
void *entry;

if (!xas_frozen(xas->xa_node))
xas->xa_index--;
if (xas_not_node(xas->xa_node))
return xas_load(xas);

if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
xas->xa_offset--;

while (xas->xa_offset == 255) {
xas->xa_offset = xas->xa_node->offset - 1;
xas->xa_node = xa_parent(xas->xa, xas->xa_node);
if (!xas->xa_node)
return set_bounds(xas);
}

for (;;) {
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
if (!xa_is_node(entry))
return entry;

xas->xa_node = xa_to_node(entry);
xas_set_offset(xas);
}
}
EXPORT_SYMBOL_GPL(__xas_prev);

/*
* __xas_next() - Find the next entry in the XArray.
* @xas: XArray operation state.
*
* Helper function for xas_next() which handles all the complex cases
* out of line.
*/
void *__xas_next(struct xa_state *xas)
{
void *entry;

if (!xas_frozen(xas->xa_node))
xas->xa_index++;
if (xas_not_node(xas->xa_node))
return xas_load(xas);

if (xas->xa_offset != get_offset(xas->xa_index, xas->xa_node))
xas->xa_offset++;

while (xas->xa_offset == XA_CHUNK_SIZE) {
xas->xa_offset = xas->xa_node->offset + 1;
xas->xa_node = xa_parent(xas->xa, xas->xa_node);
if (!xas->xa_node)
return set_bounds(xas);
}

for (;;) {
entry = xa_entry(xas->xa, xas->xa_node, xas->xa_offset);
if (!xa_is_node(entry))
return entry;

xas->xa_node = xa_to_node(entry);
xas_set_offset(xas);
}
}
EXPORT_SYMBOL_GPL(__xas_next);

/**
* xas_find() - Find the next present entry in the XArray.
* @xas: XArray operation state.
Expand Down

0 comments on commit 64d3e9a

Please sign in to comment.