radix-tree: rewrite gang lookup using iterator

Rewrite radix_tree_gang_lookup_* functions using the new radix-tree
iterator.

Signed-off-by: Konstantin Khlebnikov <khlebnikov@openvz.org>
Tested-by: Hugh Dickins <hughd@google.com>
Cc: Christoph Hellwig <hch@lst.de>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>

diff --git a/lib/radix-tree.c b/lib/radix-tree.c
index fefa76e..86516f5 100644 (file)
--- a/lib/radix-tree.c
+++ b/lib/radix-tree.c
@@ -968,57 +968,6 @@ unsigned long radix_tree_prev_hole(struct radix_tree_root *root,
 }
 EXPORT_SYMBOL(radix_tree_prev_hole);
 
-static unsigned int
-__lookup(struct radix_tree_node *slot, void ***results, unsigned long *indices,
-       unsigned long index, unsigned int max_items, unsigned long *next_index)
-{
-       unsigned int nr_found = 0;
-       unsigned int shift, height;
-       unsigned long i;
-
-       height = slot->height;
-       if (height == 0)
-               goto out;
-       shift = (height-1) * RADIX_TREE_MAP_SHIFT;
-
-       for ( ; height > 1; height--) {
-               i = (index >> shift) & RADIX_TREE_MAP_MASK;
-               for (;;) {
-                       if (slot->slots[i] != NULL)
-                               break;
-                       index &= ~((1UL << shift) - 1);
-                       index += 1UL << shift;
-                       if (index == 0)
-                               goto out;       /* 32-bit wraparound */
-                       i++;
-                       if (i == RADIX_TREE_MAP_SIZE)
-                               goto out;
-               }
-
-               shift -= RADIX_TREE_MAP_SHIFT;
-               slot = rcu_dereference_raw(slot->slots[i]);
-               if (slot == NULL)
-                       goto out;
-       }
-
-       /* Bottom level: grab some items */
-       for (i = index & RADIX_TREE_MAP_MASK; i < RADIX_TREE_MAP_SIZE; i++) {
-               if (slot->slots[i]) {
-                       results[nr_found] = &(slot->slots[i]);
-                       if (indices)
-                               indices[nr_found] = index;
-                       if (++nr_found == max_items) {
-                               index++;
-                               goto out;
-                       }
-               }
-               index++;
-       }
-out:
-       *next_index = index;
-       return nr_found;
-}
-
 /**
  *     radix_tree_gang_lookup - perform multiple lookup on a radix tree
  *     @root:          radix tree root
@@ -1042,48 +991,19 @@ unsigned int
 radix_tree_gang_lookup(struct radix_tree_root *root, void **results,
                        unsigned long first_index, unsigned int max_items)
 {
-       unsigned long max_index;
-       struct radix_tree_node *node;
-       unsigned long cur_index = first_index;
-       unsigned int ret;
+       struct radix_tree_iter iter;
+       void **slot;
+       unsigned int ret = 0;
 
-       node = rcu_dereference_raw(root->rnode);
-       if (!node)
+       if (unlikely(!max_items))
                return 0;
 
-       if (!radix_tree_is_indirect_ptr(node)) {
-               if (first_index > 0)
-                       return 0;
-               results[0] = node;
-               return 1;
-       }
-       node = indirect_to_ptr(node);
-
-       max_index = radix_tree_maxindex(node->height);
-
-       ret = 0;
-       while (ret < max_items) {
-               unsigned int nr_found, slots_found, i;
-               unsigned long next_index;       /* Index of next search */
-
-               if (cur_index > max_index)
-                       break;
-               slots_found = __lookup(node, (void ***)results + ret, NULL,
-                               cur_index, max_items - ret, &next_index);
-               nr_found = 0;
-               for (i = 0; i < slots_found; i++) {
-                       struct radix_tree_node *slot;
-                       slot = *(((void ***)results)[ret + i]);
-                       if (!slot)
-                               continue;
-                       results[ret + nr_found] =
-                               indirect_to_ptr(rcu_dereference_raw(slot));
-                       nr_found++;
-               }
-               ret += nr_found;
-               if (next_index == 0)
+       radix_tree_for_each_slot(slot, root, &iter, first_index) {
+               results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
+               if (!results[ret])
+                       continue;
+               if (++ret == max_items)
                        break;
-               cur_index = next_index;
        }
 
        return ret;
@@ -1113,112 +1033,25 @@ radix_tree_gang_lookup_slot(struct radix_tree_root *root,
                        void ***results, unsigned long *indices,
                        unsigned long first_index, unsigned int max_items)
 {
-       unsigned long max_index;
-       struct radix_tree_node *node;
-       unsigned long cur_index = first_index;
-       unsigned int ret;
+       struct radix_tree_iter iter;
+       void **slot;
+       unsigned int ret = 0;
 
-       node = rcu_dereference_raw(root->rnode);
-       if (!node)
+       if (unlikely(!max_items))
                return 0;
 
-       if (!radix_tree_is_indirect_ptr(node)) {
-               if (first_index > 0)
-                       return 0;
-               results[0] = (void **)&root->rnode;
+       radix_tree_for_each_slot(slot, root, &iter, first_index) {
+               results[ret] = slot;
                if (indices)
-                       indices[0] = 0;
-               return 1;
-       }
-       node = indirect_to_ptr(node);
-
-       max_index = radix_tree_maxindex(node->height);
-
-       ret = 0;
-       while (ret < max_items) {
-               unsigned int slots_found;
-               unsigned long next_index;       /* Index of next search */
-
-               if (cur_index > max_index)
+                       indices[ret] = iter.index;
+               if (++ret == max_items)
                        break;
-               slots_found = __lookup(node, results + ret,
-                               indices ? indices + ret : NULL,
-                               cur_index, max_items - ret, &next_index);
-               ret += slots_found;
-               if (next_index == 0)
-                       break;
-               cur_index = next_index;
        }
 
        return ret;
 }
 EXPORT_SYMBOL(radix_tree_gang_lookup_slot);
 
-/*
- * FIXME: the two tag_get()s here should use find_next_bit() instead of
- * open-coding the search.
- */
-static unsigned int
-__lookup_tag(struct radix_tree_node *slot, void ***results, unsigned long index,
-       unsigned int max_items, unsigned long *next_index, unsigned int tag)
-{
-       unsigned int nr_found = 0;
-       unsigned int shift, height;
-
-       height = slot->height;
-       if (height == 0)
-               goto out;
-       shift = (height-1) * RADIX_TREE_MAP_SHIFT;
-
-       while (height > 0) {
-               unsigned long i = (index >> shift) & RADIX_TREE_MAP_MASK ;
-
-               for (;;) {
-                       if (tag_get(slot, tag, i))
-                               break;
-                       index &= ~((1UL << shift) - 1);
-                       index += 1UL << shift;
-                       if (index == 0)
-                               goto out;       /* 32-bit wraparound */
-                       i++;
-                       if (i == RADIX_TREE_MAP_SIZE)
-                               goto out;
-               }
-               height--;
-               if (height == 0) {      /* Bottom level: grab some items */
-                       unsigned long j = index & RADIX_TREE_MAP_MASK;
-
-                       for ( ; j < RADIX_TREE_MAP_SIZE; j++) {
-                               index++;
-                               if (!tag_get(slot, tag, j))
-                                       continue;
-                               /*
-                                * Even though the tag was found set, we need to
-                                * recheck that we have a non-NULL node, because
-                                * if this lookup is lockless, it may have been
-                                * subsequently deleted.
-                                *
-                                * Similar care must be taken in any place that
-                                * lookup ->slots[x] without a lock (ie. can't
-                                * rely on its value remaining the same).
-                                */
-                               if (slot->slots[j]) {
-                                       results[nr_found++] = &(slot->slots[j]);
-                                       if (nr_found == max_items)
-                                               goto out;
-                               }
-                       }
-               }
-               shift -= RADIX_TREE_MAP_SHIFT;
-               slot = rcu_dereference_raw(slot->slots[i]);
-               if (slot == NULL)
-                       break;
-       }
-out:
-       *next_index = index;
-       return nr_found;
-}
-
 /**
  *     radix_tree_gang_lookup_tag - perform multiple lookup on a radix tree
  *                                  based on a tag
@@ -1237,52 +1070,19 @@ radix_tree_gang_lookup_tag(struct radix_tree_root *root, void **results,
                unsigned long first_index, unsigned int max_items,
                unsigned int tag)
 {
-       struct radix_tree_node *node;
-       unsigned long max_index;
-       unsigned long cur_index = first_index;
-       unsigned int ret;
+       struct radix_tree_iter iter;
+       void **slot;
+       unsigned int ret = 0;
 
-       /* check the root's tag bit */
-       if (!root_tag_get(root, tag))
+       if (unlikely(!max_items))
                return 0;
 
-       node = rcu_dereference_raw(root->rnode);
-       if (!node)
-               return 0;
-
-       if (!radix_tree_is_indirect_ptr(node)) {
-               if (first_index > 0)
-                       return 0;
-               results[0] = node;
-               return 1;
-       }
-       node = indirect_to_ptr(node);
-
-       max_index = radix_tree_maxindex(node->height);
-
-       ret = 0;
-       while (ret < max_items) {
-               unsigned int nr_found, slots_found, i;
-               unsigned long next_index;       /* Index of next search */
-
-               if (cur_index > max_index)
-                       break;
-               slots_found = __lookup_tag(node, (void ***)results + ret,
-                               cur_index, max_items - ret, &next_index, tag);
-               nr_found = 0;
-               for (i = 0; i < slots_found; i++) {
-                       struct radix_tree_node *slot;
-                       slot = *(((void ***)results)[ret + i]);
-                       if (!slot)
-                               continue;
-                       results[ret + nr_found] =
-                               indirect_to_ptr(rcu_dereference_raw(slot));
-                       nr_found++;
-               }
-               ret += nr_found;
-               if (next_index == 0)
+       radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
+               results[ret] = indirect_to_ptr(rcu_dereference_raw(*slot));
+               if (!results[ret])
+                       continue;
+               if (++ret == max_items)
                        break;
-               cur_index = next_index;
        }
 
        return ret;
@@ -1307,42 +1107,17 @@ radix_tree_gang_lookup_tag_slot(struct radix_tree_root *root, void ***results,
                unsigned long first_index, unsigned int max_items,
                unsigned int tag)
 {
-       struct radix_tree_node *node;
-       unsigned long max_index;
-       unsigned long cur_index = first_index;
-       unsigned int ret;
+       struct radix_tree_iter iter;
+       void **slot;
+       unsigned int ret = 0;
 
-       /* check the root's tag bit */
-       if (!root_tag_get(root, tag))
-               return 0;
-
-       node = rcu_dereference_raw(root->rnode);
-       if (!node)
+       if (unlikely(!max_items))
                return 0;
 
-       if (!radix_tree_is_indirect_ptr(node)) {
-               if (first_index > 0)
-                       return 0;
-               results[0] = (void **)&root->rnode;
-               return 1;
-       }
-       node = indirect_to_ptr(node);
-
-       max_index = radix_tree_maxindex(node->height);
-
-       ret = 0;
-       while (ret < max_items) {
-               unsigned int slots_found;
-               unsigned long next_index;       /* Index of next search */
-
-               if (cur_index > max_index)
-                       break;
-               slots_found = __lookup_tag(node, results + ret,
-                               cur_index, max_items - ret, &next_index, tag);
-               ret += slots_found;
-               if (next_index == 0)
+       radix_tree_for_each_tagged(slot, root, &iter, first_index, tag) {
+               results[ret] = slot;
+               if (++ret == max_items)
                        break;
-               cur_index = next_index;
        }
 
        return ret;