lib/find_bit: add find_next{,_and}_bit_wrap

The helper is better optimized for the worst case: in case of empty
cpumask, current code traverses 2 * size:

  next = cpumask_next_and(prev, src1p, src2p);
  if (next >= nr_cpu_ids)
  	next = cpumask_first_and(src1p, src2p);

At bitmap level we can stop earlier after checking 'size + offset' bits.

Signed-off-by: Yury Norov <yury.norov@gmail.com>

diff --git a/include/linux/find.h b/include/linux/find.h
index 128615a..77c087b 100644 (file)
--- a/include/linux/find.h
+++ b/include/linux/find.h
@@ -290,6 +290,52 @@ unsigned long find_last_bit(const unsigned long *addr, unsigned long size)
 }
 #endif
 
+/**
+ * find_next_and_bit_wrap - find the next set bit in both memory regions
+ * @addr1: The first address to base the search on
+ * @addr2: The second address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Returns the bit number for the next set bit, or first set bit up to @offset
+ * If no bits are set, returns @size.
+ */
+static inline
+unsigned long find_next_and_bit_wrap(const unsigned long *addr1,
+                                       const unsigned long *addr2,
+                                       unsigned long size, unsigned long offset)
+{
+       unsigned long bit = find_next_and_bit(addr1, addr2, size, offset);
+
+       if (bit < size)
+               return bit;
+
+       bit = find_first_and_bit(addr1, addr2, offset);
+       return bit < offset ? bit : size;
+}
+
+/**
+ * find_next_bit_wrap - find the next set bit in both memory regions
+ * @addr: The first address to base the search on
+ * @size: The bitmap size in bits
+ * @offset: The bitnumber to start searching at
+ *
+ * Returns the bit number for the next set bit, or first set bit up to @offset
+ * If no bits are set, returns @size.
+ */
+static inline
+unsigned long find_next_bit_wrap(const unsigned long *addr,
+                                       unsigned long size, unsigned long offset)
+{
+       unsigned long bit = find_next_bit(addr, size, offset);
+
+       if (bit < size)
+               return bit;
+
+       bit = find_first_bit(addr, offset);
+       return bit < offset ? bit : size;
+}
+
 /**
  * find_next_clump8 - find next 8-bit clump with set bits in a memory region
  * @clump: location to store copy of found clump

diff --git a/lib/cpumask.c b/lib/cpumask.c
index 2c4a63b..c7c3925 100644 (file)
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -166,10 +166,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);
 
-       next = cpumask_next_and(prev, src1p, src2p);
-       if (next >= nr_cpu_ids)
-               next = cpumask_first_and(src1p, src2p);
-
+       next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),
+                                       nr_cpumask_bits, prev + 1);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);
 
@@ -183,11 +181,7 @@ unsigned int cpumask_any_distribute(const struct cpumask *srcp)
 
        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);
-
-       next = cpumask_next(prev, srcp);
-       if (next >= nr_cpu_ids)
-               next = cpumask_first(srcp);
-
+       next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);