Linux 6.9-rc1

[linux-2.6-microblaze.git] / lib / cpumask.c

diff --git a/lib/cpumask.c b/lib/cpumask.c
index f0ae119..e77ee9d 100644 (file)
--- a/lib/cpumask.c
+++ b/lib/cpumask.c
@@ -14,7 +14,7 @@
  * @start: the start point of the iteration
  * @wrap: assume @n crossing @start terminates the iteration
  *
- * Returns >= nr_cpu_ids on completion
+ * Return: >= nr_cpu_ids on completion
  *
  * Note: the @wrap argument is required for the start condition when
  * we cannot assume @start is set in @mask.
@@ -45,10 +45,12 @@ EXPORT_SYMBOL(cpumask_next_wrap);
  * alloc_cpumask_var_node - allocate a struct cpumask on a given node
  * @mask: pointer to cpumask_var_t where the cpumask is returned
  * @flags: GFP_ flags
+ * @node: memory node from which to allocate or %NUMA_NO_NODE
  *
  * Only defined when CONFIG_CPUMASK_OFFSTACK=y, otherwise is
- * a nop returning a constant 1 (in <linux/cpumask.h>)
- * Returns TRUE if memory allocation succeeded, FALSE otherwise.
+ * a nop returning a constant 1 (in <linux/cpumask.h>).
+ *
+ * Return: TRUE if memory allocation succeeded, FALSE otherwise.
  *
  * In addition, mask will be NULL if this fails.  Note that gcc is
  * usually smart enough to know that mask can never be NULL if
@@ -110,15 +112,33 @@ void __init free_bootmem_cpumask_var(cpumask_var_t mask)
 #endif
 
 /**
- * cpumask_local_spread - select the i'th cpu with local numa cpu's first
+ * cpumask_local_spread - select the i'th cpu based on NUMA distances
  * @i: index number
  * @node: local numa_node
  *
- * This function selects an online CPU according to a numa aware policy;
- * local cpus are returned first, followed by non-local ones, then it
- * wraps around.
+ * Return: online CPU according to a numa aware policy; local cpus are returned
+ * first, followed by non-local ones, then it wraps around.
+ *
+ * For those who wants to enumerate all CPUs based on their NUMA distances,
+ * i.e. call this function in a loop, like:
+ *
+ * for (i = 0; i < num_online_cpus(); i++) {
+ *     cpu = cpumask_local_spread(i, node);
+ *     do_something(cpu);
+ * }
+ *
+ * There's a better alternative based on for_each()-like iterators:
  *
- * It's not very efficient, but useful for setup.
+ *     for_each_numa_hop_mask(mask, node) {
+ *             for_each_cpu_andnot(cpu, mask, prev)
+ *                     do_something(cpu);
+ *             prev = mask;
+ *     }
+ *
+ * It's simpler and more verbose than above. Complexity of iterator-based
+ * enumeration is O(sched_domains_numa_levels * nr_cpu_ids), while
+ * cpumask_local_spread() when called for each cpu is
+ * O(sched_domains_numa_levels * nr_cpu_ids * log(nr_cpu_ids)).
  */
 unsigned int cpumask_local_spread(unsigned int i, int node)
 {
@@ -127,38 +147,24 @@ unsigned int cpumask_local_spread(unsigned int i, int node)
        /* Wrap: we always want a cpu. */
        i %= num_online_cpus();
 
-       if (node == NUMA_NO_NODE) {
-               for_each_cpu(cpu, cpu_online_mask)
-                       if (i-- == 0)
-                               return cpu;
-       } else {
-               /* NUMA first. */
-               for_each_cpu_and(cpu, cpumask_of_node(node), cpu_online_mask)
-                       if (i-- == 0)
-                               return cpu;
-
-               for_each_cpu(cpu, cpu_online_mask) {
-                       /* Skip NUMA nodes, done above. */
-                       if (cpumask_test_cpu(cpu, cpumask_of_node(node)))
-                               continue;
-
-                       if (i-- == 0)
-                               return cpu;
-               }
-       }
-       BUG();
+       cpu = sched_numa_find_nth_cpu(cpu_online_mask, i, node);
+
+       WARN_ON(cpu >= nr_cpu_ids);
+       return cpu;
 }
 EXPORT_SYMBOL(cpumask_local_spread);
 
 static DEFINE_PER_CPU(int, distribute_cpu_mask_prev);
 
 /**
- * Returns an arbitrary cpu within srcp1 & srcp2.
+ * cpumask_any_and_distribute - Return an arbitrary cpu within src1p & src2p.
+ * @src1p: first &cpumask for intersection
+ * @src2p: second &cpumask for intersection
  *
  * Iterated calls using the same srcp1 and srcp2 will be distributed within
  * their intersection.
  *
- * Returns >= nr_cpu_ids if the intersection is empty.
+ * Return: >= nr_cpu_ids if the intersection is empty.
  */
 unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
                               const struct cpumask *src2p)
@@ -168,10 +174,8 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);
 
-       next = cpumask_next_and(prev, src1p, src2p);
-       if (next >= nr_cpu_ids)
-               next = cpumask_first_and(src1p, src2p);
-
+       next = find_next_and_bit_wrap(cpumask_bits(src1p), cpumask_bits(src2p),
+                                       nr_cpumask_bits, prev + 1);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);
 
@@ -179,17 +183,19 @@ unsigned int cpumask_any_and_distribute(const struct cpumask *src1p,
 }
 EXPORT_SYMBOL(cpumask_any_and_distribute);
 
+/**
+ * cpumask_any_distribute - Return an arbitrary cpu from srcp
+ * @srcp: &cpumask for selection
+ *
+ * Return: >= nr_cpu_ids if the intersection is empty.
+ */
 unsigned int cpumask_any_distribute(const struct cpumask *srcp)
 {
        unsigned int next, prev;
 
        /* NOTE: our first selection will skip 0. */
        prev = __this_cpu_read(distribute_cpu_mask_prev);
-
-       next = cpumask_next(prev, srcp);
-       if (next >= nr_cpu_ids)
-               next = cpumask_first(srcp);
-
+       next = find_next_bit_wrap(cpumask_bits(srcp), nr_cpumask_bits, prev + 1);
        if (next < nr_cpu_ids)
                __this_cpu_write(distribute_cpu_mask_prev, next);