kernel/rcu/rcu_segcblist.h

   1 /* SPDX-License-Identifier: GPL-2.0+ */
   2 /*
   3  * RCU segmented callback lists, internal-to-rcu header file
   4  *
   5  * Copyright IBM Corporation, 2017
   6  *
   7  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
   8  */
   9
  10 #include <linux/rcu_segcblist.h>
  11
  12 /* Return number of callbacks in the specified callback list. */
  13 static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
  14 {
  15         return READ_ONCE(rclp->len);
  16 }
  17
  18 void rcu_cblist_init(struct rcu_cblist *rclp);
  19 void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
  20 void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
  21                               struct rcu_cblist *srclp,
  22                               struct rcu_head *rhp);
  23 struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
  24
  25 /*
  26  * Is the specified rcu_segcblist structure empty?
  27  *
  28  * But careful!  The fact that the ->head field is NULL does not
  29  * necessarily imply that there are no callbacks associated with
  30  * this structure.  When callbacks are being invoked, they are
  31  * removed as a group.  If callback invocation must be preempted,
  32  * the remaining callbacks will be added back to the list.  Either
  33  * way, the counts are updated later.
  34  *
  35  * So it is often the case that rcu_segcblist_n_cbs() should be used
  36  * instead.
  37  */
  38 static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
  39 {
  40         return !READ_ONCE(rsclp->head);
  41 }
  42
  43 /* Return number of callbacks in segmented callback list. */
  44 static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
  45 {
  46 #ifdef CONFIG_RCU_NOCB_CPU
  47         return atomic_long_read(&rsclp->len);
  48 #else
  49         return READ_ONCE(rsclp->len);
  50 #endif
  51 }
  52
  53 /*
  54  * Is the specified rcu_segcblist enabled, for example, not corresponding
  55  * to an offline CPU?
  56  */
  57 static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
  58 {
  59         return rsclp->enabled;
  60 }
  61
  62 /* Is the specified rcu_segcblist offloaded?  */
  63 static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
  64 {
  65         return IS_ENABLED(CONFIG_RCU_NOCB_CPU) && rsclp->offloaded;
  66 }
  67
  68 /*
  69  * Are all segments following the specified segment of the specified
  70  * rcu_segcblist structure empty of callbacks?  (The specified
  71  * segment might well contain callbacks.)
  72  */
  73 static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
  74 {
  75         return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
  76 }
  77
  78 void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
  79 void rcu_segcblist_init(struct rcu_segcblist *rsclp);
  80 void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
  81 void rcu_segcblist_offload(struct rcu_segcblist *rsclp);
  82 bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
  83 bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
  84 struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
  85 struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
  86 bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
  87 void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
  88                            struct rcu_head *rhp);
  89 bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
  90                            struct rcu_head *rhp);
  91 void rcu_segcblist_extract_count(struct rcu_segcblist *rsclp,
  92                                  struct rcu_cblist *rclp);
  93 void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
  94                                     struct rcu_cblist *rclp);
  95 void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
  96                                     struct rcu_cblist *rclp);
  97 void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
  98                                 struct rcu_cblist *rclp);
  99 void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
 100                                    struct rcu_cblist *rclp);
 101 void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
 102                                    struct rcu_cblist *rclp);
 103 void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
 104 bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
 105 void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
 106                          struct rcu_segcblist *src_rsclp);