tracepoint: Fix static call function vs data state mismatch

[linux-2.6-microblaze.git] / kernel / tracepoint.c

diff --git a/kernel/tracepoint.c b/kernel/tracepoint.c
index 976bf8c..8d772bd 100644 (file)
--- a/kernel/tracepoint.c
+++ b/kernel/tracepoint.c
@@ -15,6 +15,13 @@
 #include <linux/sched/task.h>
 #include <linux/static_key.h>
 
+enum tp_func_state {
+       TP_FUNC_0,
+       TP_FUNC_1,
+       TP_FUNC_2,
+       TP_FUNC_N,
+};
+
 extern tracepoint_ptr_t __start___tracepoints_ptrs[];
 extern tracepoint_ptr_t __stop___tracepoints_ptrs[];
 
@@ -246,26 +253,29 @@ static void *func_remove(struct tracepoint_func **funcs,
        return old;
 }
 
-static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs, bool sync)
+/*
+ * Count the number of functions (enum tp_func_state) in a tp_funcs array.
+ */
+static enum tp_func_state nr_func_state(const struct tracepoint_func *tp_funcs)
+{
+       if (!tp_funcs)
+               return TP_FUNC_0;
+       if (!tp_funcs[1].func)
+               return TP_FUNC_1;
+       if (!tp_funcs[2].func)
+               return TP_FUNC_2;
+       return TP_FUNC_N;       /* 3 or more */
+}
+
+static void tracepoint_update_call(struct tracepoint *tp, struct tracepoint_func *tp_funcs)
 {
        void *func = tp->iterator;
 
        /* Synthetic events do not have static call sites */
        if (!tp->static_call_key)
                return;
-
-       if (!tp_funcs[1].func) {
+       if (nr_func_state(tp_funcs) == TP_FUNC_1)
                func = tp_funcs[0].func;
-               /*
-                * If going from the iterator back to a single caller,
-                * we need to synchronize with __DO_TRACE to make sure
-                * that the data passed to the callback is the one that
-                * belongs to that callback.
-                */
-               if (sync)
-                       tracepoint_synchronize_unregister();
-       }
-
        __static_call_update(tp->static_call_key, tp->static_call_tramp, func);
 }
 
@@ -299,9 +309,31 @@ static int tracepoint_add_func(struct tracepoint *tp,
         * a pointer to it.  This array is referenced by __DO_TRACE from
         * include/linux/tracepoint.h using rcu_dereference_sched().
         */
-       rcu_assign_pointer(tp->funcs, tp_funcs);
-       tracepoint_update_call(tp, tp_funcs, false);
-       static_key_enable(&tp->key);
+       switch (nr_func_state(tp_funcs)) {
+       case TP_FUNC_1:         /* 0->1 */
+               /* Set static call to first function */
+               tracepoint_update_call(tp, tp_funcs);
+               /* Both iterator and static call handle NULL tp->funcs */
+               rcu_assign_pointer(tp->funcs, tp_funcs);
+               static_key_enable(&tp->key);
+               break;
+       case TP_FUNC_2:         /* 1->2 */
+               /* Set iterator static call */
+               tracepoint_update_call(tp, tp_funcs);
+               /*
+                * Iterator callback installed before updating tp->funcs.
+                * Requires ordering between RCU assign/dereference and
+                * static call update/call.
+                */
+               rcu_assign_pointer(tp->funcs, tp_funcs);
+               break;
+       case TP_FUNC_N:         /* N->N+1 (N>1) */
+               rcu_assign_pointer(tp->funcs, tp_funcs);
+               break;
+       default:
+               WARN_ON_ONCE(1);
+               break;
+       }
 
        release_probes(old);
        return 0;
@@ -328,17 +360,47 @@ static int tracepoint_remove_func(struct tracepoint *tp,
                /* Failed allocating new tp_funcs, replaced func with stub */
                return 0;
 
-       if (!tp_funcs) {
+       switch (nr_func_state(tp_funcs)) {
+       case TP_FUNC_0:         /* 1->0 */
                /* Removed last function */
                if (tp->unregfunc && static_key_enabled(&tp->key))
                        tp->unregfunc();
 
                static_key_disable(&tp->key);
+               /* Set iterator static call */
+               tracepoint_update_call(tp, tp_funcs);
+               /* Both iterator and static call handle NULL tp->funcs */
+               rcu_assign_pointer(tp->funcs, NULL);
+               /*
+                * Make sure new func never uses old data after a 1->0->1
+                * transition sequence.
+                * Considering that transition 0->1 is the common case
+                * and don't have rcu-sync, issue rcu-sync after
+                * transition 1->0 to break that sequence by waiting for
+                * readers to be quiescent.
+                */
+               tracepoint_synchronize_unregister();
+               break;
+       case TP_FUNC_1:         /* 2->1 */
                rcu_assign_pointer(tp->funcs, tp_funcs);
-       } else {
+               /*
+                * On 2->1 transition, RCU sync is needed before setting
+                * static call to first callback, because the observer
+                * may have loaded any prior tp->funcs after the last one
+                * associated with an rcu-sync.
+                */
+               tracepoint_synchronize_unregister();
+               /* Set static call to first function */
+               tracepoint_update_call(tp, tp_funcs);
+               break;
+       case TP_FUNC_2:         /* N->N-1 (N>2) */
+               fallthrough;
+       case TP_FUNC_N:
                rcu_assign_pointer(tp->funcs, tp_funcs);
-               tracepoint_update_call(tp, tp_funcs,
-                                      tp_funcs[0].func != old[0].func);
+               break;
+       default:
+               WARN_ON_ONCE(1);
+               break;
        }
        release_probes(old);
        return 0;