Merge remote-tracking branch 'spi/for-5.14' into spi-next

[linux-2.6-microblaze.git] / include / net / sch_generic.h

diff --git a/include/net/sch_generic.h b/include/net/sch_generic.h
index f7a6e14..1e62551 100644 (file)
--- a/include/net/sch_generic.h
+++ b/include/net/sch_generic.h
@@ -36,6 +36,7 @@ struct qdisc_rate_table {
 enum qdisc_state_t {
        __QDISC_STATE_SCHED,
        __QDISC_STATE_DEACTIVATED,
+       __QDISC_STATE_MISSED,
 };
 
 struct qdisc_size_table {
@@ -159,8 +160,33 @@ static inline bool qdisc_is_empty(const struct Qdisc *qdisc)
 static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 {
        if (qdisc->flags & TCQ_F_NOLOCK) {
+               if (spin_trylock(&qdisc->seqlock))
+                       goto nolock_empty;
+
+               /* If the MISSED flag is set, it means other thread has
+                * set the MISSED flag before second spin_trylock(), so
+                * we can return false here to avoid multi cpus doing
+                * the set_bit() and second spin_trylock() concurrently.
+                */
+               if (test_bit(__QDISC_STATE_MISSED, &qdisc->state))
+                       return false;
+
+               /* Set the MISSED flag before the second spin_trylock(),
+                * if the second spin_trylock() return false, it means
+                * other cpu holding the lock will do dequeuing for us
+                * or it will see the MISSED flag set after releasing
+                * lock and reschedule the net_tx_action() to do the
+                * dequeuing.
+                */
+               set_bit(__QDISC_STATE_MISSED, &qdisc->state);
+
+               /* Retry again in case other CPU may not see the new flag
+                * after it releases the lock at the end of qdisc_run_end().
+                */
                if (!spin_trylock(&qdisc->seqlock))
                        return false;
+
+nolock_empty:
                WRITE_ONCE(qdisc->empty, false);
        } else if (qdisc_is_running(qdisc)) {
                return false;
@@ -176,8 +202,15 @@ static inline bool qdisc_run_begin(struct Qdisc *qdisc)
 static inline void qdisc_run_end(struct Qdisc *qdisc)
 {
        write_seqcount_end(&qdisc->running);
-       if (qdisc->flags & TCQ_F_NOLOCK)
+       if (qdisc->flags & TCQ_F_NOLOCK) {
                spin_unlock(&qdisc->seqlock);
+
+               if (unlikely(test_bit(__QDISC_STATE_MISSED,
+                                     &qdisc->state))) {
+                       clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
+                       __netif_schedule(qdisc);
+               }
+       }
 }
 
 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc)