net: add sk_wake_async_rcu() helper

While looking at UDP receive performance, I saw sk_wake_async()
was no longer inlined.

This matters at least on AMD Zen1-4 platforms (see SRSO)

This might be because rcu_read_lock() and rcu_read_unlock()
are no longer nops in recent kernels ?

Add sk_wake_async_rcu() variant, which must be called from
contexts already holding rcu lock.

As SOCK_FASYNC is deprecated in modern days, use unlikely()
to give a hint to the compiler.

sk_wake_async_rcu() is properly inlined from
__udp_enqueue_schedule_skb() and sock_def_readable().

Signed-off-by: Eric Dumazet <edumazet@google.com>
Link: https://lore.kernel.org/r/20240328144032.1864988-5-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>

diff --git a/crypto/af_alg.c b/crypto/af_alg.c
index 68cc929..5bc6d0f 100644 (file)
--- a/crypto/af_alg.c
+++ b/crypto/af_alg.c
@@ -847,7 +847,7 @@ void af_alg_wmem_wakeup(struct sock *sk)
                wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
                                                           EPOLLRDNORM |
                                                           EPOLLRDBAND);
-       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+       sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
        rcu_read_unlock();
 }
 EXPORT_SYMBOL_GPL(af_alg_wmem_wakeup);
@@ -914,7 +914,7 @@ static void af_alg_data_wakeup(struct sock *sk)
                wake_up_interruptible_sync_poll(&wq->wait, EPOLLOUT |
                                                           EPOLLRDNORM |
                                                           EPOLLRDBAND);
-       sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+       sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        rcu_read_unlock();
 }
 

diff --git a/include/net/sock.h b/include/net/sock.h
index f57bfd8..2253eef 100644 (file)
--- a/include/net/sock.h
+++ b/include/net/sock.h
@@ -2513,6 +2513,12 @@ static inline void sk_wake_async(const struct sock *sk, int how, int band)
        }
 }
 
+static inline void sk_wake_async_rcu(const struct sock *sk, int how, int band)
+{
+       if (unlikely(sock_flag(sk, SOCK_FASYNC)))
+               sock_wake_async(rcu_dereference(sk->sk_wq), how, band);
+}
+
 /* Since sk_{r,w}mem_alloc sums skb->truesize, even a small frame might
  * need sizeof(sk_buff) + MTU + padding, unless net driver perform copybreak.
  * Note: for send buffers, TCP works better if we can build two skbs at

diff --git a/net/atm/common.c b/net/atm/common.c
index 2a1ec01..9b75699 100644 (file)
--- a/net/atm/common.c
+++ b/net/atm/common.c
@@ -116,7 +116,7 @@ static void vcc_write_space(struct sock *sk)
                if (skwq_has_sleeper(wq))
                        wake_up_interruptible(&wq->wait);
 
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
 
        rcu_read_unlock();

diff --git a/net/core/sock.c b/net/core/sock.c
index 0963689..5ed4112 100644 (file)
--- a/net/core/sock.c
+++ b/net/core/sock.c
@@ -3338,7 +3338,7 @@ static void sock_def_error_report(struct sock *sk)
        wq = rcu_dereference(sk->sk_wq);
        if (skwq_has_sleeper(wq))
                wake_up_interruptible_poll(&wq->wait, EPOLLERR);
-       sk_wake_async(sk, SOCK_WAKE_IO, POLL_ERR);
+       sk_wake_async_rcu(sk, SOCK_WAKE_IO, POLL_ERR);
        rcu_read_unlock();
 }
 
@@ -3353,7 +3353,7 @@ void sock_def_readable(struct sock *sk)
        if (skwq_has_sleeper(wq))
                wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
                                                EPOLLRDNORM | EPOLLRDBAND);
-       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+       sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
        rcu_read_unlock();
 }
 
@@ -3373,7 +3373,7 @@ static void sock_def_write_space(struct sock *sk)
                                                EPOLLWRNORM | EPOLLWRBAND);
 
                /* Should agree with poll, otherwise some programs break */
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
 
        rcu_read_unlock();
@@ -3398,7 +3398,7 @@ static void sock_def_write_space_wfree(struct sock *sk)
                                                EPOLLWRNORM | EPOLLWRBAND);
 
                /* Should agree with poll, otherwise some programs break */
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
 }
 

diff --git a/net/dccp/output.c b/net/dccp/output.c
index fd2eb14..5c2e24f 100644 (file)
--- a/net/dccp/output.c
+++ b/net/dccp/output.c
@@ -204,7 +204,7 @@ void dccp_write_space(struct sock *sk)
                wake_up_interruptible(&wq->wait);
        /* Should agree with poll, otherwise some programs break */
        if (sock_writeable(sk))
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
 
        rcu_read_unlock();
 }

diff --git a/net/ipv4/udp.c b/net/ipv4/udp.c
index 143043c..11460d7 100644 (file)
--- a/net/ipv4/udp.c
+++ b/net/ipv4/udp.c
@@ -1544,7 +1544,7 @@ int __udp_enqueue_schedule_skb(struct sock *sk, struct sk_buff *skb)
                        INDIRECT_CALL_1(sk->sk_data_ready,
                                        sock_def_readable, sk);
                else
-                       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+                       sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
        }
        busylock_release(busy);
        return 0;

diff --git a/net/iucv/af_iucv.c b/net/iucv/af_iucv.c
index 7c8c3ad..c951bb9 100644 (file)
--- a/net/iucv/af_iucv.c
+++ b/net/iucv/af_iucv.c
@@ -184,7 +184,7 @@ static void iucv_sock_wake_msglim(struct sock *sk)
        wq = rcu_dereference(sk->sk_wq);
        if (skwq_has_sleeper(wq))
                wake_up_interruptible_all(&wq->wait);
-       sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+       sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        rcu_read_unlock();
 }
 

diff --git a/net/rxrpc/af_rxrpc.c b/net/rxrpc/af_rxrpc.c
index 5222bc9..f484468 100644 (file)
--- a/net/rxrpc/af_rxrpc.c
+++ b/net/rxrpc/af_rxrpc.c
@@ -65,7 +65,7 @@ static void rxrpc_write_space(struct sock *sk)
 
                if (skwq_has_sleeper(wq))
                        wake_up_interruptible(&wq->wait);
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
        rcu_read_unlock();
 }

diff --git a/net/sctp/socket.c b/net/sctp/socket.c
index c67679a..e416b6d 100644 (file)
--- a/net/sctp/socket.c
+++ b/net/sctp/socket.c
@@ -9276,7 +9276,7 @@ void sctp_data_ready(struct sock *sk)
        if (skwq_has_sleeper(wq))
                wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN |
                                                EPOLLRDNORM | EPOLLRDBAND);
-       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+       sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
        rcu_read_unlock();
 }
 

diff --git a/net/smc/smc_rx.c b/net/smc/smc_rx.c
index 9a2f363..f0cbe77 100644 (file)
--- a/net/smc/smc_rx.c
+++ b/net/smc/smc_rx.c
@@ -42,10 +42,10 @@ static void smc_rx_wake_up(struct sock *sk)
        if (skwq_has_sleeper(wq))
                wake_up_interruptible_sync_poll(&wq->wait, EPOLLIN | EPOLLPRI |
                                                EPOLLRDNORM | EPOLLRDBAND);
-       sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
+       sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_IN);
        if ((sk->sk_shutdown == SHUTDOWN_MASK) ||
            (sk->sk_state == SMC_CLOSED))
-               sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
+               sk_wake_async_rcu(sk, SOCK_WAKE_WAITD, POLL_HUP);
        rcu_read_unlock();
 }
 

diff --git a/net/unix/af_unix.c b/net/unix/af_unix.c
index 27ca50a..533fb68 100644 (file)
--- a/net/unix/af_unix.c
+++ b/net/unix/af_unix.c
@@ -546,7 +546,7 @@ static void unix_write_space(struct sock *sk)
                if (skwq_has_sleeper(wq))
                        wake_up_interruptible_sync_poll(&wq->wait,
                                EPOLLOUT | EPOLLWRNORM | EPOLLWRBAND);
-               sk_wake_async(sk, SOCK_WAKE_SPACE, POLL_OUT);
+               sk_wake_async_rcu(sk, SOCK_WAKE_SPACE, POLL_OUT);
        }
        rcu_read_unlock();
 }