net/rxrpc/call_accept.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /* incoming call handling
   3  *
   4  * Copyright (C) 2007 Red Hat, Inc. All Rights Reserved.
   5  * Written by David Howells (dhowells@redhat.com)
   6  */
   7
   8 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   9
  10 #include <linux/module.h>
  11 #include <linux/net.h>
  12 #include <linux/skbuff.h>
  13 #include <linux/errqueue.h>
  14 #include <linux/udp.h>
  15 #include <linux/in.h>
  16 #include <linux/in6.h>
  17 #include <linux/icmp.h>
  18 #include <linux/gfp.h>
  19 #include <linux/circ_buf.h>
  20 #include <net/sock.h>
  21 #include <net/af_rxrpc.h>
  22 #include <net/ip.h>
  23 #include "ar-internal.h"
  24
  25 static void rxrpc_dummy_notify(struct sock *sk, struct rxrpc_call *call,
  26                                unsigned long user_call_ID)
  27 {
  28 }
  29
  30 /*
  31  * Preallocate a single service call, connection and peer and, if possible,
  32  * give them a user ID and attach the user's side of the ID to them.
  33  */
  34 static int rxrpc_service_prealloc_one(struct rxrpc_sock *rx,
  35                                       struct rxrpc_backlog *b,
  36                                       rxrpc_notify_rx_t notify_rx,
  37                                       rxrpc_user_attach_call_t user_attach_call,
  38                                       unsigned long user_call_ID, gfp_t gfp,
  39                                       unsigned int debug_id)
  40 {
  41         const void *here = __builtin_return_address(0);
  42         struct rxrpc_call *call, *xcall;
  43         struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
  44         struct rb_node *parent, **pp;
  45         int max, tmp;
  46         unsigned int size = RXRPC_BACKLOG_MAX;
  47         unsigned int head, tail, call_head, call_tail;
  48
  49         max = rx->sk.sk_max_ack_backlog;
  50         tmp = rx->sk.sk_ack_backlog;
  51         if (tmp >= max) {
  52                 _leave(" = -ENOBUFS [full %u]", max);
  53                 return -ENOBUFS;
  54         }
  55         max -= tmp;
  56
  57         /* We don't need more conns and peers than we have calls, but on the
  58          * other hand, we shouldn't ever use more peers than conns or conns
  59          * than calls.
  60          */
  61         call_head = b->call_backlog_head;
  62         call_tail = READ_ONCE(b->call_backlog_tail);
  63         tmp = CIRC_CNT(call_head, call_tail, size);
  64         if (tmp >= max) {
  65                 _leave(" = -ENOBUFS [enough %u]", tmp);
  66                 return -ENOBUFS;
  67         }
  68         max = tmp + 1;
  69
  70         head = b->peer_backlog_head;
  71         tail = READ_ONCE(b->peer_backlog_tail);
  72         if (CIRC_CNT(head, tail, size) < max) {
  73                 struct rxrpc_peer *peer = rxrpc_alloc_peer(rx->local, gfp);
  74                 if (!peer)
  75                         return -ENOMEM;
  76                 b->peer_backlog[head] = peer;
  77                 smp_store_release(&b->peer_backlog_head,
  78                                   (head + 1) & (size - 1));
  79         }
  80
  81         head = b->conn_backlog_head;
  82         tail = READ_ONCE(b->conn_backlog_tail);
  83         if (CIRC_CNT(head, tail, size) < max) {
  84                 struct rxrpc_connection *conn;
  85
  86                 conn = rxrpc_prealloc_service_connection(rxnet, gfp);
  87                 if (!conn)
  88                         return -ENOMEM;
  89                 b->conn_backlog[head] = conn;
  90                 smp_store_release(&b->conn_backlog_head,
  91                                   (head + 1) & (size - 1));
  92
  93                 trace_rxrpc_conn(conn->debug_id, rxrpc_conn_new_service,
  94                                  atomic_read(&conn->usage), here);
  95         }
  96
  97         /* Now it gets complicated, because calls get registered with the
  98          * socket here, with a user ID preassigned by the user.
  99          */
 100         call = rxrpc_alloc_call(rx, gfp, debug_id);
 101         if (!call)
 102                 return -ENOMEM;
 103         call->flags |= (1 << RXRPC_CALL_IS_SERVICE);
 104         call->state = RXRPC_CALL_SERVER_PREALLOC;
 105
 106         trace_rxrpc_call(call->debug_id, rxrpc_call_new_service,
 107                          atomic_read(&call->usage),
 108                          here, (const void *)user_call_ID);
 109
 110         write_lock(&rx->call_lock);
 111
 112         /* Check the user ID isn't already in use */
 113         pp = &rx->calls.rb_node;
 114         parent = NULL;
 115         while (*pp) {
 116                 parent = *pp;
 117                 xcall = rb_entry(parent, struct rxrpc_call, sock_node);
 118                 if (user_call_ID < xcall->user_call_ID)
 119                         pp = &(*pp)->rb_left;
 120                 else if (user_call_ID > xcall->user_call_ID)
 121                         pp = &(*pp)->rb_right;
 122                 else
 123                         goto id_in_use;
 124         }
 125
 126         call->user_call_ID = user_call_ID;
 127         call->notify_rx = notify_rx;
 128         if (user_attach_call) {
 129                 rxrpc_get_call(call, rxrpc_call_got_kernel);
 130                 user_attach_call(call, user_call_ID);
 131         }
 132
 133         rxrpc_get_call(call, rxrpc_call_got_userid);
 134         rb_link_node(&call->sock_node, parent, pp);
 135         rb_insert_color(&call->sock_node, &rx->calls);
 136         set_bit(RXRPC_CALL_HAS_USERID, &call->flags);
 137
 138         list_add(&call->sock_link, &rx->sock_calls);
 139
 140         write_unlock(&rx->call_lock);
 141
 142         rxnet = call->rxnet;
 143         write_lock(&rxnet->call_lock);
 144         list_add_tail(&call->link, &rxnet->calls);
 145         write_unlock(&rxnet->call_lock);
 146
 147         b->call_backlog[call_head] = call;
 148         smp_store_release(&b->call_backlog_head, (call_head + 1) & (size - 1));
 149         _leave(" = 0 [%d -> %lx]", call->debug_id, user_call_ID);
 150         return 0;
 151
 152 id_in_use:
 153         write_unlock(&rx->call_lock);
 154         rxrpc_cleanup_call(call);
 155         _leave(" = -EBADSLT");
 156         return -EBADSLT;
 157 }
 158
 159 /*
 160  * Allocate the preallocation buffers for incoming service calls.  These must
 161  * be charged manually.
 162  */
 163 int rxrpc_service_prealloc(struct rxrpc_sock *rx, gfp_t gfp)
 164 {
 165         struct rxrpc_backlog *b = rx->backlog;
 166
 167         if (!b) {
 168                 b = kzalloc(sizeof(struct rxrpc_backlog), gfp);
 169                 if (!b)
 170                         return -ENOMEM;
 171                 rx->backlog = b;
 172         }
 173
 174         return 0;
 175 }
 176
 177 /*
 178  * Discard the preallocation on a service.
 179  */
 180 void rxrpc_discard_prealloc(struct rxrpc_sock *rx)
 181 {
 182         struct rxrpc_backlog *b = rx->backlog;
 183         struct rxrpc_net *rxnet = rxrpc_net(sock_net(&rx->sk));
 184         unsigned int size = RXRPC_BACKLOG_MAX, head, tail;
 185
 186         if (!b)
 187                 return;
 188         rx->backlog = NULL;
 189
 190         /* Make sure that there aren't any incoming calls in progress before we
 191          * clear the preallocation buffers.
 192          */
 193         spin_lock_bh(&rx->incoming_lock);
 194         spin_unlock_bh(&rx->incoming_lock);
 195
 196         head = b->peer_backlog_head;
 197         tail = b->peer_backlog_tail;
 198         while (CIRC_CNT(head, tail, size) > 0) {
 199                 struct rxrpc_peer *peer = b->peer_backlog[tail];
 200                 kfree(peer);
 201                 tail = (tail + 1) & (size - 1);
 202         }
 203
 204         head = b->conn_backlog_head;
 205         tail = b->conn_backlog_tail;
 206         while (CIRC_CNT(head, tail, size) > 0) {
 207                 struct rxrpc_connection *conn = b->conn_backlog[tail];
 208                 write_lock(&rxnet->conn_lock);
 209                 list_del(&conn->link);
 210                 list_del(&conn->proc_link);
 211                 write_unlock(&rxnet->conn_lock);
 212                 kfree(conn);
 213                 if (atomic_dec_and_test(&rxnet->nr_conns))
 214                         wake_up_var(&rxnet->nr_conns);
 215                 tail = (tail + 1) & (size - 1);
 216         }
 217
 218         head = b->call_backlog_head;
 219         tail = b->call_backlog_tail;
 220         while (CIRC_CNT(head, tail, size) > 0) {
 221                 struct rxrpc_call *call = b->call_backlog[tail];
 222                 rcu_assign_pointer(call->socket, rx);
 223                 if (rx->discard_new_call) {
 224                         _debug("discard %lx", call->user_call_ID);
 225                         rx->discard_new_call(call, call->user_call_ID);
 226                         if (call->notify_rx)
 227                                 call->notify_rx = rxrpc_dummy_notify;
 228                         rxrpc_put_call(call, rxrpc_call_put_kernel);
 229                 }
 230                 rxrpc_call_completed(call);
 231                 rxrpc_release_call(rx, call);
 232                 rxrpc_put_call(call, rxrpc_call_put);
 233                 tail = (tail + 1) & (size - 1);
 234         }
 235
 236         kfree(b);
 237 }
 238
 239 /*
 240  * Ping the other end to fill our RTT cache and to retrieve the rwind
 241  * and MTU parameters.
 242  */
 243 static void rxrpc_send_ping(struct rxrpc_call *call, struct sk_buff *skb)
 244 {
 245         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 246         ktime_t now = skb->tstamp;
 247
 248         if (call->peer->rtt_count < 3 ||
 249             ktime_before(ktime_add_ms(call->peer->rtt_last_req, 1000), now))
 250                 rxrpc_propose_ACK(call, RXRPC_ACK_PING, sp->hdr.serial,
 251                                   true, true,
 252                                   rxrpc_propose_ack_ping_for_params);
 253 }
 254
 255 /*
 256  * Allocate a new incoming call from the prealloc pool, along with a connection
 257  * and a peer as necessary.
 258  */
 259 static struct rxrpc_call *rxrpc_alloc_incoming_call(struct rxrpc_sock *rx,
 260                                                     struct rxrpc_local *local,
 261                                                     struct rxrpc_peer *peer,
 262                                                     struct rxrpc_connection *conn,
 263                                                     const struct rxrpc_security *sec,
 264                                                     struct sk_buff *skb)
 265 {
 266         struct rxrpc_backlog *b = rx->backlog;
 267         struct rxrpc_call *call;
 268         unsigned short call_head, conn_head, peer_head;
 269         unsigned short call_tail, conn_tail, peer_tail;
 270         unsigned short call_count, conn_count;
 271
 272         /* #calls >= #conns >= #peers must hold true. */
 273         call_head = smp_load_acquire(&b->call_backlog_head);
 274         call_tail = b->call_backlog_tail;
 275         call_count = CIRC_CNT(call_head, call_tail, RXRPC_BACKLOG_MAX);
 276         conn_head = smp_load_acquire(&b->conn_backlog_head);
 277         conn_tail = b->conn_backlog_tail;
 278         conn_count = CIRC_CNT(conn_head, conn_tail, RXRPC_BACKLOG_MAX);
 279         ASSERTCMP(conn_count, >=, call_count);
 280         peer_head = smp_load_acquire(&b->peer_backlog_head);
 281         peer_tail = b->peer_backlog_tail;
 282         ASSERTCMP(CIRC_CNT(peer_head, peer_tail, RXRPC_BACKLOG_MAX), >=,
 283                   conn_count);
 284
 285         if (call_count == 0)
 286                 return NULL;
 287
 288         if (!conn) {
 289                 if (peer && !rxrpc_get_peer_maybe(peer))
 290                         peer = NULL;
 291                 if (!peer) {
 292                         peer = b->peer_backlog[peer_tail];
 293                         if (rxrpc_extract_addr_from_skb(&peer->srx, skb) < 0)
 294                                 return NULL;
 295                         b->peer_backlog[peer_tail] = NULL;
 296                         smp_store_release(&b->peer_backlog_tail,
 297                                           (peer_tail + 1) &
 298                                           (RXRPC_BACKLOG_MAX - 1));
 299
 300                         rxrpc_new_incoming_peer(rx, local, peer);
 301                 }
 302
 303                 /* Now allocate and set up the connection */
 304                 conn = b->conn_backlog[conn_tail];
 305                 b->conn_backlog[conn_tail] = NULL;
 306                 smp_store_release(&b->conn_backlog_tail,
 307                                   (conn_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
 308                 conn->params.local = rxrpc_get_local(local);
 309                 conn->params.peer = peer;
 310                 rxrpc_see_connection(conn);
 311                 rxrpc_new_incoming_connection(rx, conn, sec, skb);
 312         } else {
 313                 rxrpc_get_connection(conn);
 314         }
 315
 316         /* And now we can allocate and set up a new call */
 317         call = b->call_backlog[call_tail];
 318         b->call_backlog[call_tail] = NULL;
 319         smp_store_release(&b->call_backlog_tail,
 320                           (call_tail + 1) & (RXRPC_BACKLOG_MAX - 1));
 321
 322         rxrpc_see_call(call);
 323         call->conn = conn;
 324         call->security = conn->security;
 325         call->security_ix = conn->security_ix;
 326         call->peer = rxrpc_get_peer(conn->params.peer);
 327         call->cong_cwnd = call->peer->cong_cwnd;
 328         return call;
 329 }
 330
 331 /*
 332  * Set up a new incoming call.  Called in BH context with the RCU read lock
 333  * held.
 334  *
 335  * If this is for a kernel service, when we allocate the call, it will have
 336  * three refs on it: (1) the kernel service, (2) the user_call_ID tree, (3) the
 337  * retainer ref obtained from the backlog buffer.  Prealloc calls for userspace
 338  * services only have the ref from the backlog buffer.  We want to pass this
 339  * ref to non-BH context to dispose of.
 340  *
 341  * If we want to report an error, we mark the skb with the packet type and
 342  * abort code and return NULL.
 343  *
 344  * The call is returned with the user access mutex held.
 345  */
 346 struct rxrpc_call *rxrpc_new_incoming_call(struct rxrpc_local *local,
 347                                            struct rxrpc_sock *rx,
 348                                            struct sk_buff *skb)
 349 {
 350         struct rxrpc_skb_priv *sp = rxrpc_skb(skb);
 351         const struct rxrpc_security *sec = NULL;
 352         struct rxrpc_connection *conn;
 353         struct rxrpc_peer *peer = NULL;
 354         struct rxrpc_call *call = NULL;
 355
 356         _enter("");
 357
 358         spin_lock(&rx->incoming_lock);
 359         if (rx->sk.sk_state == RXRPC_SERVER_LISTEN_DISABLED ||
 360             rx->sk.sk_state == RXRPC_CLOSE) {
 361                 trace_rxrpc_abort(0, "CLS", sp->hdr.cid, sp->hdr.callNumber,
 362                                   sp->hdr.seq, RX_INVALID_OPERATION, ESHUTDOWN);
 363                 skb->mark = RXRPC_SKB_MARK_REJECT_ABORT;
 364                 skb->priority = RX_INVALID_OPERATION;
 365                 goto no_call;
 366         }
 367
 368         /* The peer, connection and call may all have sprung into existence due
 369          * to a duplicate packet being handled on another CPU in parallel, so
 370          * we have to recheck the routing.  However, we're now holding
 371          * rx->incoming_lock, so the values should remain stable.
 372          */
 373         conn = rxrpc_find_connection_rcu(local, skb, &peer);
 374
 375         if (!conn) {
 376                 sec = rxrpc_get_incoming_security(rx, skb);
 377                 if (!sec)
 378                         goto no_call;
 379         }
 380
 381         call = rxrpc_alloc_incoming_call(rx, local, peer, conn, sec, skb);
 382         if (!call) {
 383                 skb->mark = RXRPC_SKB_MARK_REJECT_BUSY;
 384                 goto no_call;
 385         }
 386
 387         trace_rxrpc_receive(call, rxrpc_receive_incoming,
 388                             sp->hdr.serial, sp->hdr.seq);
 389
 390         /* Make the call live. */
 391         rxrpc_incoming_call(rx, call, skb);
 392         conn = call->conn;
 393
 394         if (rx->notify_new_call)
 395                 rx->notify_new_call(&rx->sk, call, call->user_call_ID);
 396
 397         spin_lock(&conn->state_lock);
 398         switch (conn->state) {
 399         case RXRPC_CONN_SERVICE_UNSECURED:
 400                 conn->state = RXRPC_CONN_SERVICE_CHALLENGING;
 401                 set_bit(RXRPC_CONN_EV_CHALLENGE, &call->conn->events);
 402                 rxrpc_queue_conn(call->conn);
 403                 break;
 404
 405         case RXRPC_CONN_SERVICE:
 406                 write_lock(&call->state_lock);
 407                 if (call->state < RXRPC_CALL_COMPLETE)
 408                         call->state = RXRPC_CALL_SERVER_RECV_REQUEST;
 409                 write_unlock(&call->state_lock);
 410                 break;
 411
 412         case RXRPC_CONN_REMOTELY_ABORTED:
 413                 rxrpc_set_call_completion(call, RXRPC_CALL_REMOTELY_ABORTED,
 414                                           conn->abort_code, conn->error);
 415                 break;
 416         case RXRPC_CONN_LOCALLY_ABORTED:
 417                 rxrpc_abort_call("CON", call, sp->hdr.seq,
 418                                  conn->abort_code, conn->error);
 419                 break;
 420         default:
 421                 BUG();
 422         }
 423         spin_unlock(&conn->state_lock);
 424         spin_unlock(&rx->incoming_lock);
 425
 426         rxrpc_send_ping(call, skb);
 427
 428         /* We have to discard the prealloc queue's ref here and rely on a
 429          * combination of the RCU read lock and refs held either by the socket
 430          * (recvmsg queue, to-be-accepted queue or user ID tree) or the kernel
 431          * service to prevent the call from being deallocated too early.
 432          */
 433         rxrpc_put_call(call, rxrpc_call_put);
 434
 435         _leave(" = %p{%d}", call, call->debug_id);
 436         return call;
 437
 438 no_call:
 439         spin_unlock(&rx->incoming_lock);
 440         _leave(" = NULL [%u]", skb->mark);
 441         return NULL;
 442 }
 443
 444 /*
 445  * Charge up socket with preallocated calls, attaching user call IDs.
 446  */
 447 int rxrpc_user_charge_accept(struct rxrpc_sock *rx, unsigned long user_call_ID)
 448 {
 449         struct rxrpc_backlog *b = rx->backlog;
 450
 451         if (rx->sk.sk_state == RXRPC_CLOSE)
 452                 return -ESHUTDOWN;
 453
 454         return rxrpc_service_prealloc_one(rx, b, NULL, NULL, user_call_ID,
 455                                           GFP_KERNEL,
 456                                           atomic_inc_return(&rxrpc_debug_id));
 457 }
 458
 459 /*
 460  * rxrpc_kernel_charge_accept - Charge up socket with preallocated calls
 461  * @sock: The socket on which to preallocate
 462  * @notify_rx: Event notification function for the call
 463  * @user_attach_call: Func to attach call to user_call_ID
 464  * @user_call_ID: The tag to attach to the preallocated call
 465  * @gfp: The allocation conditions.
 466  * @debug_id: The tracing debug ID.
 467  *
 468  * Charge up the socket with preallocated calls, each with a user ID.  A
 469  * function should be provided to effect the attachment from the user's side.
 470  * The user is given a ref to hold on the call.
 471  *
 472  * Note that the call may be come connected before this function returns.
 473  */
 474 int rxrpc_kernel_charge_accept(struct socket *sock,
 475                                rxrpc_notify_rx_t notify_rx,
 476                                rxrpc_user_attach_call_t user_attach_call,
 477                                unsigned long user_call_ID, gfp_t gfp,
 478                                unsigned int debug_id)
 479 {
 480         struct rxrpc_sock *rx = rxrpc_sk(sock->sk);
 481         struct rxrpc_backlog *b = rx->backlog;
 482
 483         if (sock->sk->sk_state == RXRPC_CLOSE)
 484                 return -ESHUTDOWN;
 485
 486         return rxrpc_service_prealloc_one(rx, b, notify_rx,
 487                                           user_attach_call, user_call_ID,
 488                                           gfp, debug_id);
 489 }
 490 EXPORT_SYMBOL(rxrpc_kernel_charge_accept);