drivers/net/ethernet/chelsio/inline_crypto/chtls/chtls_cm.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (c) 2018 Chelsio Communications, Inc.
   4  *
   5  * Written by: Atul Gupta (atul.gupta@chelsio.com)
   6  */
   7
   8 #include <linux/module.h>
   9 #include <linux/list.h>
  10 #include <linux/workqueue.h>
  11 #include <linux/skbuff.h>
  12 #include <linux/timer.h>
  13 #include <linux/notifier.h>
  14 #include <linux/inetdevice.h>
  15 #include <linux/ip.h>
  16 #include <linux/tcp.h>
  17 #include <linux/sched/signal.h>
  18 #include <linux/kallsyms.h>
  19 #include <linux/kprobes.h>
  20 #include <linux/if_vlan.h>
  21 #include <linux/ipv6.h>
  22 #include <net/ipv6.h>
  23 #include <net/transp_v6.h>
  24 #include <net/ip6_route.h>
  25 #include <net/inet_common.h>
  26 #include <net/tcp.h>
  27 #include <net/dst.h>
  28 #include <net/tls.h>
  29 #include <net/addrconf.h>
  30 #include <net/secure_seq.h>
  31
  32 #include "chtls.h"
  33 #include "chtls_cm.h"
  34 #include "clip_tbl.h"
  35
  36 /*
  37  * State transitions and actions for close.  Note that if we are in SYN_SENT
  38  * we remain in that state as we cannot control a connection while it's in
  39  * SYN_SENT; such connections are allowed to establish and are then aborted.
  40  */
  41 static unsigned char new_state[16] = {
  42         /* current state:     new state:      action: */
  43         /* (Invalid)       */ TCP_CLOSE,
  44         /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  45         /* TCP_SYN_SENT    */ TCP_SYN_SENT,
  46         /* TCP_SYN_RECV    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  47         /* TCP_FIN_WAIT1   */ TCP_FIN_WAIT1,
  48         /* TCP_FIN_WAIT2   */ TCP_FIN_WAIT2,
  49         /* TCP_TIME_WAIT   */ TCP_CLOSE,
  50         /* TCP_CLOSE       */ TCP_CLOSE,
  51         /* TCP_CLOSE_WAIT  */ TCP_LAST_ACK | TCP_ACTION_FIN,
  52         /* TCP_LAST_ACK    */ TCP_LAST_ACK,
  53         /* TCP_LISTEN      */ TCP_CLOSE,
  54         /* TCP_CLOSING     */ TCP_CLOSING,
  55 };
  56
  57 static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
  58 {
  59         struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
  60
  61         if (!csk)
  62                 return NULL;
  63
  64         csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
  65         if (!csk->txdata_skb_cache) {
  66                 kfree(csk);
  67                 return NULL;
  68         }
  69
  70         kref_init(&csk->kref);
  71         csk->cdev = cdev;
  72         skb_queue_head_init(&csk->txq);
  73         csk->wr_skb_head = NULL;
  74         csk->wr_skb_tail = NULL;
  75         csk->mss = MAX_MSS;
  76         csk->tlshws.ofld = 1;
  77         csk->tlshws.txkey = -1;
  78         csk->tlshws.rxkey = -1;
  79         csk->tlshws.mfs = TLS_MFS;
  80         skb_queue_head_init(&csk->tlshws.sk_recv_queue);
  81         return csk;
  82 }
  83
  84 static void chtls_sock_release(struct kref *ref)
  85 {
  86         struct chtls_sock *csk =
  87                 container_of(ref, struct chtls_sock, kref);
  88
  89         kfree(csk);
  90 }
  91
  92 static struct net_device *chtls_find_netdev(struct chtls_dev *cdev,
  93                                             struct sock *sk)
  94 {
  95         struct adapter *adap = pci_get_drvdata(cdev->pdev);
  96         struct net_device *ndev = cdev->ports[0];
  97 #if IS_ENABLED(CONFIG_IPV6)
  98         struct net_device *temp;
  99         int addr_type;
 100 #endif
 101         int i;
 102
 103         switch (sk->sk_family) {
 104         case PF_INET:
 105                 if (likely(!inet_sk(sk)->inet_rcv_saddr))
 106                         return ndev;
 107                 ndev = __ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr, false);
 108                 break;
 109 #if IS_ENABLED(CONFIG_IPV6)
 110         case PF_INET6:
 111                 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
 112                 if (likely(addr_type == IPV6_ADDR_ANY))
 113                         return ndev;
 114
 115                 for_each_netdev_rcu(&init_net, temp) {
 116                         if (ipv6_chk_addr(&init_net, (struct in6_addr *)
 117                                           &sk->sk_v6_rcv_saddr, temp, 1)) {
 118                                 ndev = temp;
 119                                 break;
 120                         }
 121                 }
 122         break;
 123 #endif
 124         default:
 125                 return NULL;
 126         }
 127
 128         if (!ndev)
 129                 return NULL;
 130
 131         if (is_vlan_dev(ndev))
 132                 ndev = vlan_dev_real_dev(ndev);
 133
 134         for_each_port(adap, i)
 135                 if (cdev->ports[i] == ndev)
 136                         return ndev;
 137         return NULL;
 138 }
 139
 140 static void assign_rxopt(struct sock *sk, unsigned int opt)
 141 {
 142         const struct chtls_dev *cdev;
 143         struct chtls_sock *csk;
 144         struct tcp_sock *tp;
 145
 146         csk = rcu_dereference_sk_user_data(sk);
 147         tp = tcp_sk(sk);
 148
 149         cdev = csk->cdev;
 150         tp->tcp_header_len           = sizeof(struct tcphdr);
 151         tp->rx_opt.mss_clamp         = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
 152         tp->mss_cache                = tp->rx_opt.mss_clamp;
 153         tp->rx_opt.tstamp_ok         = TCPOPT_TSTAMP_G(opt);
 154         tp->rx_opt.snd_wscale        = TCPOPT_SACK_G(opt);
 155         tp->rx_opt.wscale_ok         = TCPOPT_WSCALE_OK_G(opt);
 156         SND_WSCALE(tp)               = TCPOPT_SND_WSCALE_G(opt);
 157         if (!tp->rx_opt.wscale_ok)
 158                 tp->rx_opt.rcv_wscale = 0;
 159         if (tp->rx_opt.tstamp_ok) {
 160                 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
 161                 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
 162         } else if (csk->opt2 & TSTAMPS_EN_F) {
 163                 csk->opt2 &= ~TSTAMPS_EN_F;
 164                 csk->mtu_idx = TCPOPT_MSS_G(opt);
 165         }
 166 }
 167
 168 static void chtls_purge_receive_queue(struct sock *sk)
 169 {
 170         struct sk_buff *skb;
 171
 172         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
 173                 skb_dst_set(skb, (void *)NULL);
 174                 kfree_skb(skb);
 175         }
 176 }
 177
 178 static void chtls_purge_write_queue(struct sock *sk)
 179 {
 180         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 181         struct sk_buff *skb;
 182
 183         while ((skb = __skb_dequeue(&csk->txq))) {
 184                 sk->sk_wmem_queued -= skb->truesize;
 185                 __kfree_skb(skb);
 186         }
 187 }
 188
 189 static void chtls_purge_recv_queue(struct sock *sk)
 190 {
 191         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 192         struct chtls_hws *tlsk = &csk->tlshws;
 193         struct sk_buff *skb;
 194
 195         while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
 196                 skb_dst_set(skb, NULL);
 197                 kfree_skb(skb);
 198         }
 199 }
 200
 201 static void abort_arp_failure(void *handle, struct sk_buff *skb)
 202 {
 203         struct cpl_abort_req *req = cplhdr(skb);
 204         struct chtls_dev *cdev;
 205
 206         cdev = (struct chtls_dev *)handle;
 207         req->cmd = CPL_ABORT_NO_RST;
 208         cxgb4_ofld_send(cdev->lldi->ports[0], skb);
 209 }
 210
 211 static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
 212 {
 213         if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
 214                 __skb_trim(skb, 0);
 215                 refcount_add(2, &skb->users);
 216         } else {
 217                 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 218         }
 219         return skb;
 220 }
 221
 222 static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
 223 {
 224         struct cpl_abort_req *req;
 225         struct chtls_sock *csk;
 226         struct tcp_sock *tp;
 227
 228         csk = rcu_dereference_sk_user_data(sk);
 229         tp = tcp_sk(sk);
 230
 231         if (!skb)
 232                 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
 233
 234         req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
 235         INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
 236         skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
 237         req->rsvd0 = htonl(tp->snd_nxt);
 238         req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
 239         req->cmd = mode;
 240         t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
 241         send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
 242 }
 243
 244 static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
 245 {
 246         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 247
 248         if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
 249                      !csk->cdev)) {
 250                 if (sk->sk_state == TCP_SYN_RECV)
 251                         csk_set_flag(csk, CSK_RST_ABORTED);
 252                 goto out;
 253         }
 254
 255         if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
 256                 struct tcp_sock *tp = tcp_sk(sk);
 257
 258                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
 259                         WARN_ONCE(1, "send tx flowc error");
 260                 csk_set_flag(csk, CSK_TX_DATA_SENT);
 261         }
 262
 263         csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
 264         chtls_purge_write_queue(sk);
 265
 266         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
 267         if (sk->sk_state != TCP_SYN_RECV)
 268                 chtls_send_abort(sk, mode, skb);
 269         else
 270                 goto out;
 271
 272         return;
 273 out:
 274         kfree_skb(skb);
 275 }
 276
 277 static void release_tcp_port(struct sock *sk)
 278 {
 279         if (inet_csk(sk)->icsk_bind_hash)
 280                 inet_put_port(sk);
 281 }
 282
 283 static void tcp_uncork(struct sock *sk)
 284 {
 285         struct tcp_sock *tp = tcp_sk(sk);
 286
 287         if (tp->nonagle & TCP_NAGLE_CORK) {
 288                 tp->nonagle &= ~TCP_NAGLE_CORK;
 289                 chtls_tcp_push(sk, 0);
 290         }
 291 }
 292
 293 static void chtls_close_conn(struct sock *sk)
 294 {
 295         struct cpl_close_con_req *req;
 296         struct chtls_sock *csk;
 297         struct sk_buff *skb;
 298         unsigned int tid;
 299         unsigned int len;
 300
 301         len = roundup(sizeof(struct cpl_close_con_req), 16);
 302         csk = rcu_dereference_sk_user_data(sk);
 303         tid = csk->tid;
 304
 305         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 306         req = (struct cpl_close_con_req *)__skb_put(skb, len);
 307         memset(req, 0, len);
 308         req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
 309                               FW_WR_IMMDLEN_V(sizeof(*req) -
 310                                               sizeof(req->wr)));
 311         req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
 312                                FW_WR_FLOWID_V(tid));
 313
 314         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 315
 316         tcp_uncork(sk);
 317         skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
 318         if (sk->sk_state != TCP_SYN_SENT)
 319                 chtls_push_frames(csk, 1);
 320 }
 321
 322 /*
 323  * Perform a state transition during close and return the actions indicated
 324  * for the transition.  Do not make this function inline, the main reason
 325  * it exists at all is to avoid multiple inlining of tcp_set_state.
 326  */
 327 static int make_close_transition(struct sock *sk)
 328 {
 329         int next = (int)new_state[sk->sk_state];
 330
 331         tcp_set_state(sk, next & TCP_STATE_MASK);
 332         return next & TCP_ACTION_FIN;
 333 }
 334
 335 void chtls_close(struct sock *sk, long timeout)
 336 {
 337         int data_lost, prev_state;
 338         struct chtls_sock *csk;
 339
 340         csk = rcu_dereference_sk_user_data(sk);
 341
 342         lock_sock(sk);
 343         sk->sk_shutdown |= SHUTDOWN_MASK;
 344
 345         data_lost = skb_queue_len(&sk->sk_receive_queue);
 346         data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
 347         chtls_purge_recv_queue(sk);
 348         chtls_purge_receive_queue(sk);
 349
 350         if (sk->sk_state == TCP_CLOSE) {
 351                 goto wait;
 352         } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
 353                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 354                 release_tcp_port(sk);
 355                 goto unlock;
 356         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
 357                 sk->sk_prot->disconnect(sk, 0);
 358         } else if (make_close_transition(sk)) {
 359                 chtls_close_conn(sk);
 360         }
 361 wait:
 362         if (timeout)
 363                 sk_stream_wait_close(sk, timeout);
 364
 365 unlock:
 366         prev_state = sk->sk_state;
 367         sock_hold(sk);
 368         sock_orphan(sk);
 369
 370         release_sock(sk);
 371
 372         local_bh_disable();
 373         bh_lock_sock(sk);
 374
 375         if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
 376                 goto out;
 377
 378         if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
 379             !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
 380                 struct sk_buff *skb;
 381
 382                 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
 383                 if (skb)
 384                         chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
 385         }
 386
 387         if (sk->sk_state == TCP_CLOSE)
 388                 inet_csk_destroy_sock(sk);
 389
 390 out:
 391         bh_unlock_sock(sk);
 392         local_bh_enable();
 393         sock_put(sk);
 394 }
 395
 396 /*
 397  * Wait until a socket enters on of the given states.
 398  */
 399 static int wait_for_states(struct sock *sk, unsigned int states)
 400 {
 401         DECLARE_WAITQUEUE(wait, current);
 402         struct socket_wq _sk_wq;
 403         long current_timeo;
 404         int err = 0;
 405
 406         current_timeo = 200;
 407
 408         /*
 409          * We want this to work even when there's no associated struct socket.
 410          * In that case we provide a temporary wait_queue_head_t.
 411          */
 412         if (!sk->sk_wq) {
 413                 init_waitqueue_head(&_sk_wq.wait);
 414                 _sk_wq.fasync_list = NULL;
 415                 init_rcu_head_on_stack(&_sk_wq.rcu);
 416                 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
 417         }
 418
 419         add_wait_queue(sk_sleep(sk), &wait);
 420         while (!sk_in_state(sk, states)) {
 421                 if (!current_timeo) {
 422                         err = -EBUSY;
 423                         break;
 424                 }
 425                 if (signal_pending(current)) {
 426                         err = sock_intr_errno(current_timeo);
 427                         break;
 428                 }
 429                 set_current_state(TASK_UNINTERRUPTIBLE);
 430                 release_sock(sk);
 431                 if (!sk_in_state(sk, states))
 432                         current_timeo = schedule_timeout(current_timeo);
 433                 __set_current_state(TASK_RUNNING);
 434                 lock_sock(sk);
 435         }
 436         remove_wait_queue(sk_sleep(sk), &wait);
 437
 438         if (rcu_dereference(sk->sk_wq) == &_sk_wq)
 439                 sk->sk_wq = NULL;
 440         return err;
 441 }
 442
 443 int chtls_disconnect(struct sock *sk, int flags)
 444 {
 445         struct tcp_sock *tp;
 446         int err;
 447
 448         tp = tcp_sk(sk);
 449         chtls_purge_recv_queue(sk);
 450         chtls_purge_receive_queue(sk);
 451         chtls_purge_write_queue(sk);
 452
 453         if (sk->sk_state != TCP_CLOSE) {
 454                 sk->sk_err = ECONNRESET;
 455                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 456                 err = wait_for_states(sk, TCPF_CLOSE);
 457                 if (err)
 458                         return err;
 459         }
 460         chtls_purge_recv_queue(sk);
 461         chtls_purge_receive_queue(sk);
 462         tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
 463         return tcp_disconnect(sk, flags);
 464 }
 465
 466 #define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
 467                                  TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
 468 void chtls_shutdown(struct sock *sk, int how)
 469 {
 470         if ((how & SEND_SHUTDOWN) &&
 471             sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
 472             make_close_transition(sk))
 473                 chtls_close_conn(sk);
 474 }
 475
 476 void chtls_destroy_sock(struct sock *sk)
 477 {
 478         struct chtls_sock *csk;
 479
 480         csk = rcu_dereference_sk_user_data(sk);
 481         chtls_purge_recv_queue(sk);
 482         csk->ulp_mode = ULP_MODE_NONE;
 483         chtls_purge_write_queue(sk);
 484         free_tls_keyid(sk);
 485         kref_put(&csk->kref, chtls_sock_release);
 486         if (sk->sk_family == AF_INET)
 487                 sk->sk_prot = &tcp_prot;
 488 #if IS_ENABLED(CONFIG_IPV6)
 489         else
 490                 sk->sk_prot = &tcpv6_prot;
 491 #endif
 492         sk->sk_prot->destroy(sk);
 493 }
 494
 495 static void reset_listen_child(struct sock *child)
 496 {
 497         struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
 498         struct sk_buff *skb;
 499
 500         skb = alloc_ctrl_skb(csk->txdata_skb_cache,
 501                              sizeof(struct cpl_abort_req));
 502
 503         chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
 504         sock_orphan(child);
 505         INC_ORPHAN_COUNT(child);
 506         if (child->sk_state == TCP_CLOSE)
 507                 inet_csk_destroy_sock(child);
 508 }
 509
 510 static void chtls_disconnect_acceptq(struct sock *listen_sk)
 511 {
 512         struct request_sock **pprev;
 513
 514         pprev = ACCEPT_QUEUE(listen_sk);
 515         while (*pprev) {
 516                 struct request_sock *req = *pprev;
 517
 518                 if (req->rsk_ops == &chtls_rsk_ops ||
 519                     req->rsk_ops == &chtls_rsk_opsv6) {
 520                         struct sock *child = req->sk;
 521
 522                         *pprev = req->dl_next;
 523                         sk_acceptq_removed(listen_sk);
 524                         reqsk_put(req);
 525                         sock_hold(child);
 526                         local_bh_disable();
 527                         bh_lock_sock(child);
 528                         release_tcp_port(child);
 529                         reset_listen_child(child);
 530                         bh_unlock_sock(child);
 531                         local_bh_enable();
 532                         sock_put(child);
 533                 } else {
 534                         pprev = &req->dl_next;
 535                 }
 536         }
 537 }
 538
 539 static int listen_hashfn(const struct sock *sk)
 540 {
 541         return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
 542 }
 543
 544 static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
 545                                            struct sock *sk,
 546                                            unsigned int stid)
 547 {
 548         struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
 549
 550         if (p) {
 551                 int key = listen_hashfn(sk);
 552
 553                 p->sk = sk;
 554                 p->stid = stid;
 555                 spin_lock(&cdev->listen_lock);
 556                 p->next = cdev->listen_hash_tab[key];
 557                 cdev->listen_hash_tab[key] = p;
 558                 spin_unlock(&cdev->listen_lock);
 559         }
 560         return p;
 561 }
 562
 563 static int listen_hash_find(struct chtls_dev *cdev,
 564                             struct sock *sk)
 565 {
 566         struct listen_info *p;
 567         int stid = -1;
 568         int key;
 569
 570         key = listen_hashfn(sk);
 571
 572         spin_lock(&cdev->listen_lock);
 573         for (p = cdev->listen_hash_tab[key]; p; p = p->next)
 574                 if (p->sk == sk) {
 575                         stid = p->stid;
 576                         break;
 577                 }
 578         spin_unlock(&cdev->listen_lock);
 579         return stid;
 580 }
 581
 582 static int listen_hash_del(struct chtls_dev *cdev,
 583                            struct sock *sk)
 584 {
 585         struct listen_info *p, **prev;
 586         int stid = -1;
 587         int key;
 588
 589         key = listen_hashfn(sk);
 590         prev = &cdev->listen_hash_tab[key];
 591
 592         spin_lock(&cdev->listen_lock);
 593         for (p = *prev; p; prev = &p->next, p = p->next)
 594                 if (p->sk == sk) {
 595                         stid = p->stid;
 596                         *prev = p->next;
 597                         kfree(p);
 598                         break;
 599                 }
 600         spin_unlock(&cdev->listen_lock);
 601         return stid;
 602 }
 603
 604 static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
 605 {
 606         struct request_sock *req;
 607         struct chtls_sock *csk;
 608
 609         csk = rcu_dereference_sk_user_data(child);
 610         req = csk->passive_reap_next;
 611
 612         reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
 613         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
 614         chtls_reqsk_free(req);
 615         csk->passive_reap_next = NULL;
 616 }
 617
 618 static void chtls_reset_synq(struct listen_ctx *listen_ctx)
 619 {
 620         struct sock *listen_sk = listen_ctx->lsk;
 621
 622         while (!skb_queue_empty(&listen_ctx->synq)) {
 623                 struct chtls_sock *csk =
 624                         container_of((struct synq *)__skb_dequeue
 625                                 (&listen_ctx->synq), struct chtls_sock, synq);
 626                 struct sock *child = csk->sk;
 627
 628                 cleanup_syn_rcv_conn(child, listen_sk);
 629                 sock_hold(child);
 630                 local_bh_disable();
 631                 bh_lock_sock(child);
 632                 release_tcp_port(child);
 633                 reset_listen_child(child);
 634                 bh_unlock_sock(child);
 635                 local_bh_enable();
 636                 sock_put(child);
 637         }
 638 }
 639
 640 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 641 {
 642         struct net_device *ndev;
 643 #if IS_ENABLED(CONFIG_IPV6)
 644         bool clip_valid = false;
 645 #endif
 646         struct listen_ctx *ctx;
 647         struct adapter *adap;
 648         struct port_info *pi;
 649         int ret = 0;
 650         int stid;
 651
 652         rcu_read_lock();
 653         ndev = chtls_find_netdev(cdev, sk);
 654         rcu_read_unlock();
 655         if (!ndev)
 656                 return -EBADF;
 657
 658         pi = netdev_priv(ndev);
 659         adap = pi->adapter;
 660         if (!(adap->flags & CXGB4_FULL_INIT_DONE))
 661                 return -EBADF;
 662
 663         if (listen_hash_find(cdev, sk) >= 0)   /* already have it */
 664                 return -EADDRINUSE;
 665
 666         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 667         if (!ctx)
 668                 return -ENOMEM;
 669
 670         __module_get(THIS_MODULE);
 671         ctx->lsk = sk;
 672         ctx->cdev = cdev;
 673         ctx->state = T4_LISTEN_START_PENDING;
 674         skb_queue_head_init(&ctx->synq);
 675
 676         stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
 677         if (stid < 0)
 678                 goto free_ctx;
 679
 680         sock_hold(sk);
 681         if (!listen_hash_add(cdev, sk, stid))
 682                 goto free_stid;
 683
 684         if (sk->sk_family == PF_INET) {
 685                 ret = cxgb4_create_server(ndev, stid,
 686                                           inet_sk(sk)->inet_rcv_saddr,
 687                                           inet_sk(sk)->inet_sport, 0,
 688                                           cdev->lldi->rxq_ids[0]);
 689 #if IS_ENABLED(CONFIG_IPV6)
 690         } else {
 691                 int addr_type;
 692
 693                 addr_type = ipv6_addr_type(&sk->sk_v6_rcv_saddr);
 694                 if (addr_type != IPV6_ADDR_ANY) {
 695                         ret = cxgb4_clip_get(ndev, (const u32 *)
 696                                              &sk->sk_v6_rcv_saddr, 1);
 697                         if (ret)
 698                                 goto del_hash;
 699                         clip_valid = true;
 700                 }
 701                 ret = cxgb4_create_server6(ndev, stid,
 702                                            &sk->sk_v6_rcv_saddr,
 703                                            inet_sk(sk)->inet_sport,
 704                                            cdev->lldi->rxq_ids[0]);
 705 #endif
 706         }
 707         if (ret > 0)
 708                 ret = net_xmit_errno(ret);
 709         if (ret)
 710                 goto del_hash;
 711         return 0;
 712 del_hash:
 713 #if IS_ENABLED(CONFIG_IPV6)
 714         if (clip_valid)
 715                 cxgb4_clip_release(ndev, (const u32 *)&sk->sk_v6_rcv_saddr, 1);
 716 #endif
 717         listen_hash_del(cdev, sk);
 718 free_stid:
 719         cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
 720         sock_put(sk);
 721 free_ctx:
 722         kfree(ctx);
 723         module_put(THIS_MODULE);
 724         return -EBADF;
 725 }
 726
 727 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 728 {
 729         struct listen_ctx *listen_ctx;
 730         int stid;
 731
 732         stid = listen_hash_del(cdev, sk);
 733         if (stid < 0)
 734                 return;
 735
 736         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 737         chtls_reset_synq(listen_ctx);
 738
 739         cxgb4_remove_server(cdev->lldi->ports[0], stid,
 740                             cdev->lldi->rxq_ids[0], sk->sk_family == PF_INET6);
 741
 742 #if IS_ENABLED(CONFIG_IPV6)
 743         if (sk->sk_family == PF_INET6) {
 744                 struct net_device *ndev = chtls_find_netdev(cdev, sk);
 745                 int addr_type = 0;
 746
 747                 addr_type = ipv6_addr_type((const struct in6_addr *)
 748                                           &sk->sk_v6_rcv_saddr);
 749                 if (addr_type != IPV6_ADDR_ANY)
 750                         cxgb4_clip_release(ndev, (const u32 *)
 751                                            &sk->sk_v6_rcv_saddr, 1);
 752         }
 753 #endif
 754         chtls_disconnect_acceptq(sk);
 755 }
 756
 757 static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 758 {
 759         struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
 760         unsigned int stid = GET_TID(rpl);
 761         struct listen_ctx *listen_ctx;
 762
 763         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 764         if (!listen_ctx)
 765                 return CPL_RET_BUF_DONE;
 766
 767         if (listen_ctx->state == T4_LISTEN_START_PENDING) {
 768                 listen_ctx->state = T4_LISTEN_STARTED;
 769                 return CPL_RET_BUF_DONE;
 770         }
 771
 772         if (rpl->status != CPL_ERR_NONE) {
 773                 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
 774                         rpl->status, stid);
 775         } else {
 776                 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 777                 sock_put(listen_ctx->lsk);
 778                 kfree(listen_ctx);
 779                 module_put(THIS_MODULE);
 780         }
 781         return CPL_RET_BUF_DONE;
 782 }
 783
 784 static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 785 {
 786         struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
 787         struct listen_ctx *listen_ctx;
 788         unsigned int stid;
 789         void *data;
 790
 791         stid = GET_TID(rpl);
 792         data = lookup_stid(cdev->tids, stid);
 793         listen_ctx = (struct listen_ctx *)data;
 794
 795         if (rpl->status != CPL_ERR_NONE) {
 796                 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
 797                         rpl->status, stid);
 798         } else {
 799                 cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 800                 sock_put(listen_ctx->lsk);
 801                 kfree(listen_ctx);
 802                 module_put(THIS_MODULE);
 803         }
 804         return CPL_RET_BUF_DONE;
 805 }
 806
 807 static void chtls_purge_wr_queue(struct sock *sk)
 808 {
 809         struct sk_buff *skb;
 810
 811         while ((skb = dequeue_wr(sk)) != NULL)
 812                 kfree_skb(skb);
 813 }
 814
 815 static void chtls_release_resources(struct sock *sk)
 816 {
 817         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 818         struct chtls_dev *cdev = csk->cdev;
 819         unsigned int tid = csk->tid;
 820         struct tid_info *tids;
 821
 822         if (!cdev)
 823                 return;
 824
 825         tids = cdev->tids;
 826         kfree_skb(csk->txdata_skb_cache);
 827         csk->txdata_skb_cache = NULL;
 828
 829         if (csk->wr_credits != csk->wr_max_credits) {
 830                 chtls_purge_wr_queue(sk);
 831                 chtls_reset_wr_list(csk);
 832         }
 833
 834         if (csk->l2t_entry) {
 835                 cxgb4_l2t_release(csk->l2t_entry);
 836                 csk->l2t_entry = NULL;
 837         }
 838
 839         if (sk->sk_state != TCP_SYN_SENT) {
 840                 cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
 841                 sock_put(sk);
 842         }
 843 }
 844
 845 static void chtls_conn_done(struct sock *sk)
 846 {
 847         if (sock_flag(sk, SOCK_DEAD))
 848                 chtls_purge_receive_queue(sk);
 849         sk_wakeup_sleepers(sk, 0);
 850         tcp_done(sk);
 851 }
 852
 853 static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
 854 {
 855         /*
 856          * If the server is still open we clean up the child connection,
 857          * otherwise the server already did the clean up as it was purging
 858          * its SYN queue and the skb was just sitting in its backlog.
 859          */
 860         if (likely(parent->sk_state == TCP_LISTEN)) {
 861                 cleanup_syn_rcv_conn(child, parent);
 862                 /* Without the below call to sock_orphan,
 863                  * we leak the socket resource with syn_flood test
 864                  * as inet_csk_destroy_sock will not be called
 865                  * in tcp_done since SOCK_DEAD flag is not set.
 866                  * Kernel handles this differently where new socket is
 867                  * created only after 3 way handshake is done.
 868                  */
 869                 sock_orphan(child);
 870                 percpu_counter_inc((child)->sk_prot->orphan_count);
 871                 chtls_release_resources(child);
 872                 chtls_conn_done(child);
 873         } else {
 874                 if (csk_flag(child, CSK_RST_ABORTED)) {
 875                         chtls_release_resources(child);
 876                         chtls_conn_done(child);
 877                 }
 878         }
 879 }
 880
 881 static void pass_open_abort(struct sock *child, struct sock *parent,
 882                             struct sk_buff *skb)
 883 {
 884         do_abort_syn_rcv(child, parent);
 885         kfree_skb(skb);
 886 }
 887
 888 static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
 889 {
 890         pass_open_abort(skb->sk, lsk, skb);
 891 }
 892
 893 static void chtls_pass_open_arp_failure(struct sock *sk,
 894                                         struct sk_buff *skb)
 895 {
 896         const struct request_sock *oreq;
 897         struct chtls_sock *csk;
 898         struct chtls_dev *cdev;
 899         struct sock *parent;
 900         void *data;
 901
 902         csk = rcu_dereference_sk_user_data(sk);
 903         cdev = csk->cdev;
 904
 905         /*
 906          * If the connection is being aborted due to the parent listening
 907          * socket going away there's nothing to do, the ABORT_REQ will close
 908          * the connection.
 909          */
 910         if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
 911                 kfree_skb(skb);
 912                 return;
 913         }
 914
 915         oreq = csk->passive_reap_next;
 916         data = lookup_stid(cdev->tids, oreq->ts_recent);
 917         parent = ((struct listen_ctx *)data)->lsk;
 918
 919         bh_lock_sock(parent);
 920         if (!sock_owned_by_user(parent)) {
 921                 pass_open_abort(sk, parent, skb);
 922         } else {
 923                 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
 924                 __sk_add_backlog(parent, skb);
 925         }
 926         bh_unlock_sock(parent);
 927 }
 928
 929 static void chtls_accept_rpl_arp_failure(void *handle,
 930                                          struct sk_buff *skb)
 931 {
 932         struct sock *sk = (struct sock *)handle;
 933
 934         sock_hold(sk);
 935         process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
 936         sock_put(sk);
 937 }
 938
 939 static unsigned int chtls_select_mss(const struct chtls_sock *csk,
 940                                      unsigned int pmtu,
 941                                      struct cpl_pass_accept_req *req)
 942 {
 943         struct chtls_dev *cdev;
 944         struct dst_entry *dst;
 945         unsigned int tcpoptsz;
 946         unsigned int iphdrsz;
 947         unsigned int mtu_idx;
 948         struct tcp_sock *tp;
 949         unsigned int mss;
 950         struct sock *sk;
 951
 952         mss = ntohs(req->tcpopt.mss);
 953         sk = csk->sk;
 954         dst = __sk_dst_get(sk);
 955         cdev = csk->cdev;
 956         tp = tcp_sk(sk);
 957         tcpoptsz = 0;
 958
 959 #if IS_ENABLED(CONFIG_IPV6)
 960         if (sk->sk_family == AF_INET6)
 961                 iphdrsz = sizeof(struct ipv6hdr) + sizeof(struct tcphdr);
 962         else
 963 #endif
 964                 iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
 965         if (req->tcpopt.tstamp)
 966                 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
 967
 968         tp->advmss = dst_metric_advmss(dst);
 969         if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
 970                 tp->advmss = USER_MSS(tp);
 971         if (tp->advmss > pmtu - iphdrsz)
 972                 tp->advmss = pmtu - iphdrsz;
 973         if (mss && tp->advmss > mss)
 974                 tp->advmss = mss;
 975
 976         tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
 977                                             iphdrsz + tcpoptsz,
 978                                             tp->advmss - tcpoptsz,
 979                                             8, &mtu_idx);
 980         tp->advmss -= iphdrsz;
 981
 982         inet_csk(sk)->icsk_pmtu_cookie = pmtu;
 983         return mtu_idx;
 984 }
 985
 986 static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
 987 {
 988         int wscale = 0;
 989
 990         if (space > MAX_RCV_WND)
 991                 space = MAX_RCV_WND;
 992         if (win_clamp && win_clamp < space)
 993                 space = win_clamp;
 994
 995         if (wscale_ok) {
 996                 while (wscale < 14 && (65535 << wscale) < space)
 997                         wscale++;
 998         }
 999         return wscale;
1000 }
1001
1002 static void chtls_pass_accept_rpl(struct sk_buff *skb,
1003                                   struct cpl_pass_accept_req *req,
1004                                   unsigned int tid)
1005
1006 {
1007         struct cpl_t5_pass_accept_rpl *rpl5;
1008         struct cxgb4_lld_info *lldi;
1009         const struct tcphdr *tcph;
1010         const struct tcp_sock *tp;
1011         struct chtls_sock *csk;
1012         unsigned int len;
1013         struct sock *sk;
1014         u32 opt2, hlen;
1015         u64 opt0;
1016
1017         sk = skb->sk;
1018         tp = tcp_sk(sk);
1019         csk = sk->sk_user_data;
1020         csk->tid = tid;
1021         lldi = csk->cdev->lldi;
1022         len = roundup(sizeof(*rpl5), 16);
1023
1024         rpl5 = __skb_put_zero(skb, len);
1025         INIT_TP_WR(rpl5, tid);
1026
1027         OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
1028                                                      csk->tid));
1029         csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
1030                                         req);
1031         opt0 = TCAM_BYPASS_F |
1032                WND_SCALE_V(RCV_WSCALE(tp)) |
1033                MSS_IDX_V(csk->mtu_idx) |
1034                L2T_IDX_V(csk->l2t_entry->idx) |
1035                NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
1036                TX_CHAN_V(csk->tx_chan) |
1037                SMAC_SEL_V(csk->smac_idx) |
1038                DSCP_V(csk->tos >> 2) |
1039                ULP_MODE_V(ULP_MODE_TLS) |
1040                RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
1041
1042         opt2 = RX_CHANNEL_V(0) |
1043                 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
1044
1045         if (!is_t5(lldi->adapter_type))
1046                 opt2 |= RX_FC_DISABLE_F;
1047         if (req->tcpopt.tstamp)
1048                 opt2 |= TSTAMPS_EN_F;
1049         if (req->tcpopt.sack)
1050                 opt2 |= SACK_EN_F;
1051         hlen = ntohl(req->hdr_len);
1052
1053         tcph = (struct tcphdr *)((u8 *)(req + 1) +
1054                         T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
1055         if (tcph->ece && tcph->cwr)
1056                 opt2 |= CCTRL_ECN_V(1);
1057         opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
1058         opt2 |= T5_ISS_F;
1059         opt2 |= T5_OPT_2_VALID_F;
1060         opt2 |= WND_SCALE_EN_V(WSCALE_OK(tp));
1061         rpl5->opt0 = cpu_to_be64(opt0);
1062         rpl5->opt2 = cpu_to_be32(opt2);
1063         rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
1064         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
1065         t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
1066         cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
1067 }
1068
1069 static void inet_inherit_port(struct inet_hashinfo *hash_info,
1070                               struct sock *lsk, struct sock *newsk)
1071 {
1072         local_bh_disable();
1073         __inet_inherit_port(lsk, newsk);
1074         local_bh_enable();
1075 }
1076
1077 static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
1078 {
1079         if (skb->protocol) {
1080                 kfree_skb(skb);
1081                 return 0;
1082         }
1083         BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
1084         return 0;
1085 }
1086
1087 static void chtls_set_tcp_window(struct chtls_sock *csk)
1088 {
1089         struct net_device *ndev = csk->egress_dev;
1090         struct port_info *pi = netdev_priv(ndev);
1091         unsigned int linkspeed;
1092         u8 scale;
1093
1094         linkspeed = pi->link_cfg.speed;
1095         scale = linkspeed / SPEED_10000;
1096 #define CHTLS_10G_RCVWIN (256 * 1024)
1097         csk->rcv_win = CHTLS_10G_RCVWIN;
1098         if (scale)
1099                 csk->rcv_win *= scale;
1100 #define CHTLS_10G_SNDWIN (256 * 1024)
1101         csk->snd_win = CHTLS_10G_SNDWIN;
1102         if (scale)
1103                 csk->snd_win *= scale;
1104 }
1105
1106 static struct sock *chtls_recv_sock(struct sock *lsk,
1107                                     struct request_sock *oreq,
1108                                     void *network_hdr,
1109                                     const struct cpl_pass_accept_req *req,
1110                                     struct chtls_dev *cdev)
1111 {
1112         struct neighbour *n = NULL;
1113         struct inet_sock *newinet;
1114         const struct iphdr *iph;
1115         struct tls_context *ctx;
1116         struct net_device *ndev;
1117         struct chtls_sock *csk;
1118         struct dst_entry *dst;
1119         struct tcp_sock *tp;
1120         struct sock *newsk;
1121         u16 port_id;
1122         int rxq_idx;
1123         int step;
1124
1125         iph = (const struct iphdr *)network_hdr;
1126         newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1127         if (!newsk)
1128                 goto free_oreq;
1129
1130         if (lsk->sk_family == AF_INET) {
1131                 dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1132                 if (!dst)
1133                         goto free_sk;
1134
1135                 n = dst_neigh_lookup(dst, &iph->saddr);
1136 #if IS_ENABLED(CONFIG_IPV6)
1137         } else {
1138                 const struct ipv6hdr *ip6h;
1139                 struct flowi6 fl6;
1140
1141                 ip6h = (const struct ipv6hdr *)network_hdr;
1142                 memset(&fl6, 0, sizeof(fl6));
1143                 fl6.flowi6_proto = IPPROTO_TCP;
1144                 fl6.saddr = ip6h->daddr;
1145                 fl6.daddr = ip6h->saddr;
1146                 fl6.fl6_dport = inet_rsk(oreq)->ir_rmt_port;
1147                 fl6.fl6_sport = htons(inet_rsk(oreq)->ir_num);
1148                 security_req_classify_flow(oreq, flowi6_to_flowi(&fl6));
1149                 dst = ip6_dst_lookup_flow(sock_net(lsk), lsk, &fl6, NULL);
1150                 if (IS_ERR(dst))
1151                         goto free_sk;
1152                 n = dst_neigh_lookup(dst, &ip6h->saddr);
1153 #endif
1154         }
1155         if (!n)
1156                 goto free_sk;
1157
1158         ndev = n->dev;
1159         if (!ndev)
1160                 goto free_dst;
1161         if (is_vlan_dev(ndev))
1162                 ndev = vlan_dev_real_dev(ndev);
1163
1164         port_id = cxgb4_port_idx(ndev);
1165
1166         csk = chtls_sock_create(cdev);
1167         if (!csk)
1168                 goto free_dst;
1169
1170         csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1171         if (!csk->l2t_entry)
1172                 goto free_csk;
1173
1174         newsk->sk_user_data = csk;
1175         newsk->sk_backlog_rcv = chtls_backlog_rcv;
1176
1177         tp = tcp_sk(newsk);
1178         newinet = inet_sk(newsk);
1179
1180         if (iph->version == 0x4) {
1181                 newinet->inet_daddr = iph->saddr;
1182                 newinet->inet_rcv_saddr = iph->daddr;
1183                 newinet->inet_saddr = iph->daddr;
1184 #if IS_ENABLED(CONFIG_IPV6)
1185         } else {
1186                 struct tcp6_sock *newtcp6sk = (struct tcp6_sock *)newsk;
1187                 struct inet_request_sock *treq = inet_rsk(oreq);
1188                 struct ipv6_pinfo *newnp = inet6_sk(newsk);
1189                 struct ipv6_pinfo *np = inet6_sk(lsk);
1190
1191                 inet_sk(newsk)->pinet6 = &newtcp6sk->inet6;
1192                 memcpy(newnp, np, sizeof(struct ipv6_pinfo));
1193                 newsk->sk_v6_daddr = treq->ir_v6_rmt_addr;
1194                 newsk->sk_v6_rcv_saddr = treq->ir_v6_loc_addr;
1195                 inet6_sk(newsk)->saddr = treq->ir_v6_loc_addr;
1196                 newnp->ipv6_fl_list = NULL;
1197                 newnp->pktoptions = NULL;
1198                 newsk->sk_bound_dev_if = treq->ir_iif;
1199                 newinet->inet_opt = NULL;
1200                 newinet->inet_daddr = LOOPBACK4_IPV6;
1201                 newinet->inet_saddr = LOOPBACK4_IPV6;
1202 #endif
1203         }
1204
1205         oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1206         sk_setup_caps(newsk, dst);
1207         ctx = tls_get_ctx(lsk);
1208         newsk->sk_destruct = ctx->sk_destruct;
1209         csk->sk = newsk;
1210         csk->passive_reap_next = oreq;
1211         csk->tx_chan = cxgb4_port_chan(ndev);
1212         csk->port_id = port_id;
1213         csk->egress_dev = ndev;
1214         csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1215         chtls_set_tcp_window(csk);
1216         tp->rcv_wnd = csk->rcv_win;
1217         csk->sndbuf = csk->snd_win;
1218         csk->ulp_mode = ULP_MODE_TLS;
1219         step = cdev->lldi->nrxq / cdev->lldi->nchan;
1220         csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
1221         rxq_idx = port_id * step;
1222         csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1223                         port_id * step;
1224         csk->sndbuf = newsk->sk_sndbuf;
1225         csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
1226         RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
1227                                            sock_net(newsk)->
1228                                                 ipv4.sysctl_tcp_window_scaling,
1229                                            tp->window_clamp);
1230         neigh_release(n);
1231         inet_inherit_port(&tcp_hashinfo, lsk, newsk);
1232         csk_set_flag(csk, CSK_CONN_INLINE);
1233         bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1234
1235         return newsk;
1236 free_csk:
1237         chtls_sock_release(&csk->kref);
1238 free_dst:
1239         dst_release(dst);
1240 free_sk:
1241         inet_csk_prepare_forced_close(newsk);
1242         tcp_done(newsk);
1243 free_oreq:
1244         chtls_reqsk_free(oreq);
1245         return NULL;
1246 }
1247
1248 /*
1249  * Populate a TID_RELEASE WR.  The skb must be already propely sized.
1250  */
1251 static  void mk_tid_release(struct sk_buff *skb,
1252                             unsigned int chan, unsigned int tid)
1253 {
1254         struct cpl_tid_release *req;
1255         unsigned int len;
1256
1257         len = roundup(sizeof(struct cpl_tid_release), 16);
1258         req = (struct cpl_tid_release *)__skb_put(skb, len);
1259         memset(req, 0, len);
1260         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1261         INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1262 }
1263
1264 static int chtls_get_module(struct sock *sk)
1265 {
1266         struct inet_connection_sock *icsk = inet_csk(sk);
1267
1268         if (!try_module_get(icsk->icsk_ulp_ops->owner))
1269                 return -1;
1270
1271         return 0;
1272 }
1273
1274 static void chtls_pass_accept_request(struct sock *sk,
1275                                       struct sk_buff *skb)
1276 {
1277         struct cpl_t5_pass_accept_rpl *rpl;
1278         struct cpl_pass_accept_req *req;
1279         struct listen_ctx *listen_ctx;
1280         struct vlan_ethhdr *vlan_eh;
1281         struct request_sock *oreq;
1282         struct sk_buff *reply_skb;
1283         struct chtls_sock *csk;
1284         struct chtls_dev *cdev;
1285         struct ipv6hdr *ip6h;
1286         struct tcphdr *tcph;
1287         struct sock *newsk;
1288         struct ethhdr *eh;
1289         struct iphdr *iph;
1290         void *network_hdr;
1291         unsigned int stid;
1292         unsigned int len;
1293         unsigned int tid;
1294         bool th_ecn, ect;
1295         __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1296         u16 eth_hdr_len;
1297         bool ecn_ok;
1298
1299         req = cplhdr(skb) + RSS_HDR;
1300         tid = GET_TID(req);
1301         cdev = BLOG_SKB_CB(skb)->cdev;
1302         newsk = lookup_tid(cdev->tids, tid);
1303         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1304         if (newsk) {
1305                 pr_info("tid (%d) already in use\n", tid);
1306                 return;
1307         }
1308
1309         len = roundup(sizeof(*rpl), 16);
1310         reply_skb = alloc_skb(len, GFP_ATOMIC);
1311         if (!reply_skb) {
1312                 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1313                 kfree_skb(skb);
1314                 return;
1315         }
1316
1317         if (sk->sk_state != TCP_LISTEN)
1318                 goto reject;
1319
1320         if (inet_csk_reqsk_queue_is_full(sk))
1321                 goto reject;
1322
1323         if (sk_acceptq_is_full(sk))
1324                 goto reject;
1325
1326
1327         eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1328         if (eth_hdr_len == ETH_HLEN) {
1329                 eh = (struct ethhdr *)(req + 1);
1330                 iph = (struct iphdr *)(eh + 1);
1331                 ip6h = (struct ipv6hdr *)(eh + 1);
1332                 network_hdr = (void *)(eh + 1);
1333         } else {
1334                 vlan_eh = (struct vlan_ethhdr *)(req + 1);
1335                 iph = (struct iphdr *)(vlan_eh + 1);
1336                 ip6h = (struct ipv6hdr *)(vlan_eh + 1);
1337                 network_hdr = (void *)(vlan_eh + 1);
1338         }
1339
1340         if (iph->version == 0x4) {
1341                 tcph = (struct tcphdr *)(iph + 1);
1342                 skb_set_network_header(skb, (void *)iph - (void *)req);
1343                 oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1344         } else {
1345                 tcph = (struct tcphdr *)(ip6h + 1);
1346                 skb_set_network_header(skb, (void *)ip6h - (void *)req);
1347                 oreq = inet_reqsk_alloc(&chtls_rsk_opsv6, sk, false);
1348         }
1349
1350         if (!oreq)
1351                 goto reject;
1352
1353         oreq->rsk_rcv_wnd = 0;
1354         oreq->rsk_window_clamp = 0;
1355         oreq->syncookie = 0;
1356         oreq->mss = 0;
1357         oreq->ts_recent = 0;
1358
1359         tcp_rsk(oreq)->tfo_listener = false;
1360         tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1361         chtls_set_req_port(oreq, tcph->source, tcph->dest);
1362         if (iph->version == 0x4) {
1363                 chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1364                 ip_dsfield = ipv4_get_dsfield(iph);
1365 #if IS_ENABLED(CONFIG_IPV6)
1366         } else {
1367                 inet_rsk(oreq)->ir_v6_rmt_addr = ipv6_hdr(skb)->saddr;
1368                 inet_rsk(oreq)->ir_v6_loc_addr = ipv6_hdr(skb)->daddr;
1369                 ip_dsfield = ipv6_get_dsfield(ipv6_hdr(skb));
1370 #endif
1371         }
1372         if (req->tcpopt.wsf <= 14 &&
1373             sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
1374                 inet_rsk(oreq)->wscale_ok = 1;
1375                 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1376         }
1377         inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
1378         th_ecn = tcph->ece && tcph->cwr;
1379         if (th_ecn) {
1380                 ect = !INET_ECN_is_not_ect(ip_dsfield);
1381                 ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn;
1382                 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1383                         inet_rsk(oreq)->ecn_ok = 1;
1384         }
1385
1386         newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1387         if (!newsk)
1388                 goto free_oreq;
1389
1390         if (chtls_get_module(newsk))
1391                 goto reject;
1392         inet_csk_reqsk_queue_added(sk);
1393         reply_skb->sk = newsk;
1394         chtls_install_cpl_ops(newsk);
1395         cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1396         csk = rcu_dereference_sk_user_data(newsk);
1397         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1398         csk->listen_ctx = listen_ctx;
1399         __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1400         chtls_pass_accept_rpl(reply_skb, req, tid);
1401         kfree_skb(skb);
1402         return;
1403
1404 free_oreq:
1405         chtls_reqsk_free(oreq);
1406 reject:
1407         mk_tid_release(reply_skb, 0, tid);
1408         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1409         kfree_skb(skb);
1410 }
1411
1412 /*
1413  * Handle a CPL_PASS_ACCEPT_REQ message.
1414  */
1415 static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1416 {
1417         struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1418         struct listen_ctx *ctx;
1419         unsigned int stid;
1420         unsigned int tid;
1421         struct sock *lsk;
1422         void *data;
1423
1424         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1425         tid = GET_TID(req);
1426
1427         data = lookup_stid(cdev->tids, stid);
1428         if (!data)
1429                 return 1;
1430
1431         ctx = (struct listen_ctx *)data;
1432         lsk = ctx->lsk;
1433
1434         if (unlikely(tid_out_of_range(cdev->tids, tid))) {
1435                 pr_info("passive open TID %u too large\n", tid);
1436                 return 1;
1437         }
1438
1439         BLOG_SKB_CB(skb)->cdev = cdev;
1440         process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1441         return 0;
1442 }
1443
1444 /*
1445  * Completes some final bits of initialization for just established connections
1446  * and changes their state to TCP_ESTABLISHED.
1447  *
1448  * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1449  */
1450 static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1451 {
1452         struct tcp_sock *tp = tcp_sk(sk);
1453
1454         tp->pushed_seq = snd_isn;
1455         tp->write_seq = snd_isn;
1456         tp->snd_nxt = snd_isn;
1457         tp->snd_una = snd_isn;
1458         inet_sk(sk)->inet_id = prandom_u32();
1459         assign_rxopt(sk, opt);
1460
1461         if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1462                 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1463
1464         smp_mb();
1465         tcp_set_state(sk, TCP_ESTABLISHED);
1466 }
1467
1468 static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1469 {
1470         struct sk_buff *abort_skb;
1471
1472         abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1473         if (abort_skb)
1474                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1475 }
1476
1477 static struct sock *reap_list;
1478 static DEFINE_SPINLOCK(reap_list_lock);
1479
1480 /*
1481  * Process the reap list.
1482  */
1483 DECLARE_TASK_FUNC(process_reap_list, task_param)
1484 {
1485         spin_lock_bh(&reap_list_lock);
1486         while (reap_list) {
1487                 struct sock *sk = reap_list;
1488                 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1489
1490                 reap_list = csk->passive_reap_next;
1491                 csk->passive_reap_next = NULL;
1492                 spin_unlock(&reap_list_lock);
1493                 sock_hold(sk);
1494
1495                 bh_lock_sock(sk);
1496                 chtls_abort_conn(sk, NULL);
1497                 sock_orphan(sk);
1498                 if (sk->sk_state == TCP_CLOSE)
1499                         inet_csk_destroy_sock(sk);
1500                 bh_unlock_sock(sk);
1501                 sock_put(sk);
1502                 spin_lock(&reap_list_lock);
1503         }
1504         spin_unlock_bh(&reap_list_lock);
1505 }
1506
1507 static DECLARE_WORK(reap_task, process_reap_list);
1508
1509 static void add_to_reap_list(struct sock *sk)
1510 {
1511         struct chtls_sock *csk = sk->sk_user_data;
1512
1513         local_bh_disable();
1514         release_tcp_port(sk); /* release the port immediately */
1515
1516         spin_lock(&reap_list_lock);
1517         csk->passive_reap_next = reap_list;
1518         reap_list = sk;
1519         if (!csk->passive_reap_next)
1520                 schedule_work(&reap_task);
1521         spin_unlock(&reap_list_lock);
1522         local_bh_enable();
1523 }
1524
1525 static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1526                                     struct chtls_dev *cdev)
1527 {
1528         struct request_sock *oreq;
1529         struct chtls_sock *csk;
1530
1531         if (lsk->sk_state != TCP_LISTEN)
1532                 return;
1533
1534         csk = child->sk_user_data;
1535         oreq = csk->passive_reap_next;
1536         csk->passive_reap_next = NULL;
1537
1538         reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1539         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1540
1541         if (sk_acceptq_is_full(lsk)) {
1542                 chtls_reqsk_free(oreq);
1543                 add_to_reap_list(child);
1544         } else {
1545                 refcount_set(&oreq->rsk_refcnt, 1);
1546                 inet_csk_reqsk_queue_add(lsk, oreq, child);
1547                 lsk->sk_data_ready(lsk);
1548         }
1549 }
1550
1551 static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1552 {
1553         struct sock *child = skb->sk;
1554
1555         skb->sk = NULL;
1556         add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1557         kfree_skb(skb);
1558 }
1559
1560 static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1561 {
1562         struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1563         struct chtls_sock *csk;
1564         struct sock *lsk, *sk;
1565         unsigned int hwtid;
1566
1567         hwtid = GET_TID(req);
1568         sk = lookup_tid(cdev->tids, hwtid);
1569         if (!sk)
1570                 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1571
1572         bh_lock_sock(sk);
1573         if (unlikely(sock_owned_by_user(sk))) {
1574                 kfree_skb(skb);
1575         } else {
1576                 unsigned int stid;
1577                 void *data;
1578
1579                 csk = sk->sk_user_data;
1580                 csk->wr_max_credits = 64;
1581                 csk->wr_credits = 64;
1582                 csk->wr_unacked = 0;
1583                 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1584                 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1585                 sk->sk_state_change(sk);
1586                 if (unlikely(sk->sk_socket))
1587                         sk_wake_async(sk, 0, POLL_OUT);
1588
1589                 data = lookup_stid(cdev->tids, stid);
1590                 lsk = ((struct listen_ctx *)data)->lsk;
1591
1592                 bh_lock_sock(lsk);
1593                 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1594                         /* removed from synq */
1595                         bh_unlock_sock(lsk);
1596                         kfree_skb(skb);
1597                         goto unlock;
1598                 }
1599
1600                 if (likely(!sock_owned_by_user(lsk))) {
1601                         kfree_skb(skb);
1602                         add_pass_open_to_parent(sk, lsk, cdev);
1603                 } else {
1604                         skb->sk = sk;
1605                         BLOG_SKB_CB(skb)->cdev = cdev;
1606                         BLOG_SKB_CB(skb)->backlog_rcv =
1607                                 bl_add_pass_open_to_parent;
1608                         __sk_add_backlog(lsk, skb);
1609                 }
1610                 bh_unlock_sock(lsk);
1611         }
1612 unlock:
1613         bh_unlock_sock(sk);
1614         return 0;
1615 }
1616
1617 /*
1618  * Handle receipt of an urgent pointer.
1619  */
1620 static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1621 {
1622         struct tcp_sock *tp = tcp_sk(sk);
1623
1624         urg_seq--;
1625         if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1626                 return; /* duplicate pointer */
1627
1628         sk_send_sigurg(sk);
1629         if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1630             !sock_flag(sk, SOCK_URGINLINE) &&
1631             tp->copied_seq != tp->rcv_nxt) {
1632                 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1633
1634                 tp->copied_seq++;
1635                 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1636                         chtls_free_skb(sk, skb);
1637         }
1638
1639         tp->urg_data = TCP_URG_NOTYET;
1640         tp->urg_seq = urg_seq;
1641 }
1642
1643 static void check_sk_callbacks(struct chtls_sock *csk)
1644 {
1645         struct sock *sk = csk->sk;
1646
1647         if (unlikely(sk->sk_user_data &&
1648                      !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1649                 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1650 }
1651
1652 /*
1653  * Handles Rx data that arrives in a state where the socket isn't accepting
1654  * new data.
1655  */
1656 static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1657 {
1658         if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1659                 chtls_abort_conn(sk, skb);
1660
1661         kfree_skb(skb);
1662 }
1663
1664 static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1665 {
1666         struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1667         struct chtls_sock *csk;
1668         struct tcp_sock *tp;
1669
1670         csk = rcu_dereference_sk_user_data(sk);
1671         tp = tcp_sk(sk);
1672
1673         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1674                 handle_excess_rx(sk, skb);
1675                 return;
1676         }
1677
1678         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1679         ULP_SKB_CB(skb)->psh = hdr->psh;
1680         skb_ulp_mode(skb) = ULP_MODE_NONE;
1681
1682         skb_reset_transport_header(skb);
1683         __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1684         if (!skb->data_len)
1685                 __skb_trim(skb, ntohs(hdr->len));
1686
1687         if (unlikely(hdr->urg))
1688                 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1689         if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1690                      tp->urg_seq - tp->rcv_nxt < skb->len))
1691                 tp->urg_data = TCP_URG_VALID |
1692                                skb->data[tp->urg_seq - tp->rcv_nxt];
1693
1694         if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1695                 csk->delack_mode = hdr->dack_mode;
1696                 csk->delack_seq = tp->rcv_nxt;
1697         }
1698
1699         tcp_hdr(skb)->fin = 0;
1700         tp->rcv_nxt += skb->len;
1701
1702         __skb_queue_tail(&sk->sk_receive_queue, skb);
1703
1704         if (!sock_flag(sk, SOCK_DEAD)) {
1705                 check_sk_callbacks(csk);
1706                 sk->sk_data_ready(sk);
1707         }
1708 }
1709
1710 static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1711 {
1712         struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1713         unsigned int hwtid = GET_TID(req);
1714         struct sock *sk;
1715
1716         sk = lookup_tid(cdev->tids, hwtid);
1717         if (unlikely(!sk)) {
1718                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1719                 return -EINVAL;
1720         }
1721         skb_dst_set(skb, NULL);
1722         process_cpl_msg(chtls_recv_data, sk, skb);
1723         return 0;
1724 }
1725
1726 static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1727 {
1728         struct cpl_tls_data *hdr = cplhdr(skb);
1729         struct chtls_sock *csk;
1730         struct chtls_hws *tlsk;
1731         struct tcp_sock *tp;
1732
1733         csk = rcu_dereference_sk_user_data(sk);
1734         tlsk = &csk->tlshws;
1735         tp = tcp_sk(sk);
1736
1737         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1738                 handle_excess_rx(sk, skb);
1739                 return;
1740         }
1741
1742         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1743         ULP_SKB_CB(skb)->flags = 0;
1744         skb_ulp_mode(skb) = ULP_MODE_TLS;
1745
1746         skb_reset_transport_header(skb);
1747         __skb_pull(skb, sizeof(*hdr));
1748         if (!skb->data_len)
1749                 __skb_trim(skb,
1750                            CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1751
1752         if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1753                      tp->rcv_nxt < skb->len))
1754                 tp->urg_data = TCP_URG_VALID |
1755                                skb->data[tp->urg_seq - tp->rcv_nxt];
1756
1757         tcp_hdr(skb)->fin = 0;
1758         tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1759         __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1760 }
1761
1762 static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1763 {
1764         struct cpl_tls_data *req = cplhdr(skb);
1765         unsigned int hwtid = GET_TID(req);
1766         struct sock *sk;
1767
1768         sk = lookup_tid(cdev->tids, hwtid);
1769         if (unlikely(!sk)) {
1770                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1771                 return -EINVAL;
1772         }
1773         skb_dst_set(skb, NULL);
1774         process_cpl_msg(chtls_recv_pdu, sk, skb);
1775         return 0;
1776 }
1777
1778 static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1779 {
1780         struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1781
1782         skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1783         tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1784 }
1785
1786 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1787 {
1788         struct tlsrx_cmp_hdr *tls_hdr_pkt;
1789         struct cpl_rx_tls_cmp *cmp_cpl;
1790         struct sk_buff *skb_rec;
1791         struct chtls_sock *csk;
1792         struct chtls_hws *tlsk;
1793         struct tcp_sock *tp;
1794
1795         cmp_cpl = cplhdr(skb);
1796         csk = rcu_dereference_sk_user_data(sk);
1797         tlsk = &csk->tlshws;
1798         tp = tcp_sk(sk);
1799
1800         ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1801         ULP_SKB_CB(skb)->flags = 0;
1802
1803         skb_reset_transport_header(skb);
1804         __skb_pull(skb, sizeof(*cmp_cpl));
1805         tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1806         if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1807                 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1808         if (!skb->data_len)
1809                 __skb_trim(skb, TLS_HEADER_LENGTH);
1810
1811         tp->rcv_nxt +=
1812                 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1813
1814         ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
1815         skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1816         if (!skb_rec) {
1817                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1818         } else {
1819                 chtls_set_hdrlen(skb, tlsk->pldlen);
1820                 tlsk->pldlen = 0;
1821                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1822                 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1823         }
1824
1825         if (!sock_flag(sk, SOCK_DEAD)) {
1826                 check_sk_callbacks(csk);
1827                 sk->sk_data_ready(sk);
1828         }
1829 }
1830
1831 static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1832 {
1833         struct cpl_rx_tls_cmp *req = cplhdr(skb);
1834         unsigned int hwtid = GET_TID(req);
1835         struct sock *sk;
1836
1837         sk = lookup_tid(cdev->tids, hwtid);
1838         if (unlikely(!sk)) {
1839                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1840                 return -EINVAL;
1841         }
1842         skb_dst_set(skb, NULL);
1843         process_cpl_msg(chtls_rx_hdr, sk, skb);
1844
1845         return 0;
1846 }
1847
1848 static void chtls_timewait(struct sock *sk)
1849 {
1850         struct tcp_sock *tp = tcp_sk(sk);
1851
1852         tp->rcv_nxt++;
1853         tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
1854         tp->srtt_us = 0;
1855         tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1856 }
1857
1858 static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1859 {
1860         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1861
1862         if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1863                 goto out;
1864
1865         sk->sk_shutdown |= RCV_SHUTDOWN;
1866         sock_set_flag(sk, SOCK_DONE);
1867
1868         switch (sk->sk_state) {
1869         case TCP_SYN_RECV:
1870         case TCP_ESTABLISHED:
1871                 tcp_set_state(sk, TCP_CLOSE_WAIT);
1872                 break;
1873         case TCP_FIN_WAIT1:
1874                 tcp_set_state(sk, TCP_CLOSING);
1875                 break;
1876         case TCP_FIN_WAIT2:
1877                 chtls_release_resources(sk);
1878                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1879                         chtls_conn_done(sk);
1880                 else
1881                         chtls_timewait(sk);
1882                 break;
1883         default:
1884                 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1885         }
1886
1887         if (!sock_flag(sk, SOCK_DEAD)) {
1888                 sk->sk_state_change(sk);
1889                 /* Do not send POLL_HUP for half duplex close. */
1890
1891                 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1892                     sk->sk_state == TCP_CLOSE)
1893                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1894                 else
1895                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1896         }
1897 out:
1898         kfree_skb(skb);
1899 }
1900
1901 static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1902 {
1903         struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1904         struct chtls_sock *csk;
1905         struct tcp_sock *tp;
1906
1907         csk = rcu_dereference_sk_user_data(sk);
1908
1909         if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1910                 goto out;
1911
1912         tp = tcp_sk(sk);
1913
1914         tp->snd_una = ntohl(rpl->snd_nxt) - 1;  /* exclude FIN */
1915
1916         switch (sk->sk_state) {
1917         case TCP_CLOSING:
1918                 chtls_release_resources(sk);
1919                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1920                         chtls_conn_done(sk);
1921                 else
1922                         chtls_timewait(sk);
1923                 break;
1924         case TCP_LAST_ACK:
1925                 chtls_release_resources(sk);
1926                 chtls_conn_done(sk);
1927                 break;
1928         case TCP_FIN_WAIT1:
1929                 tcp_set_state(sk, TCP_FIN_WAIT2);
1930                 sk->sk_shutdown |= SEND_SHUTDOWN;
1931
1932                 if (!sock_flag(sk, SOCK_DEAD))
1933                         sk->sk_state_change(sk);
1934                 else if (tcp_sk(sk)->linger2 < 0 &&
1935                          !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1936                         chtls_abort_conn(sk, skb);
1937                 break;
1938         default:
1939                 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1940         }
1941 out:
1942         kfree_skb(skb);
1943 }
1944
1945 static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1946                                    size_t len, gfp_t gfp)
1947 {
1948         if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1949                 WARN_ONCE(skb->len < len, "skb alloc error");
1950                 __skb_trim(skb, len);
1951                 skb_get(skb);
1952         } else {
1953                 skb = alloc_skb(len, gfp);
1954                 if (skb)
1955                         __skb_put(skb, len);
1956         }
1957         return skb;
1958 }
1959
1960 static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1961                              int cmd)
1962 {
1963         struct cpl_abort_rpl *rpl = cplhdr(skb);
1964
1965         INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1966         rpl->cmd = cmd;
1967 }
1968
1969 static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1970 {
1971         struct cpl_abort_req_rss *req = cplhdr(skb);
1972         struct sk_buff *reply_skb;
1973
1974         reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1975                               GFP_KERNEL | __GFP_NOFAIL);
1976         __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1977         set_abort_rpl_wr(reply_skb, GET_TID(req),
1978                          (req->status & CPL_ABORT_NO_RST));
1979         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
1980         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1981         kfree_skb(skb);
1982 }
1983
1984 /*
1985  * Add an skb to the deferred skb queue for processing from process context.
1986  */
1987 static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
1988                            defer_handler_t handler)
1989 {
1990         DEFERRED_SKB_CB(skb)->handler = handler;
1991         spin_lock_bh(&cdev->deferq.lock);
1992         __skb_queue_tail(&cdev->deferq, skb);
1993         if (skb_queue_len(&cdev->deferq) == 1)
1994                 schedule_work(&cdev->deferq_task);
1995         spin_unlock_bh(&cdev->deferq.lock);
1996 }
1997
1998 static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1999                            struct chtls_dev *cdev, int status, int queue)
2000 {
2001         struct cpl_abort_req_rss *req = cplhdr(skb);
2002         struct sk_buff *reply_skb;
2003         struct chtls_sock *csk;
2004
2005         csk = rcu_dereference_sk_user_data(sk);
2006
2007         reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
2008                               GFP_KERNEL);
2009
2010         if (!reply_skb) {
2011                 req->status = (queue << 1);
2012                 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
2013                 return;
2014         }
2015
2016         set_abort_rpl_wr(reply_skb, GET_TID(req), status);
2017         kfree_skb(skb);
2018
2019         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
2020         if (csk_conn_inline(csk)) {
2021                 struct l2t_entry *e = csk->l2t_entry;
2022
2023                 if (e && sk->sk_state != TCP_SYN_RECV) {
2024                         cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2025                         return;
2026                 }
2027         }
2028         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2029 }
2030
2031 static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
2032                                  struct chtls_dev *cdev,
2033                                  int status, int queue)
2034 {
2035         struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2036         struct sk_buff *reply_skb;
2037         struct chtls_sock *csk;
2038         unsigned int tid;
2039
2040         csk = rcu_dereference_sk_user_data(sk);
2041         tid = GET_TID(req);
2042
2043         reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
2044         if (!reply_skb) {
2045                 req->status = (queue << 1) | status;
2046                 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
2047                 return;
2048         }
2049
2050         set_abort_rpl_wr(reply_skb, tid, status);
2051         kfree_skb(skb);
2052         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
2053         if (csk_conn_inline(csk)) {
2054                 struct l2t_entry *e = csk->l2t_entry;
2055
2056                 if (e && sk->sk_state != TCP_SYN_RECV) {
2057                         cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
2058                         return;
2059                 }
2060         }
2061         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
2062 }
2063
2064 /*
2065  * This is run from a listener's backlog to abort a child connection in
2066  * SYN_RCV state (i.e., one on the listener's SYN queue).
2067  */
2068 static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
2069 {
2070         struct chtls_sock *csk;
2071         struct sock *child;
2072         int queue;
2073
2074         child = skb->sk;
2075         csk = rcu_dereference_sk_user_data(child);
2076         queue = csk->txq_idx;
2077
2078         skb->sk = NULL;
2079         do_abort_syn_rcv(child, lsk);
2080         send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
2081                        CPL_ABORT_NO_RST, queue);
2082 }
2083
2084 static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
2085 {
2086         const struct request_sock *oreq;
2087         struct listen_ctx *listen_ctx;
2088         struct chtls_sock *csk;
2089         struct chtls_dev *cdev;
2090         struct sock *psk;
2091         void *ctx;
2092
2093         csk = sk->sk_user_data;
2094         oreq = csk->passive_reap_next;
2095         cdev = csk->cdev;
2096
2097         if (!oreq)
2098                 return -1;
2099
2100         ctx = lookup_stid(cdev->tids, oreq->ts_recent);
2101         if (!ctx)
2102                 return -1;
2103
2104         listen_ctx = (struct listen_ctx *)ctx;
2105         psk = listen_ctx->lsk;
2106
2107         bh_lock_sock(psk);
2108         if (!sock_owned_by_user(psk)) {
2109                 int queue = csk->txq_idx;
2110
2111                 do_abort_syn_rcv(sk, psk);
2112                 send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
2113         } else {
2114                 skb->sk = sk;
2115                 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
2116                 __sk_add_backlog(psk, skb);
2117         }
2118         bh_unlock_sock(psk);
2119         return 0;
2120 }
2121
2122 static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
2123 {
2124         const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
2125         struct chtls_sock *csk = sk->sk_user_data;
2126         int rst_status = CPL_ABORT_NO_RST;
2127         int queue = csk->txq_idx;
2128
2129         if (is_neg_adv(req->status)) {
2130                 if (sk->sk_state == TCP_SYN_RECV)
2131                         chtls_set_tcb_tflag(sk, 0, 0);
2132
2133                 kfree_skb(skb);
2134                 return;
2135         }
2136
2137         csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
2138
2139         if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
2140             !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
2141                 struct tcp_sock *tp = tcp_sk(sk);
2142
2143                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
2144                         WARN_ONCE(1, "send_tx_flowc error");
2145                 csk_set_flag(csk, CSK_TX_DATA_SENT);
2146         }
2147
2148         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
2149
2150         if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2151                 sk->sk_err = ETIMEDOUT;
2152
2153                 if (!sock_flag(sk, SOCK_DEAD))
2154                         sk->sk_error_report(sk);
2155
2156                 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
2157                         return;
2158
2159                 chtls_release_resources(sk);
2160                 chtls_conn_done(sk);
2161         }
2162
2163         chtls_send_abort_rpl(sk, skb, BLOG_SKB_CB(skb)->cdev,
2164                              rst_status, queue);
2165 }
2166
2167 static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2168 {
2169         struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2170         struct chtls_sock *csk;
2171         struct chtls_dev *cdev;
2172
2173         csk = rcu_dereference_sk_user_data(sk);
2174         cdev = csk->cdev;
2175
2176         if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2177                 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2178                 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2179                         if (sk->sk_state == TCP_SYN_SENT) {
2180                                 cxgb4_remove_tid(cdev->tids,
2181                                                  csk->port_id,
2182                                                  GET_TID(rpl),
2183                                                  sk->sk_family);
2184                                 sock_put(sk);
2185                         }
2186                         chtls_release_resources(sk);
2187                         chtls_conn_done(sk);
2188                 }
2189         }
2190         kfree_skb(skb);
2191 }
2192
2193 static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2194 {
2195         struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2196         void (*fn)(struct sock *sk, struct sk_buff *skb);
2197         unsigned int hwtid = GET_TID(req);
2198         struct chtls_sock *csk;
2199         struct sock *sk;
2200         u8 opcode;
2201
2202         opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2203
2204         sk = lookup_tid(cdev->tids, hwtid);
2205         if (!sk)
2206                 goto rel_skb;
2207
2208         csk = sk->sk_user_data;
2209
2210         switch (opcode) {
2211         case CPL_PEER_CLOSE:
2212                 fn = chtls_peer_close;
2213                 break;
2214         case CPL_CLOSE_CON_RPL:
2215                 fn = chtls_close_con_rpl;
2216                 break;
2217         case CPL_ABORT_REQ_RSS:
2218                 /*
2219                  * Save the offload device in the skb, we may process this
2220                  * message after the socket has closed.
2221                  */
2222                 BLOG_SKB_CB(skb)->cdev = csk->cdev;
2223                 fn = chtls_abort_req_rss;
2224                 break;
2225         case CPL_ABORT_RPL_RSS:
2226                 fn = chtls_abort_rpl_rss;
2227                 break;
2228         default:
2229                 goto rel_skb;
2230         }
2231
2232         process_cpl_msg(fn, sk, skb);
2233         return 0;
2234
2235 rel_skb:
2236         kfree_skb(skb);
2237         return 0;
2238 }
2239
2240 static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2241 {
2242         struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2243         struct chtls_sock *csk = sk->sk_user_data;
2244         struct tcp_sock *tp = tcp_sk(sk);
2245         u32 credits = hdr->credits;
2246         u32 snd_una;
2247
2248         snd_una = ntohl(hdr->snd_una);
2249         csk->wr_credits += credits;
2250
2251         if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2252                 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2253
2254         while (credits) {
2255                 struct sk_buff *pskb = csk->wr_skb_head;
2256                 u32 csum;
2257
2258                 if (unlikely(!pskb)) {
2259                         if (csk->wr_nondata)
2260                                 csk->wr_nondata -= credits;
2261                         break;
2262                 }
2263                 csum = (__force u32)pskb->csum;
2264                 if (unlikely(credits < csum)) {
2265                         pskb->csum = (__force __wsum)(csum - credits);
2266                         break;
2267                 }
2268                 dequeue_wr(sk);
2269                 credits -= csum;
2270                 kfree_skb(pskb);
2271         }
2272         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2273                 if (unlikely(before(snd_una, tp->snd_una))) {
2274                         kfree_skb(skb);
2275                         return;
2276                 }
2277
2278                 if (tp->snd_una != snd_una) {
2279                         tp->snd_una = snd_una;
2280                         tp->rcv_tstamp = tcp_time_stamp(tp);
2281                         if (tp->snd_una == tp->snd_nxt &&
2282                             !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2283                                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2284                 }
2285         }
2286
2287         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2288                 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2289
2290                 csk->wr_credits -= fclen16;
2291                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2292                 csk_reset_flag(csk, CSK_TX_FAILOVER);
2293         }
2294         if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2295                 sk->sk_write_space(sk);
2296
2297         kfree_skb(skb);
2298 }
2299
2300 static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2301 {
2302         struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2303         unsigned int hwtid = GET_TID(rpl);
2304         struct sock *sk;
2305
2306         sk = lookup_tid(cdev->tids, hwtid);
2307         if (unlikely(!sk)) {
2308                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2309                 return -EINVAL;
2310         }
2311         process_cpl_msg(chtls_rx_ack, sk, skb);
2312
2313         return 0;
2314 }
2315
2316 chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2317         [CPL_PASS_OPEN_RPL]     = chtls_pass_open_rpl,
2318         [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2319         [CPL_PASS_ACCEPT_REQ]   = chtls_pass_accept_req,
2320         [CPL_PASS_ESTABLISH]    = chtls_pass_establish,
2321         [CPL_RX_DATA]           = chtls_rx_data,
2322         [CPL_TLS_DATA]          = chtls_rx_pdu,
2323         [CPL_RX_TLS_CMP]        = chtls_rx_cmp,
2324         [CPL_PEER_CLOSE]        = chtls_conn_cpl,
2325         [CPL_CLOSE_CON_RPL]     = chtls_conn_cpl,
2326         [CPL_ABORT_REQ_RSS]     = chtls_conn_cpl,
2327         [CPL_ABORT_RPL_RSS]     = chtls_conn_cpl,
2328         [CPL_FW4_ACK]           = chtls_wr_ack,
2329 };