drivers/crypto/chelsio/chtls/chtls_cm.c

   1 /*
   2  * Copyright (c) 2018 Chelsio Communications, Inc.
   3  *
   4  * This program is free software; you can redistribute it and/or modify
   5  * it under the terms of the GNU General Public License version 2 as
   6  * published by the Free Software Foundation.
   7  *
   8  * Written by: Atul Gupta (atul.gupta@chelsio.com)
   9  */
  10
  11 #include <linux/module.h>
  12 #include <linux/list.h>
  13 #include <linux/workqueue.h>
  14 #include <linux/skbuff.h>
  15 #include <linux/timer.h>
  16 #include <linux/notifier.h>
  17 #include <linux/inetdevice.h>
  18 #include <linux/ip.h>
  19 #include <linux/tcp.h>
  20 #include <linux/sched/signal.h>
  21 #include <linux/kallsyms.h>
  22 #include <linux/kprobes.h>
  23 #include <linux/if_vlan.h>
  24 #include <net/inet_common.h>
  25 #include <net/tcp.h>
  26 #include <net/dst.h>
  27
  28 #include "chtls.h"
  29 #include "chtls_cm.h"
  30
  31 /*
  32  * State transitions and actions for close.  Note that if we are in SYN_SENT
  33  * we remain in that state as we cannot control a connection while it's in
  34  * SYN_SENT; such connections are allowed to establish and are then aborted.
  35  */
  36 static unsigned char new_state[16] = {
  37         /* current state:     new state:      action: */
  38         /* (Invalid)       */ TCP_CLOSE,
  39         /* TCP_ESTABLISHED */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  40         /* TCP_SYN_SENT    */ TCP_SYN_SENT,
  41         /* TCP_SYN_RECV    */ TCP_FIN_WAIT1 | TCP_ACTION_FIN,
  42         /* TCP_FIN_WAIT1   */ TCP_FIN_WAIT1,
  43         /* TCP_FIN_WAIT2   */ TCP_FIN_WAIT2,
  44         /* TCP_TIME_WAIT   */ TCP_CLOSE,
  45         /* TCP_CLOSE       */ TCP_CLOSE,
  46         /* TCP_CLOSE_WAIT  */ TCP_LAST_ACK | TCP_ACTION_FIN,
  47         /* TCP_LAST_ACK    */ TCP_LAST_ACK,
  48         /* TCP_LISTEN      */ TCP_CLOSE,
  49         /* TCP_CLOSING     */ TCP_CLOSING,
  50 };
  51
  52 static struct chtls_sock *chtls_sock_create(struct chtls_dev *cdev)
  53 {
  54         struct chtls_sock *csk = kzalloc(sizeof(*csk), GFP_ATOMIC);
  55
  56         if (!csk)
  57                 return NULL;
  58
  59         csk->txdata_skb_cache = alloc_skb(TXDATA_SKB_LEN, GFP_ATOMIC);
  60         if (!csk->txdata_skb_cache) {
  61                 kfree(csk);
  62                 return NULL;
  63         }
  64
  65         kref_init(&csk->kref);
  66         csk->cdev = cdev;
  67         skb_queue_head_init(&csk->txq);
  68         csk->wr_skb_head = NULL;
  69         csk->wr_skb_tail = NULL;
  70         csk->mss = MAX_MSS;
  71         csk->tlshws.ofld = 1;
  72         csk->tlshws.txkey = -1;
  73         csk->tlshws.rxkey = -1;
  74         csk->tlshws.mfs = TLS_MFS;
  75         skb_queue_head_init(&csk->tlshws.sk_recv_queue);
  76         return csk;
  77 }
  78
  79 static void chtls_sock_release(struct kref *ref)
  80 {
  81         struct chtls_sock *csk =
  82                 container_of(ref, struct chtls_sock, kref);
  83
  84         kfree(csk);
  85 }
  86
  87 static struct net_device *chtls_ipv4_netdev(struct chtls_dev *cdev,
  88                                             struct sock *sk)
  89 {
  90         struct net_device *ndev = cdev->ports[0];
  91
  92         if (likely(!inet_sk(sk)->inet_rcv_saddr))
  93                 return ndev;
  94
  95         ndev = ip_dev_find(&init_net, inet_sk(sk)->inet_rcv_saddr);
  96         if (!ndev)
  97                 return NULL;
  98
  99         if (is_vlan_dev(ndev))
 100                 return vlan_dev_real_dev(ndev);
 101         return ndev;
 102 }
 103
 104 static void assign_rxopt(struct sock *sk, unsigned int opt)
 105 {
 106         const struct chtls_dev *cdev;
 107         struct chtls_sock *csk;
 108         struct tcp_sock *tp;
 109
 110         csk = rcu_dereference_sk_user_data(sk);
 111         tp = tcp_sk(sk);
 112
 113         cdev = csk->cdev;
 114         tp->tcp_header_len           = sizeof(struct tcphdr);
 115         tp->rx_opt.mss_clamp         = cdev->mtus[TCPOPT_MSS_G(opt)] - 40;
 116         tp->mss_cache                = tp->rx_opt.mss_clamp;
 117         tp->rx_opt.tstamp_ok         = TCPOPT_TSTAMP_G(opt);
 118         tp->rx_opt.snd_wscale        = TCPOPT_SACK_G(opt);
 119         tp->rx_opt.wscale_ok         = TCPOPT_WSCALE_OK_G(opt);
 120         SND_WSCALE(tp)               = TCPOPT_SND_WSCALE_G(opt);
 121         if (!tp->rx_opt.wscale_ok)
 122                 tp->rx_opt.rcv_wscale = 0;
 123         if (tp->rx_opt.tstamp_ok) {
 124                 tp->tcp_header_len += TCPOLEN_TSTAMP_ALIGNED;
 125                 tp->rx_opt.mss_clamp -= TCPOLEN_TSTAMP_ALIGNED;
 126         } else if (csk->opt2 & TSTAMPS_EN_F) {
 127                 csk->opt2 &= ~TSTAMPS_EN_F;
 128                 csk->mtu_idx = TCPOPT_MSS_G(opt);
 129         }
 130 }
 131
 132 static void chtls_purge_receive_queue(struct sock *sk)
 133 {
 134         struct sk_buff *skb;
 135
 136         while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
 137                 skb_dst_set(skb, (void *)NULL);
 138                 kfree_skb(skb);
 139         }
 140 }
 141
 142 static void chtls_purge_write_queue(struct sock *sk)
 143 {
 144         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 145         struct sk_buff *skb;
 146
 147         while ((skb = __skb_dequeue(&csk->txq))) {
 148                 sk->sk_wmem_queued -= skb->truesize;
 149                 __kfree_skb(skb);
 150         }
 151 }
 152
 153 static void chtls_purge_recv_queue(struct sock *sk)
 154 {
 155         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 156         struct chtls_hws *tlsk = &csk->tlshws;
 157         struct sk_buff *skb;
 158
 159         while ((skb = __skb_dequeue(&tlsk->sk_recv_queue)) != NULL) {
 160                 skb_dst_set(skb, NULL);
 161                 kfree_skb(skb);
 162         }
 163 }
 164
 165 static void abort_arp_failure(void *handle, struct sk_buff *skb)
 166 {
 167         struct cpl_abort_req *req = cplhdr(skb);
 168         struct chtls_dev *cdev;
 169
 170         cdev = (struct chtls_dev *)handle;
 171         req->cmd = CPL_ABORT_NO_RST;
 172         cxgb4_ofld_send(cdev->lldi->ports[0], skb);
 173 }
 174
 175 static struct sk_buff *alloc_ctrl_skb(struct sk_buff *skb, int len)
 176 {
 177         if (likely(skb && !skb_shared(skb) && !skb_cloned(skb))) {
 178                 __skb_trim(skb, 0);
 179                 refcount_add(2, &skb->users);
 180         } else {
 181                 skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 182         }
 183         return skb;
 184 }
 185
 186 static void chtls_send_abort(struct sock *sk, int mode, struct sk_buff *skb)
 187 {
 188         struct cpl_abort_req *req;
 189         struct chtls_sock *csk;
 190         struct tcp_sock *tp;
 191
 192         csk = rcu_dereference_sk_user_data(sk);
 193         tp = tcp_sk(sk);
 194
 195         if (!skb)
 196                 skb = alloc_ctrl_skb(csk->txdata_skb_cache, sizeof(*req));
 197
 198         req = (struct cpl_abort_req *)skb_put(skb, sizeof(*req));
 199         INIT_TP_WR_CPL(req, CPL_ABORT_REQ, csk->tid);
 200         skb_set_queue_mapping(skb, (csk->txq_idx << 1) | CPL_PRIORITY_DATA);
 201         req->rsvd0 = htonl(tp->snd_nxt);
 202         req->rsvd1 = !csk_flag_nochk(csk, CSK_TX_DATA_SENT);
 203         req->cmd = mode;
 204         t4_set_arp_err_handler(skb, csk->cdev, abort_arp_failure);
 205         send_or_defer(sk, tp, skb, mode == CPL_ABORT_SEND_RST);
 206 }
 207
 208 static void chtls_send_reset(struct sock *sk, int mode, struct sk_buff *skb)
 209 {
 210         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 211
 212         if (unlikely(csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) ||
 213                      !csk->cdev)) {
 214                 if (sk->sk_state == TCP_SYN_RECV)
 215                         csk_set_flag(csk, CSK_RST_ABORTED);
 216                 goto out;
 217         }
 218
 219         if (!csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
 220                 struct tcp_sock *tp = tcp_sk(sk);
 221
 222                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
 223                         WARN_ONCE(1, "send tx flowc error");
 224                 csk_set_flag(csk, CSK_TX_DATA_SENT);
 225         }
 226
 227         csk_set_flag(csk, CSK_ABORT_RPL_PENDING);
 228         chtls_purge_write_queue(sk);
 229
 230         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
 231         if (sk->sk_state != TCP_SYN_RECV)
 232                 chtls_send_abort(sk, mode, skb);
 233         else
 234                 goto out;
 235
 236         return;
 237 out:
 238         kfree_skb(skb);
 239 }
 240
 241 static void release_tcp_port(struct sock *sk)
 242 {
 243         if (inet_csk(sk)->icsk_bind_hash)
 244                 inet_put_port(sk);
 245 }
 246
 247 static void tcp_uncork(struct sock *sk)
 248 {
 249         struct tcp_sock *tp = tcp_sk(sk);
 250
 251         if (tp->nonagle & TCP_NAGLE_CORK) {
 252                 tp->nonagle &= ~TCP_NAGLE_CORK;
 253                 chtls_tcp_push(sk, 0);
 254         }
 255 }
 256
 257 static void chtls_close_conn(struct sock *sk)
 258 {
 259         struct cpl_close_con_req *req;
 260         struct chtls_sock *csk;
 261         struct sk_buff *skb;
 262         unsigned int tid;
 263         unsigned int len;
 264
 265         len = roundup(sizeof(struct cpl_close_con_req), 16);
 266         csk = rcu_dereference_sk_user_data(sk);
 267         tid = csk->tid;
 268
 269         skb = alloc_skb(len, GFP_KERNEL | __GFP_NOFAIL);
 270         req = (struct cpl_close_con_req *)__skb_put(skb, len);
 271         memset(req, 0, len);
 272         req->wr.wr_hi = htonl(FW_WR_OP_V(FW_TP_WR) |
 273                               FW_WR_IMMDLEN_V(sizeof(*req) -
 274                                               sizeof(req->wr)));
 275         req->wr.wr_mid = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(sizeof(*req), 16)) |
 276                                FW_WR_FLOWID_V(tid));
 277
 278         OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_CLOSE_CON_REQ, tid));
 279
 280         tcp_uncork(sk);
 281         skb_entail(sk, skb, ULPCB_FLAG_NO_HDR | ULPCB_FLAG_NO_APPEND);
 282         if (sk->sk_state != TCP_SYN_SENT)
 283                 chtls_push_frames(csk, 1);
 284 }
 285
 286 /*
 287  * Perform a state transition during close and return the actions indicated
 288  * for the transition.  Do not make this function inline, the main reason
 289  * it exists at all is to avoid multiple inlining of tcp_set_state.
 290  */
 291 static int make_close_transition(struct sock *sk)
 292 {
 293         int next = (int)new_state[sk->sk_state];
 294
 295         tcp_set_state(sk, next & TCP_STATE_MASK);
 296         return next & TCP_ACTION_FIN;
 297 }
 298
 299 void chtls_close(struct sock *sk, long timeout)
 300 {
 301         int data_lost, prev_state;
 302         struct chtls_sock *csk;
 303
 304         csk = rcu_dereference_sk_user_data(sk);
 305
 306         lock_sock(sk);
 307         sk->sk_shutdown |= SHUTDOWN_MASK;
 308
 309         data_lost = skb_queue_len(&sk->sk_receive_queue);
 310         data_lost |= skb_queue_len(&csk->tlshws.sk_recv_queue);
 311         chtls_purge_recv_queue(sk);
 312         chtls_purge_receive_queue(sk);
 313
 314         if (sk->sk_state == TCP_CLOSE) {
 315                 goto wait;
 316         } else if (data_lost || sk->sk_state == TCP_SYN_SENT) {
 317                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 318                 release_tcp_port(sk);
 319                 goto unlock;
 320         } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
 321                 sk->sk_prot->disconnect(sk, 0);
 322         } else if (make_close_transition(sk)) {
 323                 chtls_close_conn(sk);
 324         }
 325 wait:
 326         if (timeout)
 327                 sk_stream_wait_close(sk, timeout);
 328
 329 unlock:
 330         prev_state = sk->sk_state;
 331         sock_hold(sk);
 332         sock_orphan(sk);
 333
 334         release_sock(sk);
 335
 336         local_bh_disable();
 337         bh_lock_sock(sk);
 338
 339         if (prev_state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
 340                 goto out;
 341
 342         if (sk->sk_state == TCP_FIN_WAIT2 && tcp_sk(sk)->linger2 < 0 &&
 343             !csk_flag(sk, CSK_ABORT_SHUTDOWN)) {
 344                 struct sk_buff *skb;
 345
 346                 skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
 347                 if (skb)
 348                         chtls_send_reset(sk, CPL_ABORT_SEND_RST, skb);
 349         }
 350
 351         if (sk->sk_state == TCP_CLOSE)
 352                 inet_csk_destroy_sock(sk);
 353
 354 out:
 355         bh_unlock_sock(sk);
 356         local_bh_enable();
 357         sock_put(sk);
 358 }
 359
 360 /*
 361  * Wait until a socket enters on of the given states.
 362  */
 363 static int wait_for_states(struct sock *sk, unsigned int states)
 364 {
 365         DECLARE_WAITQUEUE(wait, current);
 366         struct socket_wq _sk_wq;
 367         long current_timeo;
 368         int err = 0;
 369
 370         current_timeo = 200;
 371
 372         /*
 373          * We want this to work even when there's no associated struct socket.
 374          * In that case we provide a temporary wait_queue_head_t.
 375          */
 376         if (!sk->sk_wq) {
 377                 init_waitqueue_head(&_sk_wq.wait);
 378                 _sk_wq.fasync_list = NULL;
 379                 init_rcu_head_on_stack(&_sk_wq.rcu);
 380                 RCU_INIT_POINTER(sk->sk_wq, &_sk_wq);
 381         }
 382
 383         add_wait_queue(sk_sleep(sk), &wait);
 384         while (!sk_in_state(sk, states)) {
 385                 if (!current_timeo) {
 386                         err = -EBUSY;
 387                         break;
 388                 }
 389                 if (signal_pending(current)) {
 390                         err = sock_intr_errno(current_timeo);
 391                         break;
 392                 }
 393                 set_current_state(TASK_UNINTERRUPTIBLE);
 394                 release_sock(sk);
 395                 if (!sk_in_state(sk, states))
 396                         current_timeo = schedule_timeout(current_timeo);
 397                 __set_current_state(TASK_RUNNING);
 398                 lock_sock(sk);
 399         }
 400         remove_wait_queue(sk_sleep(sk), &wait);
 401
 402         if (rcu_dereference(sk->sk_wq) == &_sk_wq)
 403                 sk->sk_wq = NULL;
 404         return err;
 405 }
 406
 407 int chtls_disconnect(struct sock *sk, int flags)
 408 {
 409         struct tcp_sock *tp;
 410         int err;
 411
 412         tp = tcp_sk(sk);
 413         chtls_purge_recv_queue(sk);
 414         chtls_purge_receive_queue(sk);
 415         chtls_purge_write_queue(sk);
 416
 417         if (sk->sk_state != TCP_CLOSE) {
 418                 sk->sk_err = ECONNRESET;
 419                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, NULL);
 420                 err = wait_for_states(sk, TCPF_CLOSE);
 421                 if (err)
 422                         return err;
 423         }
 424         chtls_purge_recv_queue(sk);
 425         chtls_purge_receive_queue(sk);
 426         tp->max_window = 0xFFFF << (tp->rx_opt.snd_wscale);
 427         return tcp_disconnect(sk, flags);
 428 }
 429
 430 #define SHUTDOWN_ELIGIBLE_STATE (TCPF_ESTABLISHED | \
 431                                  TCPF_SYN_RECV | TCPF_CLOSE_WAIT)
 432 void chtls_shutdown(struct sock *sk, int how)
 433 {
 434         if ((how & SEND_SHUTDOWN) &&
 435             sk_in_state(sk, SHUTDOWN_ELIGIBLE_STATE) &&
 436             make_close_transition(sk))
 437                 chtls_close_conn(sk);
 438 }
 439
 440 void chtls_destroy_sock(struct sock *sk)
 441 {
 442         struct chtls_sock *csk;
 443
 444         csk = rcu_dereference_sk_user_data(sk);
 445         chtls_purge_recv_queue(sk);
 446         csk->ulp_mode = ULP_MODE_NONE;
 447         chtls_purge_write_queue(sk);
 448         free_tls_keyid(sk);
 449         kref_put(&csk->kref, chtls_sock_release);
 450         sk->sk_prot = &tcp_prot;
 451         sk->sk_prot->destroy(sk);
 452 }
 453
 454 static void reset_listen_child(struct sock *child)
 455 {
 456         struct chtls_sock *csk = rcu_dereference_sk_user_data(child);
 457         struct sk_buff *skb;
 458
 459         skb = alloc_ctrl_skb(csk->txdata_skb_cache,
 460                              sizeof(struct cpl_abort_req));
 461
 462         chtls_send_reset(child, CPL_ABORT_SEND_RST, skb);
 463         sock_orphan(child);
 464         INC_ORPHAN_COUNT(child);
 465         if (child->sk_state == TCP_CLOSE)
 466                 inet_csk_destroy_sock(child);
 467 }
 468
 469 static void chtls_disconnect_acceptq(struct sock *listen_sk)
 470 {
 471         struct request_sock **pprev;
 472
 473         pprev = ACCEPT_QUEUE(listen_sk);
 474         while (*pprev) {
 475                 struct request_sock *req = *pprev;
 476
 477                 if (req->rsk_ops == &chtls_rsk_ops) {
 478                         struct sock *child = req->sk;
 479
 480                         *pprev = req->dl_next;
 481                         sk_acceptq_removed(listen_sk);
 482                         reqsk_put(req);
 483                         sock_hold(child);
 484                         local_bh_disable();
 485                         bh_lock_sock(child);
 486                         release_tcp_port(child);
 487                         reset_listen_child(child);
 488                         bh_unlock_sock(child);
 489                         local_bh_enable();
 490                         sock_put(child);
 491                 } else {
 492                         pprev = &req->dl_next;
 493                 }
 494         }
 495 }
 496
 497 static int listen_hashfn(const struct sock *sk)
 498 {
 499         return ((unsigned long)sk >> 10) & (LISTEN_INFO_HASH_SIZE - 1);
 500 }
 501
 502 static struct listen_info *listen_hash_add(struct chtls_dev *cdev,
 503                                            struct sock *sk,
 504                                            unsigned int stid)
 505 {
 506         struct listen_info *p = kmalloc(sizeof(*p), GFP_KERNEL);
 507
 508         if (p) {
 509                 int key = listen_hashfn(sk);
 510
 511                 p->sk = sk;
 512                 p->stid = stid;
 513                 spin_lock(&cdev->listen_lock);
 514                 p->next = cdev->listen_hash_tab[key];
 515                 cdev->listen_hash_tab[key] = p;
 516                 spin_unlock(&cdev->listen_lock);
 517         }
 518         return p;
 519 }
 520
 521 static int listen_hash_find(struct chtls_dev *cdev,
 522                             struct sock *sk)
 523 {
 524         struct listen_info *p;
 525         int stid = -1;
 526         int key;
 527
 528         key = listen_hashfn(sk);
 529
 530         spin_lock(&cdev->listen_lock);
 531         for (p = cdev->listen_hash_tab[key]; p; p = p->next)
 532                 if (p->sk == sk) {
 533                         stid = p->stid;
 534                         break;
 535                 }
 536         spin_unlock(&cdev->listen_lock);
 537         return stid;
 538 }
 539
 540 static int listen_hash_del(struct chtls_dev *cdev,
 541                            struct sock *sk)
 542 {
 543         struct listen_info *p, **prev;
 544         int stid = -1;
 545         int key;
 546
 547         key = listen_hashfn(sk);
 548         prev = &cdev->listen_hash_tab[key];
 549
 550         spin_lock(&cdev->listen_lock);
 551         for (p = *prev; p; prev = &p->next, p = p->next)
 552                 if (p->sk == sk) {
 553                         stid = p->stid;
 554                         *prev = p->next;
 555                         kfree(p);
 556                         break;
 557                 }
 558         spin_unlock(&cdev->listen_lock);
 559         return stid;
 560 }
 561
 562 static void cleanup_syn_rcv_conn(struct sock *child, struct sock *parent)
 563 {
 564         struct request_sock *req;
 565         struct chtls_sock *csk;
 566
 567         csk = rcu_dereference_sk_user_data(child);
 568         req = csk->passive_reap_next;
 569
 570         reqsk_queue_removed(&inet_csk(parent)->icsk_accept_queue, req);
 571         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
 572         chtls_reqsk_free(req);
 573         csk->passive_reap_next = NULL;
 574 }
 575
 576 static void chtls_reset_synq(struct listen_ctx *listen_ctx)
 577 {
 578         struct sock *listen_sk = listen_ctx->lsk;
 579
 580         while (!skb_queue_empty(&listen_ctx->synq)) {
 581                 struct chtls_sock *csk =
 582                         container_of((struct synq *)__skb_dequeue
 583                                 (&listen_ctx->synq), struct chtls_sock, synq);
 584                 struct sock *child = csk->sk;
 585
 586                 cleanup_syn_rcv_conn(child, listen_sk);
 587                 sock_hold(child);
 588                 local_bh_disable();
 589                 bh_lock_sock(child);
 590                 release_tcp_port(child);
 591                 reset_listen_child(child);
 592                 bh_unlock_sock(child);
 593                 local_bh_enable();
 594                 sock_put(child);
 595         }
 596 }
 597
 598 int chtls_listen_start(struct chtls_dev *cdev, struct sock *sk)
 599 {
 600         struct net_device *ndev;
 601         struct listen_ctx *ctx;
 602         struct adapter *adap;
 603         struct port_info *pi;
 604         int stid;
 605         int ret;
 606
 607         if (sk->sk_family != PF_INET)
 608                 return -EAGAIN;
 609
 610         rcu_read_lock();
 611         ndev = chtls_ipv4_netdev(cdev, sk);
 612         rcu_read_unlock();
 613         if (!ndev)
 614                 return -EBADF;
 615
 616         pi = netdev_priv(ndev);
 617         adap = pi->adapter;
 618         if (!(adap->flags & FULL_INIT_DONE))
 619                 return -EBADF;
 620
 621         if (listen_hash_find(cdev, sk) >= 0)   /* already have it */
 622                 return -EADDRINUSE;
 623
 624         ctx = kmalloc(sizeof(*ctx), GFP_KERNEL);
 625         if (!ctx)
 626                 return -ENOMEM;
 627
 628         __module_get(THIS_MODULE);
 629         ctx->lsk = sk;
 630         ctx->cdev = cdev;
 631         ctx->state = T4_LISTEN_START_PENDING;
 632         skb_queue_head_init(&ctx->synq);
 633
 634         stid = cxgb4_alloc_stid(cdev->tids, sk->sk_family, ctx);
 635         if (stid < 0)
 636                 goto free_ctx;
 637
 638         sock_hold(sk);
 639         if (!listen_hash_add(cdev, sk, stid))
 640                 goto free_stid;
 641
 642         ret = cxgb4_create_server(ndev, stid,
 643                                   inet_sk(sk)->inet_rcv_saddr,
 644                                   inet_sk(sk)->inet_sport, 0,
 645                                   cdev->lldi->rxq_ids[0]);
 646         if (ret > 0)
 647                 ret = net_xmit_errno(ret);
 648         if (ret)
 649                 goto del_hash;
 650         return 0;
 651 del_hash:
 652         listen_hash_del(cdev, sk);
 653 free_stid:
 654         cxgb4_free_stid(cdev->tids, stid, sk->sk_family);
 655         sock_put(sk);
 656 free_ctx:
 657         kfree(ctx);
 658         module_put(THIS_MODULE);
 659         return -EBADF;
 660 }
 661
 662 void chtls_listen_stop(struct chtls_dev *cdev, struct sock *sk)
 663 {
 664         struct listen_ctx *listen_ctx;
 665         int stid;
 666
 667         stid = listen_hash_del(cdev, sk);
 668         if (stid < 0)
 669                 return;
 670
 671         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 672         chtls_reset_synq(listen_ctx);
 673
 674         cxgb4_remove_server(cdev->lldi->ports[0], stid,
 675                             cdev->lldi->rxq_ids[0], 0);
 676         chtls_disconnect_acceptq(sk);
 677 }
 678
 679 static int chtls_pass_open_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 680 {
 681         struct cpl_pass_open_rpl *rpl = cplhdr(skb) + RSS_HDR;
 682         unsigned int stid = GET_TID(rpl);
 683         struct listen_ctx *listen_ctx;
 684
 685         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
 686         if (!listen_ctx)
 687                 return CPL_RET_BUF_DONE;
 688
 689         if (listen_ctx->state == T4_LISTEN_START_PENDING) {
 690                 listen_ctx->state = T4_LISTEN_STARTED;
 691                 return CPL_RET_BUF_DONE;
 692         }
 693
 694         if (rpl->status != CPL_ERR_NONE) {
 695                 pr_info("Unexpected PASS_OPEN_RPL status %u for STID %u\n",
 696                         rpl->status, stid);
 697                 return CPL_RET_BUF_DONE;
 698         }
 699         cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 700         sock_put(listen_ctx->lsk);
 701         kfree(listen_ctx);
 702         module_put(THIS_MODULE);
 703
 704         return 0;
 705 }
 706
 707 static int chtls_close_listsrv_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
 708 {
 709         struct cpl_close_listsvr_rpl *rpl = cplhdr(skb) + RSS_HDR;
 710         struct listen_ctx *listen_ctx;
 711         unsigned int stid;
 712         void *data;
 713
 714         stid = GET_TID(rpl);
 715         data = lookup_stid(cdev->tids, stid);
 716         listen_ctx = (struct listen_ctx *)data;
 717
 718         if (rpl->status != CPL_ERR_NONE) {
 719                 pr_info("Unexpected CLOSE_LISTSRV_RPL status %u for STID %u\n",
 720                         rpl->status, stid);
 721                 return CPL_RET_BUF_DONE;
 722         }
 723
 724         cxgb4_free_stid(cdev->tids, stid, listen_ctx->lsk->sk_family);
 725         sock_put(listen_ctx->lsk);
 726         kfree(listen_ctx);
 727         module_put(THIS_MODULE);
 728
 729         return 0;
 730 }
 731
 732 static void chtls_release_resources(struct sock *sk)
 733 {
 734         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
 735         struct chtls_dev *cdev = csk->cdev;
 736         unsigned int tid = csk->tid;
 737         struct tid_info *tids;
 738
 739         if (!cdev)
 740                 return;
 741
 742         tids = cdev->tids;
 743         kfree_skb(csk->txdata_skb_cache);
 744         csk->txdata_skb_cache = NULL;
 745
 746         if (csk->l2t_entry) {
 747                 cxgb4_l2t_release(csk->l2t_entry);
 748                 csk->l2t_entry = NULL;
 749         }
 750
 751         cxgb4_remove_tid(tids, csk->port_id, tid, sk->sk_family);
 752         sock_put(sk);
 753 }
 754
 755 static void chtls_conn_done(struct sock *sk)
 756 {
 757         if (sock_flag(sk, SOCK_DEAD))
 758                 chtls_purge_receive_queue(sk);
 759         sk_wakeup_sleepers(sk, 0);
 760         tcp_done(sk);
 761 }
 762
 763 static void do_abort_syn_rcv(struct sock *child, struct sock *parent)
 764 {
 765         /*
 766          * If the server is still open we clean up the child connection,
 767          * otherwise the server already did the clean up as it was purging
 768          * its SYN queue and the skb was just sitting in its backlog.
 769          */
 770         if (likely(parent->sk_state == TCP_LISTEN)) {
 771                 cleanup_syn_rcv_conn(child, parent);
 772                 /* Without the below call to sock_orphan,
 773                  * we leak the socket resource with syn_flood test
 774                  * as inet_csk_destroy_sock will not be called
 775                  * in tcp_done since SOCK_DEAD flag is not set.
 776                  * Kernel handles this differently where new socket is
 777                  * created only after 3 way handshake is done.
 778                  */
 779                 sock_orphan(child);
 780                 percpu_counter_inc((child)->sk_prot->orphan_count);
 781                 chtls_release_resources(child);
 782                 chtls_conn_done(child);
 783         } else {
 784                 if (csk_flag(child, CSK_RST_ABORTED)) {
 785                         chtls_release_resources(child);
 786                         chtls_conn_done(child);
 787                 }
 788         }
 789 }
 790
 791 static void pass_open_abort(struct sock *child, struct sock *parent,
 792                             struct sk_buff *skb)
 793 {
 794         do_abort_syn_rcv(child, parent);
 795         kfree_skb(skb);
 796 }
 797
 798 static void bl_pass_open_abort(struct sock *lsk, struct sk_buff *skb)
 799 {
 800         pass_open_abort(skb->sk, lsk, skb);
 801 }
 802
 803 static void chtls_pass_open_arp_failure(struct sock *sk,
 804                                         struct sk_buff *skb)
 805 {
 806         const struct request_sock *oreq;
 807         struct chtls_sock *csk;
 808         struct chtls_dev *cdev;
 809         struct sock *parent;
 810         void *data;
 811
 812         csk = rcu_dereference_sk_user_data(sk);
 813         cdev = csk->cdev;
 814
 815         /*
 816          * If the connection is being aborted due to the parent listening
 817          * socket going away there's nothing to do, the ABORT_REQ will close
 818          * the connection.
 819          */
 820         if (csk_flag(sk, CSK_ABORT_RPL_PENDING)) {
 821                 kfree_skb(skb);
 822                 return;
 823         }
 824
 825         oreq = csk->passive_reap_next;
 826         data = lookup_stid(cdev->tids, oreq->ts_recent);
 827         parent = ((struct listen_ctx *)data)->lsk;
 828
 829         bh_lock_sock(parent);
 830         if (!sock_owned_by_user(parent)) {
 831                 pass_open_abort(sk, parent, skb);
 832         } else {
 833                 BLOG_SKB_CB(skb)->backlog_rcv = bl_pass_open_abort;
 834                 __sk_add_backlog(parent, skb);
 835         }
 836         bh_unlock_sock(parent);
 837 }
 838
 839 static void chtls_accept_rpl_arp_failure(void *handle,
 840                                          struct sk_buff *skb)
 841 {
 842         struct sock *sk = (struct sock *)handle;
 843
 844         sock_hold(sk);
 845         process_cpl_msg(chtls_pass_open_arp_failure, sk, skb);
 846         sock_put(sk);
 847 }
 848
 849 static unsigned int chtls_select_mss(const struct chtls_sock *csk,
 850                                      unsigned int pmtu,
 851                                      struct cpl_pass_accept_req *req)
 852 {
 853         struct chtls_dev *cdev;
 854         struct dst_entry *dst;
 855         unsigned int tcpoptsz;
 856         unsigned int iphdrsz;
 857         unsigned int mtu_idx;
 858         struct tcp_sock *tp;
 859         unsigned int mss;
 860         struct sock *sk;
 861
 862         mss = ntohs(req->tcpopt.mss);
 863         sk = csk->sk;
 864         dst = __sk_dst_get(sk);
 865         cdev = csk->cdev;
 866         tp = tcp_sk(sk);
 867         tcpoptsz = 0;
 868
 869         iphdrsz = sizeof(struct iphdr) + sizeof(struct tcphdr);
 870         if (req->tcpopt.tstamp)
 871                 tcpoptsz += round_up(TCPOLEN_TIMESTAMP, 4);
 872
 873         tp->advmss = dst_metric_advmss(dst);
 874         if (USER_MSS(tp) && tp->advmss > USER_MSS(tp))
 875                 tp->advmss = USER_MSS(tp);
 876         if (tp->advmss > pmtu - iphdrsz)
 877                 tp->advmss = pmtu - iphdrsz;
 878         if (mss && tp->advmss > mss)
 879                 tp->advmss = mss;
 880
 881         tp->advmss = cxgb4_best_aligned_mtu(cdev->lldi->mtus,
 882                                             iphdrsz + tcpoptsz,
 883                                             tp->advmss - tcpoptsz,
 884                                             8, &mtu_idx);
 885         tp->advmss -= iphdrsz;
 886
 887         inet_csk(sk)->icsk_pmtu_cookie = pmtu;
 888         return mtu_idx;
 889 }
 890
 891 static unsigned int select_rcv_wscale(int space, int wscale_ok, int win_clamp)
 892 {
 893         int wscale = 0;
 894
 895         if (space > MAX_RCV_WND)
 896                 space = MAX_RCV_WND;
 897         if (win_clamp && win_clamp < space)
 898                 space = win_clamp;
 899
 900         if (wscale_ok) {
 901                 while (wscale < 14 && (65535 << wscale) < space)
 902                         wscale++;
 903         }
 904         return wscale;
 905 }
 906
 907 static void chtls_pass_accept_rpl(struct sk_buff *skb,
 908                                   struct cpl_pass_accept_req *req,
 909                                   unsigned int tid)
 910
 911 {
 912         struct cpl_t5_pass_accept_rpl *rpl5;
 913         struct cxgb4_lld_info *lldi;
 914         const struct tcphdr *tcph;
 915         const struct tcp_sock *tp;
 916         struct chtls_sock *csk;
 917         unsigned int len;
 918         struct sock *sk;
 919         u32 opt2, hlen;
 920         u64 opt0;
 921
 922         sk = skb->sk;
 923         tp = tcp_sk(sk);
 924         csk = sk->sk_user_data;
 925         csk->tid = tid;
 926         lldi = csk->cdev->lldi;
 927         len = roundup(sizeof(*rpl5), 16);
 928
 929         rpl5 = __skb_put_zero(skb, len);
 930         INIT_TP_WR(rpl5, tid);
 931
 932         OPCODE_TID(rpl5) = cpu_to_be32(MK_OPCODE_TID(CPL_PASS_ACCEPT_RPL,
 933                                                      csk->tid));
 934         csk->mtu_idx = chtls_select_mss(csk, dst_mtu(__sk_dst_get(sk)),
 935                                         req);
 936         opt0 = TCAM_BYPASS_F |
 937                WND_SCALE_V(RCV_WSCALE(tp)) |
 938                MSS_IDX_V(csk->mtu_idx) |
 939                L2T_IDX_V(csk->l2t_entry->idx) |
 940                NAGLE_V(!(tp->nonagle & TCP_NAGLE_OFF)) |
 941                TX_CHAN_V(csk->tx_chan) |
 942                SMAC_SEL_V(csk->smac_idx) |
 943                DSCP_V(csk->tos >> 2) |
 944                ULP_MODE_V(ULP_MODE_TLS) |
 945                RCV_BUFSIZ_V(min(tp->rcv_wnd >> 10, RCV_BUFSIZ_M));
 946
 947         opt2 = RX_CHANNEL_V(0) |
 948                 RSS_QUEUE_VALID_F | RSS_QUEUE_V(csk->rss_qid);
 949
 950         if (!is_t5(lldi->adapter_type))
 951                 opt2 |= RX_FC_DISABLE_F;
 952         if (req->tcpopt.tstamp)
 953                 opt2 |= TSTAMPS_EN_F;
 954         if (req->tcpopt.sack)
 955                 opt2 |= SACK_EN_F;
 956         hlen = ntohl(req->hdr_len);
 957
 958         tcph = (struct tcphdr *)((u8 *)(req + 1) +
 959                         T6_ETH_HDR_LEN_G(hlen) + T6_IP_HDR_LEN_G(hlen));
 960         if (tcph->ece && tcph->cwr)
 961                 opt2 |= CCTRL_ECN_V(1);
 962         opt2 |= CONG_CNTRL_V(CONG_ALG_NEWRENO);
 963         opt2 |= T5_ISS_F;
 964         opt2 |= T5_OPT_2_VALID_F;
 965         rpl5->opt0 = cpu_to_be64(opt0);
 966         rpl5->opt2 = cpu_to_be32(opt2);
 967         rpl5->iss = cpu_to_be32((prandom_u32() & ~7UL) - 1);
 968         set_wr_txq(skb, CPL_PRIORITY_SETUP, csk->port_id);
 969         t4_set_arp_err_handler(skb, sk, chtls_accept_rpl_arp_failure);
 970         cxgb4_l2t_send(csk->egress_dev, skb, csk->l2t_entry);
 971 }
 972
 973 static void inet_inherit_port(struct inet_hashinfo *hash_info,
 974                               struct sock *lsk, struct sock *newsk)
 975 {
 976         local_bh_disable();
 977         __inet_inherit_port(lsk, newsk);
 978         local_bh_enable();
 979 }
 980
 981 static int chtls_backlog_rcv(struct sock *sk, struct sk_buff *skb)
 982 {
 983         if (skb->protocol) {
 984                 kfree_skb(skb);
 985                 return 0;
 986         }
 987         BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
 988         return 0;
 989 }
 990
 991 static void chtls_set_tcp_window(struct chtls_sock *csk)
 992 {
 993         struct net_device *ndev = csk->egress_dev;
 994         struct port_info *pi = netdev_priv(ndev);
 995         unsigned int linkspeed;
 996         u8 scale;
 997
 998         linkspeed = pi->link_cfg.speed;
 999         scale = linkspeed / SPEED_10000;
1000 #define CHTLS_10G_RCVWIN (256 * 1024)
1001         csk->rcv_win = CHTLS_10G_RCVWIN;
1002         if (scale)
1003                 csk->rcv_win *= scale;
1004 #define CHTLS_10G_SNDWIN (256 * 1024)
1005         csk->snd_win = CHTLS_10G_SNDWIN;
1006         if (scale)
1007                 csk->snd_win *= scale;
1008 }
1009
1010 static struct sock *chtls_recv_sock(struct sock *lsk,
1011                                     struct request_sock *oreq,
1012                                     void *network_hdr,
1013                                     const struct cpl_pass_accept_req *req,
1014                                     struct chtls_dev *cdev)
1015 {
1016         struct inet_sock *newinet;
1017         const struct iphdr *iph;
1018         struct net_device *ndev;
1019         struct chtls_sock *csk;
1020         struct dst_entry *dst;
1021         struct neighbour *n;
1022         struct tcp_sock *tp;
1023         struct sock *newsk;
1024         u16 port_id;
1025         int rxq_idx;
1026         int step;
1027
1028         iph = (const struct iphdr *)network_hdr;
1029         newsk = tcp_create_openreq_child(lsk, oreq, cdev->askb);
1030         if (!newsk)
1031                 goto free_oreq;
1032
1033         dst = inet_csk_route_child_sock(lsk, newsk, oreq);
1034         if (!dst)
1035                 goto free_sk;
1036
1037         n = dst_neigh_lookup(dst, &iph->saddr);
1038         if (!n)
1039                 goto free_sk;
1040
1041         ndev = n->dev;
1042         if (!ndev)
1043                 goto free_dst;
1044         port_id = cxgb4_port_idx(ndev);
1045
1046         csk = chtls_sock_create(cdev);
1047         if (!csk)
1048                 goto free_dst;
1049
1050         csk->l2t_entry = cxgb4_l2t_get(cdev->lldi->l2t, n, ndev, 0);
1051         if (!csk->l2t_entry)
1052                 goto free_csk;
1053
1054         newsk->sk_user_data = csk;
1055         newsk->sk_backlog_rcv = chtls_backlog_rcv;
1056
1057         tp = tcp_sk(newsk);
1058         newinet = inet_sk(newsk);
1059
1060         newinet->inet_daddr = iph->saddr;
1061         newinet->inet_rcv_saddr = iph->daddr;
1062         newinet->inet_saddr = iph->daddr;
1063
1064         oreq->ts_recent = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1065         sk_setup_caps(newsk, dst);
1066         csk->sk = newsk;
1067         csk->passive_reap_next = oreq;
1068         csk->tx_chan = cxgb4_port_chan(ndev);
1069         csk->port_id = port_id;
1070         csk->egress_dev = ndev;
1071         csk->tos = PASS_OPEN_TOS_G(ntohl(req->tos_stid));
1072         chtls_set_tcp_window(csk);
1073         tp->rcv_wnd = csk->rcv_win;
1074         csk->sndbuf = csk->snd_win;
1075         csk->ulp_mode = ULP_MODE_TLS;
1076         step = cdev->lldi->nrxq / cdev->lldi->nchan;
1077         csk->rss_qid = cdev->lldi->rxq_ids[port_id * step];
1078         rxq_idx = port_id * step;
1079         csk->txq_idx = (rxq_idx < cdev->lldi->ntxq) ? rxq_idx :
1080                         port_id * step;
1081         csk->sndbuf = newsk->sk_sndbuf;
1082         csk->smac_idx = ((struct port_info *)netdev_priv(ndev))->smt_idx;
1083         RCV_WSCALE(tp) = select_rcv_wscale(tcp_full_space(newsk),
1084                                            sock_net(newsk)->
1085                                                 ipv4.sysctl_tcp_window_scaling,
1086                                            tp->window_clamp);
1087         neigh_release(n);
1088         inet_inherit_port(&tcp_hashinfo, lsk, newsk);
1089         csk_set_flag(csk, CSK_CONN_INLINE);
1090         bh_unlock_sock(newsk); /* tcp_create_openreq_child ->sk_clone_lock */
1091
1092         return newsk;
1093 free_csk:
1094         chtls_sock_release(&csk->kref);
1095 free_dst:
1096         dst_release(dst);
1097 free_sk:
1098         inet_csk_prepare_forced_close(newsk);
1099         tcp_done(newsk);
1100 free_oreq:
1101         chtls_reqsk_free(oreq);
1102         return NULL;
1103 }
1104
1105 /*
1106  * Populate a TID_RELEASE WR.  The skb must be already propely sized.
1107  */
1108 static  void mk_tid_release(struct sk_buff *skb,
1109                             unsigned int chan, unsigned int tid)
1110 {
1111         struct cpl_tid_release *req;
1112         unsigned int len;
1113
1114         len = roundup(sizeof(struct cpl_tid_release), 16);
1115         req = (struct cpl_tid_release *)__skb_put(skb, len);
1116         memset(req, 0, len);
1117         set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
1118         INIT_TP_WR_CPL(req, CPL_TID_RELEASE, tid);
1119 }
1120
1121 static int chtls_get_module(struct sock *sk)
1122 {
1123         struct inet_connection_sock *icsk = inet_csk(sk);
1124
1125         if (!try_module_get(icsk->icsk_ulp_ops->owner))
1126                 return -1;
1127
1128         return 0;
1129 }
1130
1131 static void chtls_pass_accept_request(struct sock *sk,
1132                                       struct sk_buff *skb)
1133 {
1134         struct cpl_t5_pass_accept_rpl *rpl;
1135         struct cpl_pass_accept_req *req;
1136         struct listen_ctx *listen_ctx;
1137         struct vlan_ethhdr *vlan_eh;
1138         struct request_sock *oreq;
1139         struct sk_buff *reply_skb;
1140         struct chtls_sock *csk;
1141         struct chtls_dev *cdev;
1142         struct tcphdr *tcph;
1143         struct sock *newsk;
1144         struct ethhdr *eh;
1145         struct iphdr *iph;
1146         void *network_hdr;
1147         unsigned int stid;
1148         unsigned int len;
1149         unsigned int tid;
1150         bool th_ecn, ect;
1151         __u8 ip_dsfield; /* IPv4 tos or IPv6 dsfield */
1152         u16 eth_hdr_len;
1153         bool ecn_ok;
1154
1155         req = cplhdr(skb) + RSS_HDR;
1156         tid = GET_TID(req);
1157         cdev = BLOG_SKB_CB(skb)->cdev;
1158         newsk = lookup_tid(cdev->tids, tid);
1159         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1160         if (newsk) {
1161                 pr_info("tid (%d) already in use\n", tid);
1162                 return;
1163         }
1164
1165         len = roundup(sizeof(*rpl), 16);
1166         reply_skb = alloc_skb(len, GFP_ATOMIC);
1167         if (!reply_skb) {
1168                 cxgb4_remove_tid(cdev->tids, 0, tid, sk->sk_family);
1169                 kfree_skb(skb);
1170                 return;
1171         }
1172
1173         if (sk->sk_state != TCP_LISTEN)
1174                 goto reject;
1175
1176         if (inet_csk_reqsk_queue_is_full(sk))
1177                 goto reject;
1178
1179         if (sk_acceptq_is_full(sk))
1180                 goto reject;
1181
1182         oreq = inet_reqsk_alloc(&chtls_rsk_ops, sk, true);
1183         if (!oreq)
1184                 goto reject;
1185
1186         oreq->rsk_rcv_wnd = 0;
1187         oreq->rsk_window_clamp = 0;
1188         oreq->cookie_ts = 0;
1189         oreq->mss = 0;
1190         oreq->ts_recent = 0;
1191
1192         eth_hdr_len = T6_ETH_HDR_LEN_G(ntohl(req->hdr_len));
1193         if (eth_hdr_len == ETH_HLEN) {
1194                 eh = (struct ethhdr *)(req + 1);
1195                 iph = (struct iphdr *)(eh + 1);
1196                 network_hdr = (void *)(eh + 1);
1197         } else {
1198                 vlan_eh = (struct vlan_ethhdr *)(req + 1);
1199                 iph = (struct iphdr *)(vlan_eh + 1);
1200                 network_hdr = (void *)(vlan_eh + 1);
1201         }
1202         if (iph->version != 0x4)
1203                 goto free_oreq;
1204
1205         tcph = (struct tcphdr *)(iph + 1);
1206         skb_set_network_header(skb, (void *)iph - (void *)req);
1207
1208         tcp_rsk(oreq)->tfo_listener = false;
1209         tcp_rsk(oreq)->rcv_isn = ntohl(tcph->seq);
1210         chtls_set_req_port(oreq, tcph->source, tcph->dest);
1211         chtls_set_req_addr(oreq, iph->daddr, iph->saddr);
1212         ip_dsfield = ipv4_get_dsfield(iph);
1213         if (req->tcpopt.wsf <= 14 &&
1214             sock_net(sk)->ipv4.sysctl_tcp_window_scaling) {
1215                 inet_rsk(oreq)->wscale_ok = 1;
1216                 inet_rsk(oreq)->snd_wscale = req->tcpopt.wsf;
1217         }
1218         inet_rsk(oreq)->ir_iif = sk->sk_bound_dev_if;
1219         th_ecn = tcph->ece && tcph->cwr;
1220         if (th_ecn) {
1221                 ect = !INET_ECN_is_not_ect(ip_dsfield);
1222                 ecn_ok = sock_net(sk)->ipv4.sysctl_tcp_ecn;
1223                 if ((!ect && ecn_ok) || tcp_ca_needs_ecn(sk))
1224                         inet_rsk(oreq)->ecn_ok = 1;
1225         }
1226
1227         newsk = chtls_recv_sock(sk, oreq, network_hdr, req, cdev);
1228         if (!newsk)
1229                 goto reject;
1230
1231         if (chtls_get_module(newsk))
1232                 goto reject;
1233         inet_csk_reqsk_queue_added(sk);
1234         reply_skb->sk = newsk;
1235         chtls_install_cpl_ops(newsk);
1236         cxgb4_insert_tid(cdev->tids, newsk, tid, newsk->sk_family);
1237         csk = rcu_dereference_sk_user_data(newsk);
1238         listen_ctx = (struct listen_ctx *)lookup_stid(cdev->tids, stid);
1239         csk->listen_ctx = listen_ctx;
1240         __skb_queue_tail(&listen_ctx->synq, (struct sk_buff *)&csk->synq);
1241         chtls_pass_accept_rpl(reply_skb, req, tid);
1242         kfree_skb(skb);
1243         return;
1244
1245 free_oreq:
1246         chtls_reqsk_free(oreq);
1247 reject:
1248         mk_tid_release(reply_skb, 0, tid);
1249         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1250         kfree_skb(skb);
1251 }
1252
1253 /*
1254  * Handle a CPL_PASS_ACCEPT_REQ message.
1255  */
1256 static int chtls_pass_accept_req(struct chtls_dev *cdev, struct sk_buff *skb)
1257 {
1258         struct cpl_pass_accept_req *req = cplhdr(skb) + RSS_HDR;
1259         struct listen_ctx *ctx;
1260         unsigned int stid;
1261         unsigned int tid;
1262         struct sock *lsk;
1263         void *data;
1264
1265         stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1266         tid = GET_TID(req);
1267
1268         data = lookup_stid(cdev->tids, stid);
1269         if (!data)
1270                 return 1;
1271
1272         ctx = (struct listen_ctx *)data;
1273         lsk = ctx->lsk;
1274
1275         if (unlikely(tid >= cdev->tids->ntids)) {
1276                 pr_info("passive open TID %u too large\n", tid);
1277                 return 1;
1278         }
1279
1280         BLOG_SKB_CB(skb)->cdev = cdev;
1281         process_cpl_msg(chtls_pass_accept_request, lsk, skb);
1282         return 0;
1283 }
1284
1285 /*
1286  * Completes some final bits of initialization for just established connections
1287  * and changes their state to TCP_ESTABLISHED.
1288  *
1289  * snd_isn here is the ISN after the SYN, i.e., the true ISN + 1.
1290  */
1291 static void make_established(struct sock *sk, u32 snd_isn, unsigned int opt)
1292 {
1293         struct tcp_sock *tp = tcp_sk(sk);
1294
1295         tp->pushed_seq = snd_isn;
1296         tp->write_seq = snd_isn;
1297         tp->snd_nxt = snd_isn;
1298         tp->snd_una = snd_isn;
1299         inet_sk(sk)->inet_id = tp->write_seq ^ jiffies;
1300         assign_rxopt(sk, opt);
1301
1302         if (tp->rcv_wnd > (RCV_BUFSIZ_M << 10))
1303                 tp->rcv_wup -= tp->rcv_wnd - (RCV_BUFSIZ_M << 10);
1304
1305         smp_mb();
1306         tcp_set_state(sk, TCP_ESTABLISHED);
1307 }
1308
1309 static void chtls_abort_conn(struct sock *sk, struct sk_buff *skb)
1310 {
1311         struct sk_buff *abort_skb;
1312
1313         abort_skb = alloc_skb(sizeof(struct cpl_abort_req), GFP_ATOMIC);
1314         if (abort_skb)
1315                 chtls_send_reset(sk, CPL_ABORT_SEND_RST, abort_skb);
1316 }
1317
1318 static struct sock *reap_list;
1319 static DEFINE_SPINLOCK(reap_list_lock);
1320
1321 /*
1322  * Process the reap list.
1323  */
1324 DECLARE_TASK_FUNC(process_reap_list, task_param)
1325 {
1326         spin_lock_bh(&reap_list_lock);
1327         while (reap_list) {
1328                 struct sock *sk = reap_list;
1329                 struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1330
1331                 reap_list = csk->passive_reap_next;
1332                 csk->passive_reap_next = NULL;
1333                 spin_unlock(&reap_list_lock);
1334                 sock_hold(sk);
1335
1336                 bh_lock_sock(sk);
1337                 chtls_abort_conn(sk, NULL);
1338                 sock_orphan(sk);
1339                 if (sk->sk_state == TCP_CLOSE)
1340                         inet_csk_destroy_sock(sk);
1341                 bh_unlock_sock(sk);
1342                 sock_put(sk);
1343                 spin_lock(&reap_list_lock);
1344         }
1345         spin_unlock_bh(&reap_list_lock);
1346 }
1347
1348 static DECLARE_WORK(reap_task, process_reap_list);
1349
1350 static void add_to_reap_list(struct sock *sk)
1351 {
1352         struct chtls_sock *csk = sk->sk_user_data;
1353
1354         local_bh_disable();
1355         bh_lock_sock(sk);
1356         release_tcp_port(sk); /* release the port immediately */
1357
1358         spin_lock(&reap_list_lock);
1359         csk->passive_reap_next = reap_list;
1360         reap_list = sk;
1361         if (!csk->passive_reap_next)
1362                 schedule_work(&reap_task);
1363         spin_unlock(&reap_list_lock);
1364         bh_unlock_sock(sk);
1365         local_bh_enable();
1366 }
1367
1368 static void add_pass_open_to_parent(struct sock *child, struct sock *lsk,
1369                                     struct chtls_dev *cdev)
1370 {
1371         struct request_sock *oreq;
1372         struct chtls_sock *csk;
1373
1374         if (lsk->sk_state != TCP_LISTEN)
1375                 return;
1376
1377         csk = child->sk_user_data;
1378         oreq = csk->passive_reap_next;
1379         csk->passive_reap_next = NULL;
1380
1381         reqsk_queue_removed(&inet_csk(lsk)->icsk_accept_queue, oreq);
1382         __skb_unlink((struct sk_buff *)&csk->synq, &csk->listen_ctx->synq);
1383
1384         if (sk_acceptq_is_full(lsk)) {
1385                 chtls_reqsk_free(oreq);
1386                 add_to_reap_list(child);
1387         } else {
1388                 refcount_set(&oreq->rsk_refcnt, 1);
1389                 inet_csk_reqsk_queue_add(lsk, oreq, child);
1390                 lsk->sk_data_ready(lsk);
1391         }
1392 }
1393
1394 static void bl_add_pass_open_to_parent(struct sock *lsk, struct sk_buff *skb)
1395 {
1396         struct sock *child = skb->sk;
1397
1398         skb->sk = NULL;
1399         add_pass_open_to_parent(child, lsk, BLOG_SKB_CB(skb)->cdev);
1400         kfree_skb(skb);
1401 }
1402
1403 static int chtls_pass_establish(struct chtls_dev *cdev, struct sk_buff *skb)
1404 {
1405         struct cpl_pass_establish *req = cplhdr(skb) + RSS_HDR;
1406         struct chtls_sock *csk;
1407         struct sock *lsk, *sk;
1408         unsigned int hwtid;
1409
1410         hwtid = GET_TID(req);
1411         sk = lookup_tid(cdev->tids, hwtid);
1412         if (!sk)
1413                 return (CPL_RET_UNKNOWN_TID | CPL_RET_BUF_DONE);
1414
1415         bh_lock_sock(sk);
1416         if (unlikely(sock_owned_by_user(sk))) {
1417                 kfree_skb(skb);
1418         } else {
1419                 unsigned int stid;
1420                 void *data;
1421
1422                 csk = sk->sk_user_data;
1423                 csk->wr_max_credits = 64;
1424                 csk->wr_credits = 64;
1425                 csk->wr_unacked = 0;
1426                 make_established(sk, ntohl(req->snd_isn), ntohs(req->tcp_opt));
1427                 stid = PASS_OPEN_TID_G(ntohl(req->tos_stid));
1428                 sk->sk_state_change(sk);
1429                 if (unlikely(sk->sk_socket))
1430                         sk_wake_async(sk, 0, POLL_OUT);
1431
1432                 data = lookup_stid(cdev->tids, stid);
1433                 lsk = ((struct listen_ctx *)data)->lsk;
1434
1435                 bh_lock_sock(lsk);
1436                 if (unlikely(skb_queue_empty(&csk->listen_ctx->synq))) {
1437                         /* removed from synq */
1438                         bh_unlock_sock(lsk);
1439                         kfree_skb(skb);
1440                         goto unlock;
1441                 }
1442
1443                 if (likely(!sock_owned_by_user(lsk))) {
1444                         kfree_skb(skb);
1445                         add_pass_open_to_parent(sk, lsk, cdev);
1446                 } else {
1447                         skb->sk = sk;
1448                         BLOG_SKB_CB(skb)->cdev = cdev;
1449                         BLOG_SKB_CB(skb)->backlog_rcv =
1450                                 bl_add_pass_open_to_parent;
1451                         __sk_add_backlog(lsk, skb);
1452                 }
1453                 bh_unlock_sock(lsk);
1454         }
1455 unlock:
1456         bh_unlock_sock(sk);
1457         return 0;
1458 }
1459
1460 /*
1461  * Handle receipt of an urgent pointer.
1462  */
1463 static void handle_urg_ptr(struct sock *sk, u32 urg_seq)
1464 {
1465         struct tcp_sock *tp = tcp_sk(sk);
1466
1467         urg_seq--;
1468         if (tp->urg_data && !after(urg_seq, tp->urg_seq))
1469                 return; /* duplicate pointer */
1470
1471         sk_send_sigurg(sk);
1472         if (tp->urg_seq == tp->copied_seq && tp->urg_data &&
1473             !sock_flag(sk, SOCK_URGINLINE) &&
1474             tp->copied_seq != tp->rcv_nxt) {
1475                 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1476
1477                 tp->copied_seq++;
1478                 if (skb && tp->copied_seq - ULP_SKB_CB(skb)->seq >= skb->len)
1479                         chtls_free_skb(sk, skb);
1480         }
1481
1482         tp->urg_data = TCP_URG_NOTYET;
1483         tp->urg_seq = urg_seq;
1484 }
1485
1486 static void check_sk_callbacks(struct chtls_sock *csk)
1487 {
1488         struct sock *sk = csk->sk;
1489
1490         if (unlikely(sk->sk_user_data &&
1491                      !csk_flag_nochk(csk, CSK_CALLBACKS_CHKD)))
1492                 csk_set_flag(csk, CSK_CALLBACKS_CHKD);
1493 }
1494
1495 /*
1496  * Handles Rx data that arrives in a state where the socket isn't accepting
1497  * new data.
1498  */
1499 static void handle_excess_rx(struct sock *sk, struct sk_buff *skb)
1500 {
1501         if (!csk_flag(sk, CSK_ABORT_SHUTDOWN))
1502                 chtls_abort_conn(sk, skb);
1503
1504         kfree_skb(skb);
1505 }
1506
1507 static void chtls_recv_data(struct sock *sk, struct sk_buff *skb)
1508 {
1509         struct cpl_rx_data *hdr = cplhdr(skb) + RSS_HDR;
1510         struct chtls_sock *csk;
1511         struct tcp_sock *tp;
1512
1513         csk = rcu_dereference_sk_user_data(sk);
1514         tp = tcp_sk(sk);
1515
1516         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1517                 handle_excess_rx(sk, skb);
1518                 return;
1519         }
1520
1521         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1522         ULP_SKB_CB(skb)->psh = hdr->psh;
1523         skb_ulp_mode(skb) = ULP_MODE_NONE;
1524
1525         skb_reset_transport_header(skb);
1526         __skb_pull(skb, sizeof(*hdr) + RSS_HDR);
1527         if (!skb->data_len)
1528                 __skb_trim(skb, ntohs(hdr->len));
1529
1530         if (unlikely(hdr->urg))
1531                 handle_urg_ptr(sk, tp->rcv_nxt + ntohs(hdr->urg));
1532         if (unlikely(tp->urg_data == TCP_URG_NOTYET &&
1533                      tp->urg_seq - tp->rcv_nxt < skb->len))
1534                 tp->urg_data = TCP_URG_VALID |
1535                                skb->data[tp->urg_seq - tp->rcv_nxt];
1536
1537         if (unlikely(hdr->dack_mode != csk->delack_mode)) {
1538                 csk->delack_mode = hdr->dack_mode;
1539                 csk->delack_seq = tp->rcv_nxt;
1540         }
1541
1542         tcp_hdr(skb)->fin = 0;
1543         tp->rcv_nxt += skb->len;
1544
1545         __skb_queue_tail(&sk->sk_receive_queue, skb);
1546
1547         if (!sock_flag(sk, SOCK_DEAD)) {
1548                 check_sk_callbacks(csk);
1549                 sk->sk_data_ready(sk);
1550         }
1551 }
1552
1553 static int chtls_rx_data(struct chtls_dev *cdev, struct sk_buff *skb)
1554 {
1555         struct cpl_rx_data *req = cplhdr(skb) + RSS_HDR;
1556         unsigned int hwtid = GET_TID(req);
1557         struct sock *sk;
1558
1559         sk = lookup_tid(cdev->tids, hwtid);
1560         if (unlikely(!sk)) {
1561                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1562                 return -EINVAL;
1563         }
1564         skb_dst_set(skb, NULL);
1565         process_cpl_msg(chtls_recv_data, sk, skb);
1566         return 0;
1567 }
1568
1569 static void chtls_recv_pdu(struct sock *sk, struct sk_buff *skb)
1570 {
1571         struct cpl_tls_data *hdr = cplhdr(skb);
1572         struct chtls_sock *csk;
1573         struct chtls_hws *tlsk;
1574         struct tcp_sock *tp;
1575
1576         csk = rcu_dereference_sk_user_data(sk);
1577         tlsk = &csk->tlshws;
1578         tp = tcp_sk(sk);
1579
1580         if (unlikely(sk->sk_shutdown & RCV_SHUTDOWN)) {
1581                 handle_excess_rx(sk, skb);
1582                 return;
1583         }
1584
1585         ULP_SKB_CB(skb)->seq = ntohl(hdr->seq);
1586         ULP_SKB_CB(skb)->flags = 0;
1587         skb_ulp_mode(skb) = ULP_MODE_TLS;
1588
1589         skb_reset_transport_header(skb);
1590         __skb_pull(skb, sizeof(*hdr));
1591         if (!skb->data_len)
1592                 __skb_trim(skb,
1593                            CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd)));
1594
1595         if (unlikely(tp->urg_data == TCP_URG_NOTYET && tp->urg_seq -
1596                      tp->rcv_nxt < skb->len))
1597                 tp->urg_data = TCP_URG_VALID |
1598                                skb->data[tp->urg_seq - tp->rcv_nxt];
1599
1600         tcp_hdr(skb)->fin = 0;
1601         tlsk->pldlen = CPL_TLS_DATA_LENGTH_G(ntohl(hdr->length_pkd));
1602         __skb_queue_tail(&tlsk->sk_recv_queue, skb);
1603 }
1604
1605 static int chtls_rx_pdu(struct chtls_dev *cdev, struct sk_buff *skb)
1606 {
1607         struct cpl_tls_data *req = cplhdr(skb);
1608         unsigned int hwtid = GET_TID(req);
1609         struct sock *sk;
1610
1611         sk = lookup_tid(cdev->tids, hwtid);
1612         if (unlikely(!sk)) {
1613                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1614                 return -EINVAL;
1615         }
1616         skb_dst_set(skb, NULL);
1617         process_cpl_msg(chtls_recv_pdu, sk, skb);
1618         return 0;
1619 }
1620
1621 static void chtls_set_hdrlen(struct sk_buff *skb, unsigned int nlen)
1622 {
1623         struct tlsrx_cmp_hdr *tls_cmp_hdr = cplhdr(skb);
1624
1625         skb->hdr_len = ntohs((__force __be16)tls_cmp_hdr->length);
1626         tls_cmp_hdr->length = ntohs((__force __be16)nlen);
1627 }
1628
1629 static void chtls_rx_hdr(struct sock *sk, struct sk_buff *skb)
1630 {
1631         struct tlsrx_cmp_hdr *tls_hdr_pkt;
1632         struct cpl_rx_tls_cmp *cmp_cpl;
1633         struct sk_buff *skb_rec;
1634         struct chtls_sock *csk;
1635         struct chtls_hws *tlsk;
1636         struct tcp_sock *tp;
1637
1638         cmp_cpl = cplhdr(skb);
1639         csk = rcu_dereference_sk_user_data(sk);
1640         tlsk = &csk->tlshws;
1641         tp = tcp_sk(sk);
1642
1643         ULP_SKB_CB(skb)->seq = ntohl(cmp_cpl->seq);
1644         ULP_SKB_CB(skb)->flags = 0;
1645
1646         skb_reset_transport_header(skb);
1647         __skb_pull(skb, sizeof(*cmp_cpl));
1648         tls_hdr_pkt = (struct tlsrx_cmp_hdr *)skb->data;
1649         if (tls_hdr_pkt->res_to_mac_error & TLSRX_HDR_PKT_ERROR_M)
1650                 tls_hdr_pkt->type = CONTENT_TYPE_ERROR;
1651         if (!skb->data_len)
1652                 __skb_trim(skb, TLS_HEADER_LENGTH);
1653
1654         tp->rcv_nxt +=
1655                 CPL_RX_TLS_CMP_PDULENGTH_G(ntohl(cmp_cpl->pdulength_length));
1656
1657         ULP_SKB_CB(skb)->flags |= ULPCB_FLAG_TLS_HDR;
1658         skb_rec = __skb_dequeue(&tlsk->sk_recv_queue);
1659         if (!skb_rec) {
1660                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1661         } else {
1662                 chtls_set_hdrlen(skb, tlsk->pldlen);
1663                 tlsk->pldlen = 0;
1664                 __skb_queue_tail(&sk->sk_receive_queue, skb);
1665                 __skb_queue_tail(&sk->sk_receive_queue, skb_rec);
1666         }
1667
1668         if (!sock_flag(sk, SOCK_DEAD)) {
1669                 check_sk_callbacks(csk);
1670                 sk->sk_data_ready(sk);
1671         }
1672 }
1673
1674 static int chtls_rx_cmp(struct chtls_dev *cdev, struct sk_buff *skb)
1675 {
1676         struct cpl_rx_tls_cmp *req = cplhdr(skb);
1677         unsigned int hwtid = GET_TID(req);
1678         struct sock *sk;
1679
1680         sk = lookup_tid(cdev->tids, hwtid);
1681         if (unlikely(!sk)) {
1682                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
1683                 return -EINVAL;
1684         }
1685         skb_dst_set(skb, NULL);
1686         process_cpl_msg(chtls_rx_hdr, sk, skb);
1687
1688         return 0;
1689 }
1690
1691 static void chtls_timewait(struct sock *sk)
1692 {
1693         struct tcp_sock *tp = tcp_sk(sk);
1694
1695         tp->rcv_nxt++;
1696         tp->rx_opt.ts_recent_stamp = ktime_get_seconds();
1697         tp->srtt_us = 0;
1698         tcp_time_wait(sk, TCP_TIME_WAIT, 0);
1699 }
1700
1701 static void chtls_peer_close(struct sock *sk, struct sk_buff *skb)
1702 {
1703         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
1704
1705         sk->sk_shutdown |= RCV_SHUTDOWN;
1706         sock_set_flag(sk, SOCK_DONE);
1707
1708         switch (sk->sk_state) {
1709         case TCP_SYN_RECV:
1710         case TCP_ESTABLISHED:
1711                 tcp_set_state(sk, TCP_CLOSE_WAIT);
1712                 break;
1713         case TCP_FIN_WAIT1:
1714                 tcp_set_state(sk, TCP_CLOSING);
1715                 break;
1716         case TCP_FIN_WAIT2:
1717                 chtls_release_resources(sk);
1718                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1719                         chtls_conn_done(sk);
1720                 else
1721                         chtls_timewait(sk);
1722                 break;
1723         default:
1724                 pr_info("cpl_peer_close in bad state %d\n", sk->sk_state);
1725         }
1726
1727         if (!sock_flag(sk, SOCK_DEAD)) {
1728                 sk->sk_state_change(sk);
1729                 /* Do not send POLL_HUP for half duplex close. */
1730
1731                 if ((sk->sk_shutdown & SEND_SHUTDOWN) ||
1732                     sk->sk_state == TCP_CLOSE)
1733                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_HUP);
1734                 else
1735                         sk_wake_async(sk, SOCK_WAKE_WAITD, POLL_IN);
1736         }
1737 }
1738
1739 static void chtls_close_con_rpl(struct sock *sk, struct sk_buff *skb)
1740 {
1741         struct cpl_close_con_rpl *rpl = cplhdr(skb) + RSS_HDR;
1742         struct chtls_sock *csk;
1743         struct tcp_sock *tp;
1744
1745         csk = rcu_dereference_sk_user_data(sk);
1746         tp = tcp_sk(sk);
1747
1748         tp->snd_una = ntohl(rpl->snd_nxt) - 1;  /* exclude FIN */
1749
1750         switch (sk->sk_state) {
1751         case TCP_CLOSING:
1752                 chtls_release_resources(sk);
1753                 if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING))
1754                         chtls_conn_done(sk);
1755                 else
1756                         chtls_timewait(sk);
1757                 break;
1758         case TCP_LAST_ACK:
1759                 chtls_release_resources(sk);
1760                 chtls_conn_done(sk);
1761                 break;
1762         case TCP_FIN_WAIT1:
1763                 tcp_set_state(sk, TCP_FIN_WAIT2);
1764                 sk->sk_shutdown |= SEND_SHUTDOWN;
1765
1766                 if (!sock_flag(sk, SOCK_DEAD))
1767                         sk->sk_state_change(sk);
1768                 else if (tcp_sk(sk)->linger2 < 0 &&
1769                          !csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN))
1770                         chtls_abort_conn(sk, skb);
1771                 break;
1772         default:
1773                 pr_info("close_con_rpl in bad state %d\n", sk->sk_state);
1774         }
1775         kfree_skb(skb);
1776 }
1777
1778 static struct sk_buff *get_cpl_skb(struct sk_buff *skb,
1779                                    size_t len, gfp_t gfp)
1780 {
1781         if (likely(!skb_is_nonlinear(skb) && !skb_cloned(skb))) {
1782                 WARN_ONCE(skb->len < len, "skb alloc error");
1783                 __skb_trim(skb, len);
1784                 skb_get(skb);
1785         } else {
1786                 skb = alloc_skb(len, gfp);
1787                 if (skb)
1788                         __skb_put(skb, len);
1789         }
1790         return skb;
1791 }
1792
1793 static void set_abort_rpl_wr(struct sk_buff *skb, unsigned int tid,
1794                              int cmd)
1795 {
1796         struct cpl_abort_rpl *rpl = cplhdr(skb);
1797
1798         INIT_TP_WR_CPL(rpl, CPL_ABORT_RPL, tid);
1799         rpl->cmd = cmd;
1800 }
1801
1802 static void send_defer_abort_rpl(struct chtls_dev *cdev, struct sk_buff *skb)
1803 {
1804         struct cpl_abort_req_rss *req = cplhdr(skb);
1805         struct sk_buff *reply_skb;
1806
1807         reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1808                               GFP_KERNEL | __GFP_NOFAIL);
1809         __skb_put(reply_skb, sizeof(struct cpl_abort_rpl));
1810         set_abort_rpl_wr(reply_skb, GET_TID(req),
1811                          (req->status & CPL_ABORT_NO_RST));
1812         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, req->status >> 1);
1813         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1814         kfree_skb(skb);
1815 }
1816
1817 static void send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1818                            struct chtls_dev *cdev, int status, int queue)
1819 {
1820         struct cpl_abort_req_rss *req = cplhdr(skb);
1821         struct sk_buff *reply_skb;
1822         struct chtls_sock *csk;
1823
1824         csk = rcu_dereference_sk_user_data(sk);
1825
1826         reply_skb = alloc_skb(sizeof(struct cpl_abort_rpl),
1827                               GFP_KERNEL);
1828
1829         if (!reply_skb) {
1830                 req->status = (queue << 1);
1831                 send_defer_abort_rpl(cdev, skb);
1832                 return;
1833         }
1834
1835         set_abort_rpl_wr(reply_skb, GET_TID(req), status);
1836         kfree_skb(skb);
1837
1838         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1839         if (csk_conn_inline(csk)) {
1840                 struct l2t_entry *e = csk->l2t_entry;
1841
1842                 if (e && sk->sk_state != TCP_SYN_RECV) {
1843                         cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
1844                         return;
1845                 }
1846         }
1847         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1848 }
1849
1850 /*
1851  * Add an skb to the deferred skb queue for processing from process context.
1852  */
1853 static void t4_defer_reply(struct sk_buff *skb, struct chtls_dev *cdev,
1854                            defer_handler_t handler)
1855 {
1856         DEFERRED_SKB_CB(skb)->handler = handler;
1857         spin_lock_bh(&cdev->deferq.lock);
1858         __skb_queue_tail(&cdev->deferq, skb);
1859         if (skb_queue_len(&cdev->deferq) == 1)
1860                 schedule_work(&cdev->deferq_task);
1861         spin_unlock_bh(&cdev->deferq.lock);
1862 }
1863
1864 static void chtls_send_abort_rpl(struct sock *sk, struct sk_buff *skb,
1865                                  struct chtls_dev *cdev,
1866                                  int status, int queue)
1867 {
1868         struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1869         struct sk_buff *reply_skb;
1870         struct chtls_sock *csk;
1871         unsigned int tid;
1872
1873         csk = rcu_dereference_sk_user_data(sk);
1874         tid = GET_TID(req);
1875
1876         reply_skb = get_cpl_skb(skb, sizeof(struct cpl_abort_rpl), gfp_any());
1877         if (!reply_skb) {
1878                 req->status = (queue << 1) | status;
1879                 t4_defer_reply(skb, cdev, send_defer_abort_rpl);
1880                 return;
1881         }
1882
1883         set_abort_rpl_wr(reply_skb, tid, status);
1884         set_wr_txq(reply_skb, CPL_PRIORITY_DATA, queue);
1885         if (csk_conn_inline(csk)) {
1886                 struct l2t_entry *e = csk->l2t_entry;
1887
1888                 if (e && sk->sk_state != TCP_SYN_RECV) {
1889                         cxgb4_l2t_send(csk->egress_dev, reply_skb, e);
1890                         return;
1891                 }
1892         }
1893         cxgb4_ofld_send(cdev->lldi->ports[0], reply_skb);
1894         kfree_skb(skb);
1895 }
1896
1897 /*
1898  * This is run from a listener's backlog to abort a child connection in
1899  * SYN_RCV state (i.e., one on the listener's SYN queue).
1900  */
1901 static void bl_abort_syn_rcv(struct sock *lsk, struct sk_buff *skb)
1902 {
1903         struct chtls_sock *csk;
1904         struct sock *child;
1905         int queue;
1906
1907         child = skb->sk;
1908         csk = rcu_dereference_sk_user_data(child);
1909         queue = csk->txq_idx;
1910
1911         skb->sk = NULL;
1912         do_abort_syn_rcv(child, lsk);
1913         send_abort_rpl(child, skb, BLOG_SKB_CB(skb)->cdev,
1914                        CPL_ABORT_NO_RST, queue);
1915 }
1916
1917 static int abort_syn_rcv(struct sock *sk, struct sk_buff *skb)
1918 {
1919         const struct request_sock *oreq;
1920         struct listen_ctx *listen_ctx;
1921         struct chtls_sock *csk;
1922         struct chtls_dev *cdev;
1923         struct sock *psk;
1924         void *ctx;
1925
1926         csk = sk->sk_user_data;
1927         oreq = csk->passive_reap_next;
1928         cdev = csk->cdev;
1929
1930         if (!oreq)
1931                 return -1;
1932
1933         ctx = lookup_stid(cdev->tids, oreq->ts_recent);
1934         if (!ctx)
1935                 return -1;
1936
1937         listen_ctx = (struct listen_ctx *)ctx;
1938         psk = listen_ctx->lsk;
1939
1940         bh_lock_sock(psk);
1941         if (!sock_owned_by_user(psk)) {
1942                 int queue = csk->txq_idx;
1943
1944                 do_abort_syn_rcv(sk, psk);
1945                 send_abort_rpl(sk, skb, cdev, CPL_ABORT_NO_RST, queue);
1946         } else {
1947                 skb->sk = sk;
1948                 BLOG_SKB_CB(skb)->backlog_rcv = bl_abort_syn_rcv;
1949                 __sk_add_backlog(psk, skb);
1950         }
1951         bh_unlock_sock(psk);
1952         return 0;
1953 }
1954
1955 static void chtls_abort_req_rss(struct sock *sk, struct sk_buff *skb)
1956 {
1957         const struct cpl_abort_req_rss *req = cplhdr(skb) + RSS_HDR;
1958         struct chtls_sock *csk = sk->sk_user_data;
1959         int rst_status = CPL_ABORT_NO_RST;
1960         int queue = csk->txq_idx;
1961
1962         if (is_neg_adv(req->status)) {
1963                 if (sk->sk_state == TCP_SYN_RECV)
1964                         chtls_set_tcb_tflag(sk, 0, 0);
1965
1966                 kfree_skb(skb);
1967                 return;
1968         }
1969
1970         csk_reset_flag(csk, CSK_ABORT_REQ_RCVD);
1971
1972         if (!csk_flag_nochk(csk, CSK_ABORT_SHUTDOWN) &&
1973             !csk_flag_nochk(csk, CSK_TX_DATA_SENT)) {
1974                 struct tcp_sock *tp = tcp_sk(sk);
1975
1976                 if (send_tx_flowc_wr(sk, 0, tp->snd_nxt, tp->rcv_nxt) < 0)
1977                         WARN_ONCE(1, "send_tx_flowc error");
1978                 csk_set_flag(csk, CSK_TX_DATA_SENT);
1979         }
1980
1981         csk_set_flag(csk, CSK_ABORT_SHUTDOWN);
1982
1983         if (!csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
1984                 sk->sk_err = ETIMEDOUT;
1985
1986                 if (!sock_flag(sk, SOCK_DEAD))
1987                         sk->sk_error_report(sk);
1988
1989                 if (sk->sk_state == TCP_SYN_RECV && !abort_syn_rcv(sk, skb))
1990                         return;
1991
1992                 chtls_release_resources(sk);
1993                 chtls_conn_done(sk);
1994         }
1995
1996         chtls_send_abort_rpl(sk, skb, csk->cdev, rst_status, queue);
1997 }
1998
1999 static void chtls_abort_rpl_rss(struct sock *sk, struct sk_buff *skb)
2000 {
2001         struct cpl_abort_rpl_rss *rpl = cplhdr(skb) + RSS_HDR;
2002         struct chtls_sock *csk;
2003         struct chtls_dev *cdev;
2004
2005         csk = rcu_dereference_sk_user_data(sk);
2006         cdev = csk->cdev;
2007
2008         if (csk_flag_nochk(csk, CSK_ABORT_RPL_PENDING)) {
2009                 csk_reset_flag(csk, CSK_ABORT_RPL_PENDING);
2010                 if (!csk_flag_nochk(csk, CSK_ABORT_REQ_RCVD)) {
2011                         if (sk->sk_state == TCP_SYN_SENT) {
2012                                 cxgb4_remove_tid(cdev->tids,
2013                                                  csk->port_id,
2014                                                  GET_TID(rpl),
2015                                                  sk->sk_family);
2016                                 sock_put(sk);
2017                         }
2018                         chtls_release_resources(sk);
2019                         chtls_conn_done(sk);
2020                 }
2021         }
2022         kfree_skb(skb);
2023 }
2024
2025 static int chtls_conn_cpl(struct chtls_dev *cdev, struct sk_buff *skb)
2026 {
2027         struct cpl_peer_close *req = cplhdr(skb) + RSS_HDR;
2028         void (*fn)(struct sock *sk, struct sk_buff *skb);
2029         unsigned int hwtid = GET_TID(req);
2030         struct sock *sk;
2031         u8 opcode;
2032
2033         opcode = ((const struct rss_header *)cplhdr(skb))->opcode;
2034
2035         sk = lookup_tid(cdev->tids, hwtid);
2036         if (!sk)
2037                 goto rel_skb;
2038
2039         switch (opcode) {
2040         case CPL_PEER_CLOSE:
2041                 fn = chtls_peer_close;
2042                 break;
2043         case CPL_CLOSE_CON_RPL:
2044                 fn = chtls_close_con_rpl;
2045                 break;
2046         case CPL_ABORT_REQ_RSS:
2047                 fn = chtls_abort_req_rss;
2048                 break;
2049         case CPL_ABORT_RPL_RSS:
2050                 fn = chtls_abort_rpl_rss;
2051                 break;
2052         default:
2053                 goto rel_skb;
2054         }
2055
2056         process_cpl_msg(fn, sk, skb);
2057         return 0;
2058
2059 rel_skb:
2060         kfree_skb(skb);
2061         return 0;
2062 }
2063
2064 static struct sk_buff *dequeue_wr(struct sock *sk)
2065 {
2066         struct chtls_sock *csk = rcu_dereference_sk_user_data(sk);
2067         struct sk_buff *skb = csk->wr_skb_head;
2068
2069         if (likely(skb)) {
2070         /* Don't bother clearing the tail */
2071                 csk->wr_skb_head = WR_SKB_CB(skb)->next_wr;
2072                 WR_SKB_CB(skb)->next_wr = NULL;
2073         }
2074         return skb;
2075 }
2076
2077 static void chtls_rx_ack(struct sock *sk, struct sk_buff *skb)
2078 {
2079         struct cpl_fw4_ack *hdr = cplhdr(skb) + RSS_HDR;
2080         struct chtls_sock *csk = sk->sk_user_data;
2081         struct tcp_sock *tp = tcp_sk(sk);
2082         u32 credits = hdr->credits;
2083         u32 snd_una;
2084
2085         snd_una = ntohl(hdr->snd_una);
2086         csk->wr_credits += credits;
2087
2088         if (csk->wr_unacked > csk->wr_max_credits - csk->wr_credits)
2089                 csk->wr_unacked = csk->wr_max_credits - csk->wr_credits;
2090
2091         while (credits) {
2092                 struct sk_buff *pskb = csk->wr_skb_head;
2093                 u32 csum;
2094
2095                 if (unlikely(!pskb)) {
2096                         if (csk->wr_nondata)
2097                                 csk->wr_nondata -= credits;
2098                         break;
2099                 }
2100                 csum = (__force u32)pskb->csum;
2101                 if (unlikely(credits < csum)) {
2102                         pskb->csum = (__force __wsum)(csum - credits);
2103                         break;
2104                 }
2105                 dequeue_wr(sk);
2106                 credits -= csum;
2107                 kfree_skb(pskb);
2108         }
2109         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_SEQVAL) {
2110                 if (unlikely(before(snd_una, tp->snd_una))) {
2111                         kfree_skb(skb);
2112                         return;
2113                 }
2114
2115                 if (tp->snd_una != snd_una) {
2116                         tp->snd_una = snd_una;
2117                         tp->rcv_tstamp = tcp_time_stamp(tp);
2118                         if (tp->snd_una == tp->snd_nxt &&
2119                             !csk_flag_nochk(csk, CSK_TX_FAILOVER))
2120                                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2121                 }
2122         }
2123
2124         if (hdr->seq_vld & CPL_FW4_ACK_FLAGS_CH) {
2125                 unsigned int fclen16 = roundup(failover_flowc_wr_len, 16);
2126
2127                 csk->wr_credits -= fclen16;
2128                 csk_reset_flag(csk, CSK_TX_WAIT_IDLE);
2129                 csk_reset_flag(csk, CSK_TX_FAILOVER);
2130         }
2131         if (skb_queue_len(&csk->txq) && chtls_push_frames(csk, 0))
2132                 sk->sk_write_space(sk);
2133
2134         kfree_skb(skb);
2135 }
2136
2137 static int chtls_wr_ack(struct chtls_dev *cdev, struct sk_buff *skb)
2138 {
2139         struct cpl_fw4_ack *rpl = cplhdr(skb) + RSS_HDR;
2140         unsigned int hwtid = GET_TID(rpl);
2141         struct sock *sk;
2142
2143         sk = lookup_tid(cdev->tids, hwtid);
2144         if (unlikely(!sk)) {
2145                 pr_err("can't find conn. for hwtid %u.\n", hwtid);
2146                 return -EINVAL;
2147         }
2148         process_cpl_msg(chtls_rx_ack, sk, skb);
2149
2150         return 0;
2151 }
2152
2153 chtls_handler_func chtls_handlers[NUM_CPL_CMDS] = {
2154         [CPL_PASS_OPEN_RPL]     = chtls_pass_open_rpl,
2155         [CPL_CLOSE_LISTSRV_RPL] = chtls_close_listsrv_rpl,
2156         [CPL_PASS_ACCEPT_REQ]   = chtls_pass_accept_req,
2157         [CPL_PASS_ESTABLISH]    = chtls_pass_establish,
2158         [CPL_RX_DATA]           = chtls_rx_data,
2159         [CPL_TLS_DATA]          = chtls_rx_pdu,
2160         [CPL_RX_TLS_CMP]        = chtls_rx_cmp,
2161         [CPL_PEER_CLOSE]        = chtls_conn_cpl,
2162         [CPL_CLOSE_CON_RPL]     = chtls_conn_cpl,
2163         [CPL_ABORT_REQ_RSS]     = chtls_conn_cpl,
2164         [CPL_ABORT_RPL_RSS]     = chtls_conn_cpl,
2165         [CPL_FW4_ACK]           = chtls_wr_ack,
2166 };