drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
   3
   4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6 #include <linux/skbuff.h>
   7 #include <linux/module.h>
   8 #include <linux/highmem.h>
   9 #include <linux/ip.h>
  10 #include <net/ipv6.h>
  11 #include <linux/netdevice.h>
  12 #include <crypto/aes.h>
  13 #include "chcr_ktls.h"
  14
  15 static LIST_HEAD(uld_ctx_list);
  16 static DEFINE_MUTEX(dev_mutex);
  17
  18 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
  19  * @skb: skb
  20  * @start: start offset.
  21  * @len: how much data to send after @start
  22  */
  23 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
  24 {
  25         struct skb_shared_info *si = skb_shinfo(skb);
  26         u32 frag_size, skb_linear_data_len = skb_headlen(skb);
  27         u8 nfrags = 0, frag_idx = 0;
  28         skb_frag_t *frag;
  29
  30         /* if its a linear skb then return 1 */
  31         if (!skb_is_nonlinear(skb))
  32                 return 1;
  33
  34         if (unlikely(start < skb_linear_data_len)) {
  35                 frag_size = min(len, skb_linear_data_len - start);
  36                 start = 0;
  37         } else {
  38                 start -= skb_linear_data_len;
  39
  40                 frag = &si->frags[frag_idx];
  41                 frag_size = skb_frag_size(frag);
  42                 while (start >= frag_size) {
  43                         start -= frag_size;
  44                         frag_idx++;
  45                         frag = &si->frags[frag_idx];
  46                         frag_size = skb_frag_size(frag);
  47                 }
  48                 frag_size = min(len, skb_frag_size(frag) - start);
  49         }
  50         len -= frag_size;
  51         nfrags++;
  52
  53         while (len) {
  54                 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
  55                 len -= frag_size;
  56                 nfrags++;
  57                 frag_idx++;
  58         }
  59         return nfrags;
  60 }
  61
  62 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
  63 /*
  64  * chcr_ktls_save_keys: calculate and save crypto keys.
  65  * @tx_info - driver specific tls info.
  66  * @crypto_info - tls crypto information.
  67  * @direction - TX/RX direction.
  68  * return - SUCCESS/FAILURE.
  69  */
  70 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
  71                                struct tls_crypto_info *crypto_info,
  72                                enum tls_offload_ctx_dir direction)
  73 {
  74         int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
  75         unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
  76         struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
  77         struct ktls_key_ctx *kctx = &tx_info->key_ctx;
  78         struct crypto_aes_ctx aes_ctx;
  79         unsigned char *key, *salt;
  80
  81         switch (crypto_info->cipher_type) {
  82         case TLS_CIPHER_AES_GCM_128:
  83                 info_128_gcm =
  84                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
  85                 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
  86                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
  87                 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
  88                 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
  89                 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  90                 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
  91
  92                 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
  93                 key = info_128_gcm->key;
  94                 salt = info_128_gcm->salt;
  95                 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
  96
  97                 /* The SCMD fields used when encrypting a full TLS
  98                  * record. Its a one time calculation till the
  99                  * connection exists.
 100                  */
 101                 tx_info->scmd0_seqno_numivs =
 102                         SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
 103                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
 104                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
 105                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
 106                         SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
 107                         SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
 108                         SCMD_NUM_IVS_V(1);
 109
 110                 /* keys will be sent inline. */
 111                 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
 112
 113                 /* The SCMD fields used when encrypting a partial TLS
 114                  * record (no trailer and possibly a truncated payload).
 115                  */
 116                 tx_info->scmd0_short_seqno_numivs =
 117                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
 118                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
 119                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
 120                         SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
 121
 122                 tx_info->scmd0_short_ivgen_hdrlen =
 123                         tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
 124
 125                 break;
 126
 127         default:
 128                 pr_err("GCM: cipher type 0x%x not supported\n",
 129                        crypto_info->cipher_type);
 130                 ret = -EINVAL;
 131                 goto out;
 132         }
 133
 134         key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
 135                        roundup(keylen, 16) + ghash_size;
 136         /* Calculate the H = CIPH(K, 0 repeated 16 times).
 137          * It will go in key context
 138          */
 139
 140         ret = aes_expandkey(&aes_ctx, key, keylen);
 141         if (ret)
 142                 goto out;
 143
 144         memset(ghash_h, 0, ghash_size);
 145         aes_encrypt(&aes_ctx, ghash_h, ghash_h);
 146         memzero_explicit(&aes_ctx, sizeof(aes_ctx));
 147
 148         /* fill the Key context */
 149         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 150                 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
 151                                                  mac_key_size,
 152                                                  key_ctx_size >> 4);
 153         } else {
 154                 ret = -EINVAL;
 155                 goto out;
 156         }
 157
 158         memcpy(kctx->salt, salt, tx_info->salt_size);
 159         memcpy(kctx->key, key, keylen);
 160         memcpy(kctx->key + keylen, ghash_h, ghash_size);
 161         tx_info->key_ctx_len = key_ctx_size;
 162
 163 out:
 164         return ret;
 165 }
 166
 167 /*
 168  * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
 169  * @sk - tcp socket.
 170  * @tx_info - driver specific tls info.
 171  * @atid - connection active tid.
 172  * return - send success/failure.
 173  */
 174 static int chcr_ktls_act_open_req(struct sock *sk,
 175                                   struct chcr_ktls_info *tx_info,
 176                                   int atid)
 177 {
 178         struct inet_sock *inet = inet_sk(sk);
 179         struct cpl_t6_act_open_req *cpl6;
 180         struct cpl_act_open_req *cpl;
 181         struct sk_buff *skb;
 182         unsigned int len;
 183         int qid_atid;
 184         u64 options;
 185
 186         len = sizeof(*cpl6);
 187         skb = alloc_skb(len, GFP_KERNEL);
 188         if (unlikely(!skb))
 189                 return -ENOMEM;
 190         /* mark it a control pkt */
 191         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 192
 193         cpl6 = __skb_put_zero(skb, len);
 194         cpl = (struct cpl_act_open_req *)cpl6;
 195         INIT_TP_WR(cpl6, 0);
 196         qid_atid = TID_QID_V(tx_info->rx_qid) |
 197                    TID_TID_V(atid);
 198         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
 199         cpl->local_port = inet->inet_sport;
 200         cpl->peer_port = inet->inet_dport;
 201         cpl->local_ip = inet->inet_rcv_saddr;
 202         cpl->peer_ip = inet->inet_daddr;
 203
 204         /* fill first 64 bit option field. */
 205         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 206                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 207         cpl->opt0 = cpu_to_be64(options);
 208
 209         /* next 64 bit option field. */
 210         options =
 211                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 212         cpl->opt2 = htonl(options);
 213
 214         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 215 }
 216
 217 #if IS_ENABLED(CONFIG_IPV6)
 218 /*
 219  * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
 220  * @sk - tcp socket.
 221  * @tx_info - driver specific tls info.
 222  * @atid - connection active tid.
 223  * return - send success/failure.
 224  */
 225 static int chcr_ktls_act_open_req6(struct sock *sk,
 226                                    struct chcr_ktls_info *tx_info,
 227                                    int atid)
 228 {
 229         struct inet_sock *inet = inet_sk(sk);
 230         struct cpl_t6_act_open_req6 *cpl6;
 231         struct cpl_act_open_req6 *cpl;
 232         struct sk_buff *skb;
 233         unsigned int len;
 234         int qid_atid;
 235         u64 options;
 236
 237         len = sizeof(*cpl6);
 238         skb = alloc_skb(len, GFP_KERNEL);
 239         if (unlikely(!skb))
 240                 return -ENOMEM;
 241         /* mark it a control pkt */
 242         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 243
 244         cpl6 = __skb_put_zero(skb, len);
 245         cpl = (struct cpl_act_open_req6 *)cpl6;
 246         INIT_TP_WR(cpl6, 0);
 247         qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
 248         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
 249         cpl->local_port = inet->inet_sport;
 250         cpl->peer_port = inet->inet_dport;
 251         cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
 252         cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
 253         cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
 254         cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
 255
 256         /* first 64 bit option field. */
 257         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 258                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 259         cpl->opt0 = cpu_to_be64(options);
 260         /* next 64 bit option field. */
 261         options =
 262                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 263         cpl->opt2 = htonl(options);
 264
 265         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 266 }
 267 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
 268
 269 /*
 270  * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
 271  * @sk - tcp socket.
 272  * @tx_info - driver specific tls info.
 273  * return: NET_TX_OK/NET_XMIT_DROP
 274  */
 275 static int chcr_setup_connection(struct sock *sk,
 276                                  struct chcr_ktls_info *tx_info)
 277 {
 278         struct tid_info *t = &tx_info->adap->tids;
 279         int atid, ret = 0;
 280
 281         atid = cxgb4_alloc_atid(t, tx_info);
 282         if (atid == -1)
 283                 return -EINVAL;
 284
 285         tx_info->atid = atid;
 286
 287         if (tx_info->ip_family == AF_INET) {
 288                 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
 289 #if IS_ENABLED(CONFIG_IPV6)
 290         } else {
 291                 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
 292                                      &sk->sk_v6_rcv_saddr,
 293                                      1);
 294                 if (ret)
 295                         return ret;
 296                 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
 297 #endif
 298         }
 299
 300         /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
 301          * success, if any other return type clear atid and return that failure.
 302          */
 303         if (ret) {
 304                 if (ret == NET_XMIT_CN) {
 305                         ret = 0;
 306                 } else {
 307 #if IS_ENABLED(CONFIG_IPV6)
 308                         /* clear clip entry */
 309                         if (tx_info->ip_family == AF_INET6)
 310                                 cxgb4_clip_release(tx_info->netdev,
 311                                                    (const u32 *)
 312                                                    &sk->sk_v6_rcv_saddr,
 313                                                    1);
 314 #endif
 315                         cxgb4_free_atid(t, atid);
 316                 }
 317         }
 318
 319         return ret;
 320 }
 321
 322 /*
 323  * chcr_set_tcb_field: update tcb fields.
 324  * @tx_info - driver specific tls info.
 325  * @word - TCB word.
 326  * @mask - TCB word related mask.
 327  * @val - TCB word related value.
 328  * @no_reply - set 1 if not looking for TP response.
 329  */
 330 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
 331                               u64 mask, u64 val, int no_reply)
 332 {
 333         struct cpl_set_tcb_field *req;
 334         struct sk_buff *skb;
 335
 336         skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
 337         if (!skb)
 338                 return -ENOMEM;
 339
 340         req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
 341         INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
 342         req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 343                                 NO_REPLY_V(no_reply));
 344         req->word_cookie = htons(TCB_WORD_V(word));
 345         req->mask = cpu_to_be64(mask);
 346         req->val = cpu_to_be64(val);
 347
 348         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 349         return cxgb4_ofld_send(tx_info->netdev, skb);
 350 }
 351
 352 /*
 353  * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
 354  * @tx_info - driver specific tls info.
 355  * return: NET_TX_OK/NET_XMIT_DROP.
 356  */
 357 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
 358 {
 359         return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
 360                                   TCB_T_STATE_V(TCB_T_STATE_M),
 361                                   CHCR_TCB_STATE_CLOSED, 1);
 362 }
 363
 364 /*
 365  * chcr_ktls_dev_del:  call back for tls_dev_del.
 366  * Remove the tid and l2t entry and close the connection.
 367  * it per connection basis.
 368  * @netdev - net device.
 369  * @tls_cts - tls context.
 370  * @direction - TX/RX crypto direction
 371  */
 372 static void chcr_ktls_dev_del(struct net_device *netdev,
 373                               struct tls_context *tls_ctx,
 374                               enum tls_offload_ctx_dir direction)
 375 {
 376         struct chcr_ktls_ofld_ctx_tx *tx_ctx =
 377                                 chcr_get_ktls_tx_context(tls_ctx);
 378         struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
 379         struct ch_ktls_port_stats_debug *port_stats;
 380
 381         if (!tx_info)
 382                 return;
 383
 384         /* clear l2t entry */
 385         if (tx_info->l2te)
 386                 cxgb4_l2t_release(tx_info->l2te);
 387
 388 #if IS_ENABLED(CONFIG_IPV6)
 389         /* clear clip entry */
 390         if (tx_info->ip_family == AF_INET6)
 391                 cxgb4_clip_release(netdev, (const u32 *)
 392                                    &tx_info->sk->sk_v6_rcv_saddr,
 393                                    1);
 394 #endif
 395
 396         /* clear tid */
 397         if (tx_info->tid != -1) {
 398                 /* clear tcb state and then release tid */
 399                 chcr_ktls_mark_tcb_close(tx_info);
 400                 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 401                                  tx_info->tid, tx_info->ip_family);
 402         }
 403
 404         port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
 405         atomic64_inc(&port_stats->ktls_tx_connection_close);
 406         kvfree(tx_info);
 407         tx_ctx->chcr_info = NULL;
 408         /* release module refcount */
 409         module_put(THIS_MODULE);
 410 }
 411
 412 /*
 413  * chcr_ktls_dev_add:  call back for tls_dev_add.
 414  * Create a tcb entry for TP. Also add l2t entry for the connection. And
 415  * generate keys & save those keys locally.
 416  * @netdev - net device.
 417  * @tls_cts - tls context.
 418  * @direction - TX/RX crypto direction
 419  * return: SUCCESS/FAILURE.
 420  */
 421 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
 422                              enum tls_offload_ctx_dir direction,
 423                              struct tls_crypto_info *crypto_info,
 424                              u32 start_offload_tcp_sn)
 425 {
 426         struct tls_context *tls_ctx = tls_get_ctx(sk);
 427         struct ch_ktls_port_stats_debug *port_stats;
 428         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
 429         struct chcr_ktls_info *tx_info;
 430         struct dst_entry *dst;
 431         struct adapter *adap;
 432         struct port_info *pi;
 433         struct neighbour *n;
 434         u8 daaddr[16];
 435         int ret = -1;
 436
 437         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
 438
 439         pi = netdev_priv(netdev);
 440         adap = pi->adapter;
 441         port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
 442         atomic64_inc(&port_stats->ktls_tx_connection_open);
 443
 444         if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
 445                 pr_err("not expecting for RX direction\n");
 446                 goto out;
 447         }
 448
 449         if (tx_ctx->chcr_info)
 450                 goto out;
 451
 452         tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
 453         if (!tx_info)
 454                 goto out;
 455
 456         tx_info->sk = sk;
 457         spin_lock_init(&tx_info->lock);
 458         /* initialize tid and atid to -1, 0 is a also a valid id. */
 459         tx_info->tid = -1;
 460         tx_info->atid = -1;
 461
 462         tx_info->adap = adap;
 463         tx_info->netdev = netdev;
 464         tx_info->first_qset = pi->first_qset;
 465         tx_info->tx_chan = pi->tx_chan;
 466         tx_info->smt_idx = pi->smt_idx;
 467         tx_info->port_id = pi->port_id;
 468         tx_info->prev_ack = 0;
 469         tx_info->prev_win = 0;
 470
 471         tx_info->rx_qid = chcr_get_first_rx_qid(adap);
 472         if (unlikely(tx_info->rx_qid < 0))
 473                 goto free_tx_info;
 474
 475         tx_info->prev_seq = start_offload_tcp_sn;
 476         tx_info->tcp_start_seq_number = start_offload_tcp_sn;
 477
 478         /* save crypto keys */
 479         ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
 480         if (ret < 0)
 481                 goto free_tx_info;
 482
 483         /* get peer ip */
 484         if (sk->sk_family == AF_INET) {
 485                 memcpy(daaddr, &sk->sk_daddr, 4);
 486                 tx_info->ip_family = AF_INET;
 487 #if IS_ENABLED(CONFIG_IPV6)
 488         } else {
 489                 if (!sk->sk_ipv6only &&
 490                     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
 491                         memcpy(daaddr, &sk->sk_daddr, 4);
 492                         tx_info->ip_family = AF_INET;
 493                 } else {
 494                         memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
 495                         tx_info->ip_family = AF_INET6;
 496                 }
 497 #endif
 498         }
 499
 500         /* get the l2t index */
 501         dst = sk_dst_get(sk);
 502         if (!dst) {
 503                 pr_err("DST entry not found\n");
 504                 goto free_tx_info;
 505         }
 506         n = dst_neigh_lookup(dst, daaddr);
 507         if (!n || !n->dev) {
 508                 pr_err("neighbour not found\n");
 509                 dst_release(dst);
 510                 goto free_tx_info;
 511         }
 512         tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
 513
 514         neigh_release(n);
 515         dst_release(dst);
 516
 517         if (!tx_info->l2te) {
 518                 pr_err("l2t entry not found\n");
 519                 goto free_tx_info;
 520         }
 521
 522         /* Driver shouldn't be removed until any single connection exists */
 523         if (!try_module_get(THIS_MODULE))
 524                 goto free_l2t;
 525
 526         init_completion(&tx_info->completion);
 527         /* create a filter and call cxgb4_l2t_send to send the packet out, which
 528          * will take care of updating l2t entry in hw if not already done.
 529          */
 530         tx_info->open_state = CH_KTLS_OPEN_PENDING;
 531
 532         if (chcr_setup_connection(sk, tx_info))
 533                 goto put_module;
 534
 535         /* Wait for reply */
 536         wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
 537         spin_lock_bh(&tx_info->lock);
 538         if (tx_info->open_state) {
 539                 /* need to wait for hw response, can't free tx_info yet. */
 540                 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
 541                         tx_info->pending_close = true;
 542                 else
 543                         spin_unlock_bh(&tx_info->lock);
 544                 /* if in pending close, free the lock after the cleanup */
 545                 goto put_module;
 546         }
 547         spin_unlock_bh(&tx_info->lock);
 548
 549         /* initialize tcb */
 550         reinit_completion(&tx_info->completion);
 551         /* mark it pending for hw response */
 552         tx_info->open_state = CH_KTLS_OPEN_PENDING;
 553
 554         if (chcr_init_tcb_fields(tx_info))
 555                 goto free_tid;
 556
 557         /* Wait for reply */
 558         wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
 559         spin_lock_bh(&tx_info->lock);
 560         if (tx_info->open_state) {
 561                 /* need to wait for hw response, can't free tx_info yet. */
 562                 tx_info->pending_close = true;
 563                 /* free the lock after cleanup */
 564                 goto free_tid;
 565         }
 566         spin_unlock_bh(&tx_info->lock);
 567
 568         if (!cxgb4_check_l2t_valid(tx_info->l2te))
 569                 goto free_tid;
 570
 571         atomic64_inc(&port_stats->ktls_tx_ctx);
 572         tx_ctx->chcr_info = tx_info;
 573
 574         return 0;
 575
 576 free_tid:
 577         chcr_ktls_mark_tcb_close(tx_info);
 578 #if IS_ENABLED(CONFIG_IPV6)
 579         /* clear clip entry */
 580         if (tx_info->ip_family == AF_INET6)
 581                 cxgb4_clip_release(netdev, (const u32 *)
 582                                    &sk->sk_v6_rcv_saddr,
 583                                    1);
 584 #endif
 585         cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 586                          tx_info->tid, tx_info->ip_family);
 587
 588 put_module:
 589         /* release module refcount */
 590         module_put(THIS_MODULE);
 591 free_l2t:
 592         cxgb4_l2t_release(tx_info->l2te);
 593 free_tx_info:
 594         if (tx_info->pending_close)
 595                 spin_unlock_bh(&tx_info->lock);
 596         else
 597                 kvfree(tx_info);
 598 out:
 599         atomic64_inc(&port_stats->ktls_tx_connection_fail);
 600         return -1;
 601 }
 602
 603 /*
 604  * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
 605  *                        handling.
 606  * @tx_info - driver specific tls info.
 607  * return: NET_TX_OK/NET_XMIT_DROP
 608  */
 609 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
 610 {
 611         int  ret = 0;
 612
 613         /* set tcb in offload and bypass */
 614         ret =
 615         chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
 616                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
 617                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
 618         if (ret)
 619                 return ret;
 620         /* reset snd_una and snd_next fields in tcb */
 621         ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
 622                                  TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
 623                                  TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
 624                                  0, 1);
 625         if (ret)
 626                 return ret;
 627
 628         /* reset send max */
 629         ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
 630                                  TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
 631                                  0, 1);
 632         if (ret)
 633                 return ret;
 634
 635         /* update l2t index and request for tp reply to confirm tcb is
 636          * initialised to handle tx traffic.
 637          */
 638         ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
 639                                  TCB_L2T_IX_V(TCB_L2T_IX_M),
 640                                  TCB_L2T_IX_V(tx_info->l2te->idx), 0);
 641         return ret;
 642 }
 643
 644 /*
 645  * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
 646  */
 647 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
 648                                       unsigned char *input)
 649 {
 650         const struct cpl_act_open_rpl *p = (void *)input;
 651         struct chcr_ktls_info *tx_info = NULL;
 652         unsigned int atid, tid, status;
 653         struct tid_info *t;
 654
 655         tid = GET_TID(p);
 656         status = AOPEN_STATUS_G(ntohl(p->atid_status));
 657         atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
 658
 659         t = &adap->tids;
 660         tx_info = lookup_atid(t, atid);
 661
 662         if (!tx_info || tx_info->atid != atid) {
 663                 pr_err("%s: incorrect tx_info or atid\n", __func__);
 664                 return -1;
 665         }
 666
 667         cxgb4_free_atid(t, atid);
 668         tx_info->atid = -1;
 669
 670         spin_lock(&tx_info->lock);
 671         /* HW response is very close, finish pending cleanup */
 672         if (tx_info->pending_close) {
 673                 spin_unlock(&tx_info->lock);
 674                 if (!status) {
 675                         /* it's a late success, tcb status is established,
 676                          * mark it close.
 677                          */
 678                         chcr_ktls_mark_tcb_close(tx_info);
 679                         cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 680                                          tid, tx_info->ip_family);
 681                 }
 682                 kvfree(tx_info);
 683                 return 0;
 684         }
 685
 686         if (!status) {
 687                 tx_info->tid = tid;
 688                 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
 689                 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
 690         } else {
 691                 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
 692         }
 693         spin_unlock(&tx_info->lock);
 694
 695         complete(&tx_info->completion);
 696         return 0;
 697 }
 698
 699 /*
 700  * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
 701  */
 702 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
 703 {
 704         const struct cpl_set_tcb_rpl *p = (void *)input;
 705         struct chcr_ktls_info *tx_info = NULL;
 706         struct tid_info *t;
 707         u32 tid;
 708
 709         tid = GET_TID(p);
 710
 711         t = &adap->tids;
 712         tx_info = lookup_tid(t, tid);
 713
 714         if (!tx_info || tx_info->tid != tid) {
 715                 pr_err("%s: incorrect tx_info or tid\n", __func__);
 716                 return -1;
 717         }
 718
 719         spin_lock(&tx_info->lock);
 720         if (tx_info->pending_close) {
 721                 spin_unlock(&tx_info->lock);
 722                 kvfree(tx_info);
 723                 return 0;
 724         }
 725         tx_info->open_state = false;
 726         spin_unlock(&tx_info->lock);
 727
 728         complete(&tx_info->completion);
 729         return 0;
 730 }
 731
 732 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 733                                         u32 tid, void *pos, u16 word,
 734                                         struct sge_eth_txq *q, u64 mask,
 735                                         u64 val, u32 reply)
 736 {
 737         struct cpl_set_tcb_field_core *cpl;
 738         struct ulptx_idata *idata;
 739         struct ulp_txpkt *txpkt;
 740
 741         /* ULP_TXPKT */
 742         txpkt = pos;
 743         txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
 744                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
 745                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
 746                                 ULP_TXPKT_RO_F);
 747         txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
 748
 749         /* ULPTX_IDATA sub-command */
 750         idata = (struct ulptx_idata *)(txpkt + 1);
 751         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
 752         idata->len = htonl(sizeof(*cpl));
 753         pos = idata + 1;
 754
 755         cpl = pos;
 756         /* CPL_SET_TCB_FIELD */
 757         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 758         cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 759                         NO_REPLY_V(!reply));
 760         cpl->word_cookie = htons(TCB_WORD_V(word));
 761         cpl->mask = cpu_to_be64(mask);
 762         cpl->val = cpu_to_be64(val);
 763
 764         /* ULPTX_NOOP */
 765         idata = (struct ulptx_idata *)(cpl + 1);
 766         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
 767         idata->len = htonl(0);
 768         pos = idata + 1;
 769
 770         return pos;
 771 }
 772
 773
 774 /*
 775  * chcr_write_cpl_set_tcb_ulp: update tcb values.
 776  * TCB is responsible to create tcp headers, so all the related values
 777  * should be correctly updated.
 778  * @tx_info - driver specific tls info.
 779  * @q - tx queue on which packet is going out.
 780  * @tid - TCB identifier.
 781  * @pos - current index where should we start writing.
 782  * @word - TCB word.
 783  * @mask - TCB word related mask.
 784  * @val - TCB word related value.
 785  * @reply - set 1 if looking for TP response.
 786  * return - next position to write.
 787  */
 788 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 789                                         struct sge_eth_txq *q, u32 tid,
 790                                         void *pos, u16 word, u64 mask,
 791                                         u64 val, u32 reply)
 792 {
 793         int left = (void *)q->q.stat - pos;
 794
 795         if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
 796                 if (!left) {
 797                         pos = q->q.desc;
 798                 } else {
 799                         u8 buf[48] = {0};
 800
 801                         __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
 802                                                      mask, val, reply);
 803
 804                         return chcr_copy_to_txd(buf, &q->q, pos,
 805                                                 CHCR_SET_TCB_FIELD_LEN);
 806                 }
 807         }
 808
 809         pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
 810                                            mask, val, reply);
 811
 812         /* check again if we are at the end of the queue */
 813         if (left == CHCR_SET_TCB_FIELD_LEN)
 814                 pos = q->q.desc;
 815
 816         return pos;
 817 }
 818
 819 /*
 820  * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
 821  * with updated values like tcp seq, ack, window etc.
 822  * @tx_info - driver specific tls info.
 823  * @q - TX queue.
 824  * @tcp_seq
 825  * @tcp_ack
 826  * @tcp_win
 827  * return: NETDEV_TX_BUSY/NET_TX_OK.
 828  */
 829 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
 830                                    struct sge_eth_txq *q, u64 tcp_seq,
 831                                    u64 tcp_ack, u64 tcp_win, bool offset)
 832 {
 833         bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
 834         struct ch_ktls_port_stats_debug *port_stats;
 835         u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
 836         struct fw_ulptx_wr *wr;
 837         int credits;
 838         void *pos;
 839
 840         wr_len = sizeof(*wr);
 841         /* there can be max 4 cpls, check if we have enough credits */
 842         len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 843         ndesc = DIV_ROUND_UP(len, 64);
 844
 845         credits = chcr_txq_avail(&q->q) - ndesc;
 846         if (unlikely(credits < 0)) {
 847                 chcr_eth_txq_stop(q);
 848                 return NETDEV_TX_BUSY;
 849         }
 850
 851         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
 852                 chcr_eth_txq_stop(q);
 853                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
 854         }
 855
 856         pos = &q->q.desc[q->q.pidx];
 857         /* make space for WR, we'll fill it later when we know all the cpls
 858          * being sent out and have complete length.
 859          */
 860         wr = pos;
 861         pos += wr_len;
 862         /* update tx_max if its a re-transmit or the first wr */
 863         if (first_wr || tcp_seq != tx_info->prev_seq) {
 864                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 865                                                  TCB_TX_MAX_W,
 866                                                  TCB_TX_MAX_V(TCB_TX_MAX_M),
 867                                                  TCB_TX_MAX_V(tcp_seq), 0);
 868                 cpl++;
 869         }
 870         /* reset snd una if it's a re-transmit pkt */
 871         if (tcp_seq != tx_info->prev_seq || offset) {
 872                 /* reset snd_una */
 873                 port_stats =
 874                         &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
 875                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 876                                                  TCB_SND_UNA_RAW_W,
 877                                                  TCB_SND_UNA_RAW_V
 878                                                  (TCB_SND_UNA_RAW_M),
 879                                                  TCB_SND_UNA_RAW_V(0), 0);
 880                 if (tcp_seq != tx_info->prev_seq)
 881                         atomic64_inc(&port_stats->ktls_tx_ooo);
 882                 cpl++;
 883         }
 884         /* update ack */
 885         if (first_wr || tx_info->prev_ack != tcp_ack) {
 886                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 887                                                  TCB_RCV_NXT_W,
 888                                                  TCB_RCV_NXT_V(TCB_RCV_NXT_M),
 889                                                  TCB_RCV_NXT_V(tcp_ack), 0);
 890                 tx_info->prev_ack = tcp_ack;
 891                 cpl++;
 892         }
 893         /* update receive window */
 894         if (first_wr || tx_info->prev_win != tcp_win) {
 895                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 896                                                  TCB_RCV_WND_W,
 897                                                  TCB_RCV_WND_V(TCB_RCV_WND_M),
 898                                                  TCB_RCV_WND_V(tcp_win), 0);
 899                 tx_info->prev_win = tcp_win;
 900                 cpl++;
 901         }
 902
 903         if (cpl) {
 904                 /* get the actual length */
 905                 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 906                 /* ULPTX wr */
 907                 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
 908                 wr->cookie = 0;
 909                 /* fill len in wr field */
 910                 wr->flowid_len16 = htonl(wr_mid |
 911                                          FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
 912
 913                 ndesc = DIV_ROUND_UP(len, 64);
 914                 chcr_txq_advance(&q->q, ndesc);
 915                 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
 916         }
 917         return 0;
 918 }
 919
 920 /*
 921  * chcr_ktls_get_tx_flits
 922  * returns number of flits to be sent out, it includes key context length, WR
 923  * size and skb fragments.
 924  */
 925 static unsigned int
 926 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
 927 {
 928         return chcr_sgl_len(nr_frags) +
 929                DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
 930 }
 931
 932 /*
 933  * chcr_ktls_check_tcp_options: To check if there is any TCP option available
 934  * other than timestamp.
 935  * @skb - skb contains partial record..
 936  * return: 1 / 0
 937  */
 938 static int
 939 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
 940 {
 941         int cnt, opt, optlen;
 942         u_char *cp;
 943
 944         cp = (u_char *)(tcp + 1);
 945         cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
 946         for (; cnt > 0; cnt -= optlen, cp += optlen) {
 947                 opt = cp[0];
 948                 if (opt == TCPOPT_EOL)
 949                         break;
 950                 if (opt == TCPOPT_NOP) {
 951                         optlen = 1;
 952                 } else {
 953                         if (cnt < 2)
 954                                 break;
 955                         optlen = cp[1];
 956                         if (optlen < 2 || optlen > cnt)
 957                                 break;
 958                 }
 959                 switch (opt) {
 960                 case TCPOPT_NOP:
 961                         break;
 962                 default:
 963                         return 1;
 964                 }
 965         }
 966         return 0;
 967 }
 968
 969 /*
 970  * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
 971  * send out separately.
 972  * @tx_info - driver specific tls info.
 973  * @skb - skb contains partial record..
 974  * @q - TX queue.
 975  * @tx_chan - channel number.
 976  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
 977  */
 978 static int
 979 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
 980                             struct sge_eth_txq *q, uint32_t tx_chan)
 981 {
 982         struct fw_eth_tx_pkt_wr *wr;
 983         struct cpl_tx_pkt_core *cpl;
 984         u32 ctrl, iplen, maclen;
 985         struct ipv6hdr *ip6;
 986         unsigned int ndesc;
 987         struct tcphdr *tcp;
 988         int len16, pktlen;
 989         struct iphdr *ip;
 990         u32 wr_mid = 0;
 991         int credits;
 992         u8 buf[150];
 993         u64 cntrl1;
 994         void *pos;
 995
 996         iplen = skb_network_header_len(skb);
 997         maclen = skb_mac_header_len(skb);
 998
 999         /* packet length = eth hdr len + ip hdr len + tcp hdr len
1000          * (including options).
1001          */
1002         pktlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
1003
1004         ctrl = sizeof(*cpl) + pktlen;
1005         len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
1006         /* check how many descriptors needed */
1007         ndesc = DIV_ROUND_UP(len16, 4);
1008
1009         credits = chcr_txq_avail(&q->q) - ndesc;
1010         if (unlikely(credits < 0)) {
1011                 chcr_eth_txq_stop(q);
1012                 return NETDEV_TX_BUSY;
1013         }
1014
1015         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1016                 chcr_eth_txq_stop(q);
1017                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1018         }
1019
1020         pos = &q->q.desc[q->q.pidx];
1021         wr = pos;
1022
1023         /* Firmware work request header */
1024         wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1025                                FW_WR_IMMDLEN_V(ctrl));
1026
1027         wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1028         wr->r3 = 0;
1029
1030         cpl = (void *)(wr + 1);
1031
1032         /* CPL header */
1033         cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1034                            TXPKT_PF_V(tx_info->adap->pf));
1035         cpl->pack = 0;
1036         cpl->len = htons(pktlen);
1037
1038         memcpy(buf, skb->data, pktlen);
1039         if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1040                 /* we need to correct ip header len */
1041                 ip = (struct iphdr *)(buf + maclen);
1042                 ip->tot_len = htons(pktlen - maclen);
1043                 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1044         } else {
1045                 ip6 = (struct ipv6hdr *)(buf + maclen);
1046                 ip6->payload_len = htons(pktlen - maclen - iplen);
1047                 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1048         }
1049
1050         cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1051                   TXPKT_IPHDR_LEN_V(iplen);
1052         /* checksum offload */
1053         cpl->ctrl1 = cpu_to_be64(cntrl1);
1054
1055         pos = cpl + 1;
1056
1057         /* now take care of the tcp header, if fin is not set then clear push
1058          * bit as well, and if fin is set, it will be sent at the last so we
1059          * need to update the tcp sequence number as per the last packet.
1060          */
1061         tcp = (struct tcphdr *)(buf + maclen + iplen);
1062
1063         if (!tcp->fin)
1064                 tcp->psh = 0;
1065         else
1066                 tcp->seq = htonl(tx_info->prev_seq);
1067
1068         chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1069
1070         chcr_txq_advance(&q->q, ndesc);
1071         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1072         return 0;
1073 }
1074
1075 /*
1076  * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1077  * received has partial end part of the record, send out the complete record, so
1078  * that crypto block will be able to generate TAG/HASH.
1079  * @skb - segment which has complete or partial end part.
1080  * @tx_info - driver specific tls info.
1081  * @q - TX queue.
1082  * @tcp_seq
1083  * @tcp_push - tcp push bit.
1084  * @mss - segment size.
1085  * return: NETDEV_TX_BUSY/NET_TX_OK.
1086  */
1087 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1088                                       struct chcr_ktls_info *tx_info,
1089                                       struct sge_eth_txq *q, u32 tcp_seq,
1090                                       bool is_last_wr, u32 data_len,
1091                                       u32 skb_offset, u32 nfrags,
1092                                       bool tcp_push, u32 mss)
1093 {
1094         u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1095         struct adapter *adap = tx_info->adap;
1096         int credits, left, last_desc;
1097         struct tx_sw_desc *sgl_sdesc;
1098         struct cpl_tx_data *tx_data;
1099         struct cpl_tx_sec_pdu *cpl;
1100         struct ulptx_idata *idata;
1101         struct ulp_txpkt *ulptx;
1102         struct fw_ulptx_wr *wr;
1103         void *pos;
1104         u64 *end;
1105
1106         /* get the number of flits required */
1107         flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1108         /* number of descriptors */
1109         ndesc = chcr_flits_to_desc(flits);
1110         /* check if enough credits available */
1111         credits = chcr_txq_avail(&q->q) - ndesc;
1112         if (unlikely(credits < 0)) {
1113                 chcr_eth_txq_stop(q);
1114                 return NETDEV_TX_BUSY;
1115         }
1116
1117         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1118                 /* Credits are below the threshold values, stop the queue after
1119                  * injecting the Work Request for this packet.
1120                  */
1121                 chcr_eth_txq_stop(q);
1122                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1123         }
1124
1125         last_desc = q->q.pidx + ndesc - 1;
1126         if (last_desc >= q->q.size)
1127                 last_desc -= q->q.size;
1128         sgl_sdesc = &q->q.sdesc[last_desc];
1129
1130         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1131                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1132                 q->mapping_err++;
1133                 return NETDEV_TX_BUSY;
1134         }
1135
1136         if (!is_last_wr)
1137                 skb_get(skb);
1138
1139         pos = &q->q.desc[q->q.pidx];
1140         end = (u64 *)pos + flits;
1141         /* FW_ULPTX_WR */
1142         wr = pos;
1143         /* WR will need len16 */
1144         len16 = DIV_ROUND_UP(flits, 2);
1145         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1146         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1147         wr->cookie = 0;
1148         pos += sizeof(*wr);
1149         /* ULP_TXPKT */
1150         ulptx = pos;
1151         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1152                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1153                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1154                                 ULP_TXPKT_RO_F);
1155         ulptx->len = htonl(len16 - 1);
1156         /* ULPTX_IDATA sub-command */
1157         idata = (struct ulptx_idata *)(ulptx + 1);
1158         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1159         /* idata length will include cpl_tx_sec_pdu + key context size +
1160          * cpl_tx_data header.
1161          */
1162         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1163                            sizeof(*tx_data));
1164         /* SEC CPL */
1165         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1166         cpl->op_ivinsrtofst =
1167                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1168                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1169                       CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1170                       CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1171         cpl->pldlen = htonl(data_len);
1172
1173         /* encryption should start after tls header size + iv size */
1174         cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1175
1176         cpl->aadstart_cipherstop_hi =
1177                 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1178                       CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1179                       CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1180
1181         /* authentication will also start after tls header + iv size */
1182         cpl->cipherstop_lo_authinsert =
1183         htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1184               CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1185               CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1186
1187         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1188         cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1189         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1190         cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1191
1192         pos = cpl + 1;
1193         /* check if space left to fill the keys */
1194         left = (void *)q->q.stat - pos;
1195         if (!left) {
1196                 left = (void *)end - (void *)q->q.stat;
1197                 pos = q->q.desc;
1198                 end = pos + left;
1199         }
1200
1201         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1202                                tx_info->key_ctx_len);
1203         left = (void *)q->q.stat - pos;
1204
1205         if (!left) {
1206                 left = (void *)end - (void *)q->q.stat;
1207                 pos = q->q.desc;
1208                 end = pos + left;
1209         }
1210         /* CPL_TX_DATA */
1211         tx_data = (void *)pos;
1212         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1213         tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1214
1215         tx_data->rsvd = htonl(tcp_seq);
1216
1217         tx_data->flags = htonl(TX_BYPASS_F);
1218         if (tcp_push)
1219                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1220
1221         /* check left again, it might go beyond queue limit */
1222         pos = tx_data + 1;
1223         left = (void *)q->q.stat - pos;
1224
1225         /* check the position again */
1226         if (!left) {
1227                 left = (void *)end - (void *)q->q.stat;
1228                 pos = q->q.desc;
1229                 end = pos + left;
1230         }
1231
1232         /* send the complete packet except the header */
1233         cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1234                                 skb_offset, data_len);
1235         sgl_sdesc->skb = skb;
1236
1237         chcr_txq_advance(&q->q, ndesc);
1238         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1239         atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1240
1241         return 0;
1242 }
1243
1244 /*
1245  * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1246  * a middle part of a record, fetch the prior data to make it 16 byte aligned
1247  * and then only send it out.
1248  *
1249  * @skb - skb contains partial record..
1250  * @tx_info - driver specific tls info.
1251  * @q - TX queue.
1252  * @tcp_seq
1253  * @tcp_push - tcp push bit.
1254  * @mss - segment size.
1255  * @tls_rec_offset - offset from start of the tls record.
1256  * @perior_data - data before the current segment, required to make this record
1257  *                16 byte aligned.
1258  * @prior_data_len - prior_data length (less than 16)
1259  * return: NETDEV_TX_BUSY/NET_TX_OK.
1260  */
1261 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1262                                    struct chcr_ktls_info *tx_info,
1263                                    struct sge_eth_txq *q,
1264                                    u32 tcp_seq, bool tcp_push, u32 mss,
1265                                    u32 tls_rec_offset, u8 *prior_data,
1266                                    u32 prior_data_len, u32 data_len,
1267                                    u32 skb_offset)
1268 {
1269         u32 len16, wr_mid = 0, cipher_start, nfrags;
1270         struct adapter *adap = tx_info->adap;
1271         unsigned int flits = 0, ndesc;
1272         int credits, left, last_desc;
1273         struct tx_sw_desc *sgl_sdesc;
1274         struct cpl_tx_data *tx_data;
1275         struct cpl_tx_sec_pdu *cpl;
1276         struct ulptx_idata *idata;
1277         struct ulp_txpkt *ulptx;
1278         struct fw_ulptx_wr *wr;
1279         __be64 iv_record;
1280         void *pos;
1281         u64 *end;
1282
1283         nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1284         /* get the number of flits required, it's a partial record so 2 flits
1285          * (AES_BLOCK_SIZE) will be added.
1286          */
1287         flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1288         /* get the correct 8 byte IV of this record */
1289         iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1290         /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1291          * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1292          * data will be added.
1293          */
1294         if (prior_data_len)
1295                 flits += 2;
1296         /* number of descriptors */
1297         ndesc = chcr_flits_to_desc(flits);
1298         /* check if enough credits available */
1299         credits = chcr_txq_avail(&q->q) - ndesc;
1300         if (unlikely(credits < 0)) {
1301                 chcr_eth_txq_stop(q);
1302                 return NETDEV_TX_BUSY;
1303         }
1304
1305         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1306                 chcr_eth_txq_stop(q);
1307                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1308         }
1309
1310         last_desc = q->q.pidx + ndesc - 1;
1311         if (last_desc >= q->q.size)
1312                 last_desc -= q->q.size;
1313         sgl_sdesc = &q->q.sdesc[last_desc];
1314
1315         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1316                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1317                 q->mapping_err++;
1318                 return NETDEV_TX_BUSY;
1319         }
1320
1321         pos = &q->q.desc[q->q.pidx];
1322         end = (u64 *)pos + flits;
1323         /* FW_ULPTX_WR */
1324         wr = pos;
1325         /* WR will need len16 */
1326         len16 = DIV_ROUND_UP(flits, 2);
1327         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1328         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1329         wr->cookie = 0;
1330         pos += sizeof(*wr);
1331         /* ULP_TXPKT */
1332         ulptx = pos;
1333         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1334                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1335                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1336                                 ULP_TXPKT_RO_F);
1337         ulptx->len = htonl(len16 - 1);
1338         /* ULPTX_IDATA sub-command */
1339         idata = (struct ulptx_idata *)(ulptx + 1);
1340         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1341         /* idata length will include cpl_tx_sec_pdu + key context size +
1342          * cpl_tx_data header.
1343          */
1344         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1345                            sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1346         /* SEC CPL */
1347         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1348         /* cipher start will have tls header + iv size extra if its a header
1349          * part of tls record. else only 16 byte IV will be added.
1350          */
1351         cipher_start =
1352                 AES_BLOCK_LEN + 1 +
1353                 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1354
1355         cpl->op_ivinsrtofst =
1356                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1357                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1358                       CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1359         cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1360         cpl->aadstart_cipherstop_hi =
1361                 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1362         cpl->cipherstop_lo_authinsert = 0;
1363         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1364         cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1365         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1366         cpl->scmd1 = 0;
1367
1368         pos = cpl + 1;
1369         /* check if space left to fill the keys */
1370         left = (void *)q->q.stat - pos;
1371         if (!left) {
1372                 left = (void *)end - (void *)q->q.stat;
1373                 pos = q->q.desc;
1374                 end = pos + left;
1375         }
1376
1377         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1378                                tx_info->key_ctx_len);
1379         left = (void *)q->q.stat - pos;
1380
1381         if (!left) {
1382                 left = (void *)end - (void *)q->q.stat;
1383                 pos = q->q.desc;
1384                 end = pos + left;
1385         }
1386         /* CPL_TX_DATA */
1387         tx_data = (void *)pos;
1388         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1389         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1390                              TX_LENGTH_V(data_len + prior_data_len));
1391         tx_data->rsvd = htonl(tcp_seq);
1392         tx_data->flags = htonl(TX_BYPASS_F);
1393         if (tcp_push)
1394                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1395
1396         /* check left again, it might go beyond queue limit */
1397         pos = tx_data + 1;
1398         left = (void *)q->q.stat - pos;
1399
1400         /* check the position again */
1401         if (!left) {
1402                 left = (void *)end - (void *)q->q.stat;
1403                 pos = q->q.desc;
1404                 end = pos + left;
1405         }
1406         /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1407          * bytes of actual IV and 4 bytes of 16 byte-sequence.
1408          */
1409         memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1410         memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1411         *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1412                 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1413                 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1414
1415         pos += 16;
1416         /* Prior_data_len will always be less than 16 bytes, fill the
1417          * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1418          * to 0.
1419          */
1420         if (prior_data_len)
1421                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1422         /* send the complete packet except the header */
1423         cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1424                                 skb_offset, data_len);
1425         sgl_sdesc->skb = skb;
1426
1427         chcr_txq_advance(&q->q, ndesc);
1428         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1429
1430         return 0;
1431 }
1432
1433 /*
1434  * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1435  * only plain text (only tls header and iv)
1436  * @tx_info - driver specific tls info.
1437  * @skb - skb contains partial record..
1438  * @tcp_seq
1439  * @mss - segment size.
1440  * @tcp_push - tcp push bit.
1441  * @q - TX queue.
1442  * @port_id : port number
1443  * @perior_data - data before the current segment, required to make this record
1444  *               16 byte aligned.
1445  * @prior_data_len - prior_data length (less than 16)
1446  * return: NETDEV_TX_BUSY/NET_TX_OK.
1447  */
1448 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1449                                  struct sk_buff *skb, u32 tcp_seq, u32 mss,
1450                                  bool tcp_push, struct sge_eth_txq *q,
1451                                  u32 port_id, u8 *prior_data,
1452                                  u32 data_len, u32 skb_offset,
1453                                  u32 prior_data_len)
1454 {
1455         int credits, left, len16, last_desc;
1456         unsigned int flits = 0, ndesc;
1457         struct tx_sw_desc *sgl_sdesc;
1458         struct cpl_tx_data *tx_data;
1459         struct ulptx_idata *idata;
1460         struct ulp_txpkt *ulptx;
1461         struct fw_ulptx_wr *wr;
1462         u32 wr_mid = 0, nfrags;
1463         void *pos;
1464         u64 *end;
1465
1466         flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1467         nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1468         flits += chcr_sgl_len(nfrags);
1469         if (prior_data_len)
1470                 flits += 2;
1471
1472         /* WR will need len16 */
1473         len16 = DIV_ROUND_UP(flits, 2);
1474         /* check how many descriptors needed */
1475         ndesc = DIV_ROUND_UP(flits, 8);
1476
1477         credits = chcr_txq_avail(&q->q) - ndesc;
1478         if (unlikely(credits < 0)) {
1479                 chcr_eth_txq_stop(q);
1480                 return NETDEV_TX_BUSY;
1481         }
1482
1483         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1484                 chcr_eth_txq_stop(q);
1485                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1486         }
1487
1488         last_desc = q->q.pidx + ndesc - 1;
1489         if (last_desc >= q->q.size)
1490                 last_desc -= q->q.size;
1491         sgl_sdesc = &q->q.sdesc[last_desc];
1492
1493         if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1494                                    sgl_sdesc->addr) < 0)) {
1495                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1496                 q->mapping_err++;
1497                 return NETDEV_TX_BUSY;
1498         }
1499
1500         pos = &q->q.desc[q->q.pidx];
1501         end = (u64 *)pos + flits;
1502         /* FW_ULPTX_WR */
1503         wr = pos;
1504         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1505         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1506         wr->cookie = 0;
1507         pos += sizeof(*wr);
1508         /* ULP_TXPKT */
1509         ulptx = (struct ulp_txpkt *)(wr + 1);
1510         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1511                         ULP_TXPKT_DATAMODIFY_V(0) |
1512                         ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1513                         ULP_TXPKT_DEST_V(0) |
1514                         ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1515         ulptx->len = htonl(len16 - 1);
1516         /* ULPTX_IDATA sub-command */
1517         idata = (struct ulptx_idata *)(ulptx + 1);
1518         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1519         idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1520         /* CPL_TX_DATA */
1521         tx_data = (struct cpl_tx_data *)(idata + 1);
1522         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1523         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1524                              TX_LENGTH_V(data_len + prior_data_len));
1525         /* set tcp seq number */
1526         tx_data->rsvd = htonl(tcp_seq);
1527         tx_data->flags = htonl(TX_BYPASS_F);
1528         if (tcp_push)
1529                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1530
1531         pos = tx_data + 1;
1532         /* apart from prior_data_len, we should set remaining part of 16 bytes
1533          * to be zero.
1534          */
1535         if (prior_data_len)
1536                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1537
1538         /* check left again, it might go beyond queue limit */
1539         left = (void *)q->q.stat - pos;
1540
1541         /* check the position again */
1542         if (!left) {
1543                 left = (void *)end - (void *)q->q.stat;
1544                 pos = q->q.desc;
1545                 end = pos + left;
1546         }
1547         /* send the complete packet including the header */
1548         cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1549                                 skb_offset, data_len);
1550         sgl_sdesc->skb = skb;
1551
1552         chcr_txq_advance(&q->q, ndesc);
1553         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1554         return 0;
1555 }
1556
1557 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1558                                 struct sk_buff *skb,
1559                                 struct sge_eth_txq *q)
1560 {
1561         u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1562         struct tx_sw_desc *sgl_sdesc;
1563         struct fw_eth_tx_pkt_wr *wr;
1564         struct cpl_tx_pkt_core *cpl;
1565         unsigned int flits, ndesc;
1566         int credits, last_desc;
1567         u64 cntrl1, *end;
1568         void *pos;
1569
1570         ctrl = sizeof(*cpl);
1571         flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1572
1573         flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1574         len16 = DIV_ROUND_UP(flits, 2);
1575         /* check how many descriptors needed */
1576         ndesc = DIV_ROUND_UP(flits, 8);
1577
1578         credits = chcr_txq_avail(&q->q) - ndesc;
1579         if (unlikely(credits < 0)) {
1580                 chcr_eth_txq_stop(q);
1581                 return -ENOMEM;
1582         }
1583
1584         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1585                 chcr_eth_txq_stop(q);
1586                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1587         }
1588
1589         last_desc = q->q.pidx + ndesc - 1;
1590         if (last_desc >= q->q.size)
1591                 last_desc -= q->q.size;
1592         sgl_sdesc = &q->q.sdesc[last_desc];
1593
1594         if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1595                                    sgl_sdesc->addr) < 0)) {
1596                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1597                 q->mapping_err++;
1598                 return -ENOMEM;
1599         }
1600
1601         iplen = skb_network_header_len(skb);
1602         maclen = skb_mac_header_len(skb);
1603
1604         pos = &q->q.desc[q->q.pidx];
1605         end = (u64 *)pos + flits;
1606         wr = pos;
1607
1608         /* Firmware work request header */
1609         wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1610                                FW_WR_IMMDLEN_V(ctrl));
1611
1612         wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1613         wr->r3 = 0;
1614
1615         cpl = (void *)(wr + 1);
1616
1617         /* CPL header */
1618         cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1619                            TXPKT_INTF_V(tx_info->tx_chan) |
1620                            TXPKT_PF_V(tx_info->adap->pf));
1621         cpl->pack = 0;
1622         cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1623                                    TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1624         cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1625                   TXPKT_IPHDR_LEN_V(iplen);
1626         /* checksum offload */
1627         cpl->ctrl1 = cpu_to_be64(cntrl1);
1628         cpl->len = htons(skb->len);
1629
1630         pos = cpl + 1;
1631
1632         cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1633         sgl_sdesc->skb = skb;
1634         chcr_txq_advance(&q->q, ndesc);
1635         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1636         return 0;
1637 }
1638
1639 /*
1640  * chcr_ktls_copy_record_in_skb
1641  * @nskb - new skb where the frags to be added.
1642  * @skb - old skb, to copy socket and destructor details.
1643  * @record - specific record which has complete 16k record in frags.
1644  */
1645 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1646                                          struct sk_buff *skb,
1647                                          struct tls_record_info *record)
1648 {
1649         int i = 0;
1650
1651         for (i = 0; i < record->num_frags; i++) {
1652                 skb_shinfo(nskb)->frags[i] = record->frags[i];
1653                 /* increase the frag ref count */
1654                 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1655         }
1656
1657         skb_shinfo(nskb)->nr_frags = record->num_frags;
1658         nskb->data_len = record->len;
1659         nskb->len += record->len;
1660         nskb->truesize += record->len;
1661         nskb->sk = skb->sk;
1662         nskb->destructor = skb->destructor;
1663         refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1664 }
1665
1666 /*
1667  * chcr_ktls_update_snd_una:  Reset the SEND_UNA. It will be done to avoid
1668  * sending the same segment again. It will discard the segment which is before
1669  * the current tx max.
1670  * @tx_info - driver specific tls info.
1671  * @q - TX queue.
1672  * return: NET_TX_OK/NET_XMIT_DROP.
1673  */
1674 static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
1675                                     struct sge_eth_txq *q)
1676 {
1677         struct fw_ulptx_wr *wr;
1678         unsigned int ndesc;
1679         int credits;
1680         void *pos;
1681         u32 len;
1682
1683         len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
1684         ndesc = DIV_ROUND_UP(len, 64);
1685
1686         credits = chcr_txq_avail(&q->q) - ndesc;
1687         if (unlikely(credits < 0)) {
1688                 chcr_eth_txq_stop(q);
1689                 return NETDEV_TX_BUSY;
1690         }
1691
1692         pos = &q->q.desc[q->q.pidx];
1693
1694         wr = pos;
1695         /* ULPTX wr */
1696         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1697         wr->cookie = 0;
1698         /* fill len in wr field */
1699         wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
1700
1701         pos += sizeof(*wr);
1702
1703         pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
1704                                          TCB_SND_UNA_RAW_W,
1705                                          TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
1706                                          TCB_SND_UNA_RAW_V(0), 0);
1707
1708         chcr_txq_advance(&q->q, ndesc);
1709         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1710
1711         return 0;
1712 }
1713
1714 /*
1715  * chcr_end_part_handler: This handler will handle the record which
1716  * is complete or if record's end part is received. T6 adapter has a issue that
1717  * it can't send out TAG with partial record so if its an end part then we have
1718  * to send TAG as well and for which we need to fetch the complete record and
1719  * send it to crypto module.
1720  * @tx_info - driver specific tls info.
1721  * @skb - skb contains partial record.
1722  * @record - complete record of 16K size.
1723  * @tcp_seq
1724  * @mss - segment size in which TP needs to chop a packet.
1725  * @tcp_push_no_fin - tcp push if fin is not set.
1726  * @q - TX queue.
1727  * @tls_end_offset - offset from end of the record.
1728  * @last wr : check if this is the last part of the skb going out.
1729  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1730  */
1731 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1732                                  struct sk_buff *skb,
1733                                  struct tls_record_info *record,
1734                                  u32 tcp_seq, int mss, bool tcp_push_no_fin,
1735                                  struct sge_eth_txq *q, u32 skb_offset,
1736                                  u32 tls_end_offset, bool last_wr)
1737 {
1738         struct sk_buff *nskb = NULL;
1739         /* check if it is a complete record */
1740         if (tls_end_offset == record->len) {
1741                 nskb = skb;
1742                 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1743         } else {
1744                 nskb = alloc_skb(0, GFP_ATOMIC);
1745                 if (!nskb) {
1746                         dev_kfree_skb_any(skb);
1747                         return NETDEV_TX_BUSY;
1748                 }
1749
1750                 /* copy complete record in skb */
1751                 chcr_ktls_copy_record_in_skb(nskb, skb, record);
1752                 /* packet is being sent from the beginning, update the tcp_seq
1753                  * accordingly.
1754                  */
1755                 tcp_seq = tls_record_start_seq(record);
1756                 /* reset skb offset */
1757                 skb_offset = 0;
1758
1759                 if (last_wr)
1760                         dev_kfree_skb_any(skb);
1761
1762                 last_wr = true;
1763
1764                 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1765         }
1766
1767         if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1768                                        last_wr, record->len, skb_offset,
1769                                        record->num_frags,
1770                                        (last_wr && tcp_push_no_fin),
1771                                        mss)) {
1772                 goto out;
1773         }
1774         tx_info->prev_seq = record->end_seq;
1775         return 0;
1776 out:
1777         dev_kfree_skb_any(nskb);
1778         return NETDEV_TX_BUSY;
1779 }
1780
1781 /*
1782  * chcr_short_record_handler: This handler will take care of the records which
1783  * doesn't have end part (1st part or the middle part(/s) of a record). In such
1784  * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1785  * This partial record might be the first part of the record, or the middle
1786  * part. In case of middle record we should fetch the prior data to make it 16
1787  * byte aligned. If it has a partial tls header or iv then get to the start of
1788  * tls header. And if it has partial TAG, then remove the complete TAG and send
1789  * only the payload.
1790  * There is one more possibility that it gets a partial header, send that
1791  * portion as a plaintext.
1792  * @tx_info - driver specific tls info.
1793  * @skb - skb contains partial record..
1794  * @record - complete record of 16K size.
1795  * @tcp_seq
1796  * @mss - segment size in which TP needs to chop a packet.
1797  * @tcp_push_no_fin - tcp push if fin is not set.
1798  * @q - TX queue.
1799  * @tls_end_offset - offset from end of the record.
1800  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1801  */
1802 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1803                                      struct sk_buff *skb,
1804                                      struct tls_record_info *record,
1805                                      u32 tcp_seq, int mss, bool tcp_push_no_fin,
1806                                      u32 data_len, u32 skb_offset,
1807                                      struct sge_eth_txq *q, u32 tls_end_offset)
1808 {
1809         u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1810         u8 prior_data[16] = {0};
1811         u32 prior_data_len = 0;
1812
1813         /* check if the skb is ending in middle of tag/HASH, its a big
1814          * trouble, send the packet before the HASH.
1815          */
1816         int remaining_record = tls_end_offset - data_len;
1817
1818         if (remaining_record > 0 &&
1819             remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1820                 int trimmed_len = 0;
1821
1822                 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1823                         trimmed_len = data_len -
1824                                       (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1825                                        remaining_record);
1826                 if (!trimmed_len)
1827                         return FALLBACK;
1828
1829                 WARN_ON(trimmed_len > data_len);
1830
1831                 data_len = trimmed_len;
1832                 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1833         }
1834
1835         /* check if it is only the header part. */
1836         if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1837                 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1838                                           tcp_push_no_fin, q,
1839                                           tx_info->port_id, prior_data,
1840                                           data_len, skb_offset, prior_data_len))
1841                         goto out;
1842
1843                 tx_info->prev_seq = tcp_seq + data_len;
1844                 return 0;
1845         }
1846
1847         /* check if the middle record's start point is 16 byte aligned. CTR
1848          * needs 16 byte aligned start point to start encryption.
1849          */
1850         if (tls_rec_offset) {
1851                 /* there is an offset from start, means its a middle record */
1852                 int remaining = 0;
1853
1854                 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1855                         prior_data_len = tls_rec_offset;
1856                         tls_rec_offset = 0;
1857                         remaining = 0;
1858                 } else {
1859                         prior_data_len =
1860                                 (tls_rec_offset -
1861                                 (TLS_HEADER_SIZE + tx_info->iv_size))
1862                                 % AES_BLOCK_LEN;
1863                         remaining = tls_rec_offset - prior_data_len;
1864                 }
1865
1866                 /* if prior_data_len is not zero, means we need to fetch prior
1867                  * data to make this record 16 byte aligned, or we need to reach
1868                  * to start offset.
1869                  */
1870                 if (prior_data_len) {
1871                         int i = 0;
1872                         u8 *data = NULL;
1873                         skb_frag_t *f;
1874                         u8 *vaddr;
1875                         int frag_size = 0, frag_delta = 0;
1876
1877                         while (remaining > 0) {
1878                                 frag_size = skb_frag_size(&record->frags[i]);
1879                                 if (remaining < frag_size)
1880                                         break;
1881
1882                                 remaining -= frag_size;
1883                                 i++;
1884                         }
1885                         f = &record->frags[i];
1886                         vaddr = kmap_atomic(skb_frag_page(f));
1887
1888                         data = vaddr + skb_frag_off(f)  + remaining;
1889                         frag_delta = skb_frag_size(f) - remaining;
1890
1891                         if (frag_delta >= prior_data_len) {
1892                                 memcpy(prior_data, data, prior_data_len);
1893                                 kunmap_atomic(vaddr);
1894                         } else {
1895                                 memcpy(prior_data, data, frag_delta);
1896                                 kunmap_atomic(vaddr);
1897                                 /* get the next page */
1898                                 f = &record->frags[i + 1];
1899                                 vaddr = kmap_atomic(skb_frag_page(f));
1900                                 data = vaddr + skb_frag_off(f);
1901                                 memcpy(prior_data + frag_delta,
1902                                        data, (prior_data_len - frag_delta));
1903                                 kunmap_atomic(vaddr);
1904                         }
1905                         /* reset tcp_seq as per the prior_data_required len */
1906                         tcp_seq -= prior_data_len;
1907                 }
1908                 /* reset snd una, so the middle record won't send the already
1909                  * sent part.
1910                  */
1911                 if (chcr_ktls_update_snd_una(tx_info, q))
1912                         goto out;
1913                 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1914         } else {
1915                 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1916         }
1917
1918         if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1919                                     mss, tls_rec_offset, prior_data,
1920                                     prior_data_len, data_len, skb_offset)) {
1921                 goto out;
1922         }
1923
1924         tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1925         return 0;
1926 out:
1927         dev_kfree_skb_any(skb);
1928         return NETDEV_TX_BUSY;
1929 }
1930
1931 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1932                                  struct chcr_ktls_info *tx_info,
1933                                  struct sge_eth_txq *q)
1934 {
1935         u32 data_len, skb_offset;
1936         struct sk_buff *nskb;
1937         struct tcphdr *th;
1938
1939         nskb = tls_encrypt_skb(skb);
1940
1941         if (!nskb)
1942                 return 0;
1943
1944         th = tcp_hdr(nskb);
1945         skb_offset =  skb_transport_offset(nskb) + tcp_hdrlen(nskb);
1946         data_len = nskb->len - skb_offset;
1947         skb_tx_timestamp(nskb);
1948
1949         if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1950                 goto out;
1951
1952         tx_info->prev_seq = ntohl(th->seq) + data_len;
1953         atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1954         return 0;
1955 out:
1956         dev_kfree_skb_any(nskb);
1957         return 0;
1958 }
1959 /* nic tls TX handler */
1960 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1961 {
1962         u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1963         struct ch_ktls_port_stats_debug *port_stats;
1964         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1965         struct ch_ktls_stats_debug *stats;
1966         struct tcphdr *th = tcp_hdr(skb);
1967         int data_len, qidx, ret = 0, mss;
1968         struct tls_record_info *record;
1969         struct chcr_ktls_info *tx_info;
1970         struct tls_context *tls_ctx;
1971         struct sge_eth_txq *q;
1972         struct adapter *adap;
1973         unsigned long flags;
1974
1975         tcp_seq = ntohl(th->seq);
1976         skb_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
1977         skb_data_len = skb->len - skb_offset;
1978         data_len = skb_data_len;
1979
1980         mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1981
1982         tls_ctx = tls_get_ctx(skb->sk);
1983         if (unlikely(tls_ctx->netdev != dev))
1984                 goto out;
1985
1986         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1987         tx_info = tx_ctx->chcr_info;
1988
1989         if (unlikely(!tx_info))
1990                 goto out;
1991
1992         adap = tx_info->adap;
1993         stats = &adap->ch_ktls_stats;
1994         port_stats = &stats->ktls_port[tx_info->port_id];
1995
1996         qidx = skb->queue_mapping;
1997         q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1998         cxgb4_reclaim_completed_tx(adap, &q->q, true);
1999         /* if tcp options are set but finish is not send the options first */
2000         if (!th->fin && chcr_ktls_check_tcp_options(th)) {
2001                 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
2002                                                   tx_info->tx_chan);
2003                 if (ret)
2004                         return NETDEV_TX_BUSY;
2005         }
2006
2007         /* TCP segments can be in received either complete or partial.
2008          * chcr_end_part_handler will handle cases if complete record or end
2009          * part of the record is received. In case of partial end part of record,
2010          * we will send the complete record again.
2011          */
2012
2013         do {
2014                 int i;
2015
2016                 cxgb4_reclaim_completed_tx(adap, &q->q, true);
2017                 /* lock taken */
2018                 spin_lock_irqsave(&tx_ctx->base.lock, flags);
2019                 /* fetch the tls record */
2020                 record = tls_get_record(&tx_ctx->base, tcp_seq,
2021                                         &tx_info->record_no);
2022                 /* By the time packet reached to us, ACK is received, and record
2023                  * won't be found in that case, handle it gracefully.
2024                  */
2025                 if (unlikely(!record)) {
2026                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2027                         atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
2028                         goto out;
2029                 }
2030
2031                 tls_end_offset = record->end_seq - tcp_seq;
2032
2033                 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
2034                          tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
2035                 /* update tcb for the skb */
2036                 if (skb_data_len == data_len) {
2037                         u32 tx_max = tcp_seq;
2038
2039                         if (!tls_record_is_start_marker(record) &&
2040                             tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
2041                                 tx_max = record->end_seq -
2042                                         TLS_CIPHER_AES_GCM_128_TAG_SIZE;
2043
2044                         ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
2045                                                       ntohl(th->ack_seq),
2046                                                       ntohs(th->window),
2047                                                       tls_end_offset !=
2048                                                       record->len);
2049                         if (ret) {
2050                                 spin_unlock_irqrestore(&tx_ctx->base.lock,
2051                                                        flags);
2052                                 goto out;
2053                         }
2054
2055                         if (th->fin)
2056                                 skb_get(skb);
2057                 }
2058
2059                 if (unlikely(tls_record_is_start_marker(record))) {
2060                         atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
2061                         /* If tls_end_offset < data_len, means there is some
2062                          * data after start marker, which needs encryption, send
2063                          * plaintext first and take skb refcount. else send out
2064                          * complete pkt as plaintext.
2065                          */
2066                         if (tls_end_offset < data_len)
2067                                 skb_get(skb);
2068                         else
2069                                 tls_end_offset = data_len;
2070
2071                         ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2072                                                     (!th->fin && th->psh), q,
2073                                                     tx_info->port_id, NULL,
2074                                                     tls_end_offset, skb_offset,
2075                                                     0);
2076
2077                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2078                         if (ret) {
2079                                 /* free the refcount taken earlier */
2080                                 if (tls_end_offset < data_len)
2081                                         dev_kfree_skb_any(skb);
2082                                 goto out;
2083                         }
2084
2085                         data_len -= tls_end_offset;
2086                         tcp_seq = record->end_seq;
2087                         skb_offset += tls_end_offset;
2088                         continue;
2089                 }
2090
2091                 /* increase page reference count of the record, so that there
2092                  * won't be any chance of page free in middle if in case stack
2093                  * receives ACK and try to delete the record.
2094                  */
2095                 for (i = 0; i < record->num_frags; i++)
2096                         __skb_frag_ref(&record->frags[i]);
2097                 /* lock cleared */
2098                 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2099
2100
2101                 /* if a tls record is finishing in this SKB */
2102                 if (tls_end_offset <= data_len) {
2103                         ret = chcr_end_part_handler(tx_info, skb, record,
2104                                                     tcp_seq, mss,
2105                                                     (!th->fin && th->psh), q,
2106                                                     skb_offset,
2107                                                     tls_end_offset,
2108                                                     skb_offset +
2109                                                     tls_end_offset == skb->len);
2110
2111                         data_len -= tls_end_offset;
2112                         /* tcp_seq increment is required to handle next record.
2113                          */
2114                         tcp_seq += tls_end_offset;
2115                         skb_offset += tls_end_offset;
2116                 } else {
2117                         ret = chcr_short_record_handler(tx_info, skb,
2118                                                         record, tcp_seq, mss,
2119                                                         (!th->fin && th->psh),
2120                                                         data_len, skb_offset,
2121                                                         q, tls_end_offset);
2122                         data_len = 0;
2123                 }
2124
2125                 /* clear the frag ref count which increased locally before */
2126                 for (i = 0; i < record->num_frags; i++) {
2127                         /* clear the frag ref count */
2128                         __skb_frag_unref(&record->frags[i]);
2129                 }
2130                 /* if any failure, come out from the loop. */
2131                 if (ret) {
2132                         if (th->fin)
2133                                 dev_kfree_skb_any(skb);
2134
2135                         if (ret == FALLBACK)
2136                                 return chcr_ktls_sw_fallback(skb, tx_info, q);
2137
2138                         return NETDEV_TX_OK;
2139                 }
2140
2141                 /* length should never be less than 0 */
2142                 WARN_ON(data_len < 0);
2143
2144         } while (data_len > 0);
2145
2146         atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2147         atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2148
2149         /* tcp finish is set, send a separate tcp msg including all the options
2150          * as well.
2151          */
2152         if (th->fin) {
2153                 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2154                 dev_kfree_skb_any(skb);
2155         }
2156
2157         return NETDEV_TX_OK;
2158 out:
2159         dev_kfree_skb_any(skb);
2160         return NETDEV_TX_OK;
2161 }
2162
2163 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2164 {
2165         struct chcr_ktls_uld_ctx *u_ctx;
2166
2167         pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2168                      CHCR_KTLS_DRV_VERSION);
2169         u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2170         if (!u_ctx) {
2171                 u_ctx = ERR_PTR(-ENOMEM);
2172                 goto out;
2173         }
2174         u_ctx->lldi = *lldi;
2175 out:
2176         return u_ctx;
2177 }
2178
2179 static const struct tlsdev_ops chcr_ktls_ops = {
2180         .tls_dev_add = chcr_ktls_dev_add,
2181         .tls_dev_del = chcr_ktls_dev_del,
2182 };
2183
2184 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2185         [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2186         [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2187 };
2188
2189 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2190                                     const struct pkt_gl *pgl)
2191 {
2192         const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2193         struct chcr_ktls_uld_ctx *u_ctx = handle;
2194         u8 opcode = rpl->ot.opcode;
2195         struct adapter *adap;
2196
2197         adap = pci_get_drvdata(u_ctx->lldi.pdev);
2198
2199         if (!work_handlers[opcode]) {
2200                 pr_err("Unsupported opcode %d received\n", opcode);
2201                 return 0;
2202         }
2203
2204         work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2205         return 0;
2206 }
2207
2208 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2209 {
2210         struct chcr_ktls_uld_ctx *u_ctx = handle;
2211
2212         switch (new_state) {
2213         case CXGB4_STATE_UP:
2214                 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2215                 mutex_lock(&dev_mutex);
2216                 list_add_tail(&u_ctx->entry, &uld_ctx_list);
2217                 mutex_unlock(&dev_mutex);
2218                 break;
2219         case CXGB4_STATE_START_RECOVERY:
2220         case CXGB4_STATE_DOWN:
2221         case CXGB4_STATE_DETACH:
2222                 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2223                 mutex_lock(&dev_mutex);
2224                 list_del(&u_ctx->entry);
2225                 mutex_unlock(&dev_mutex);
2226                 break;
2227         default:
2228                 break;
2229         }
2230
2231         return 0;
2232 }
2233
2234 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2235         .name = CHCR_KTLS_DRV_MODULE_NAME,
2236         .nrxq = 1,
2237         .rxq_size = 1024,
2238         .add = chcr_ktls_uld_add,
2239         .tx_handler = chcr_ktls_xmit,
2240         .rx_handler = chcr_ktls_uld_rx_handler,
2241         .state_change = chcr_ktls_uld_state_change,
2242         .tlsdev_ops = &chcr_ktls_ops,
2243 };
2244
2245 static int __init chcr_ktls_init(void)
2246 {
2247         cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2248         return 0;
2249 }
2250
2251 static void __exit chcr_ktls_exit(void)
2252 {
2253         struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2254         struct adapter *adap;
2255
2256         pr_info("driver unloaded\n");
2257
2258         mutex_lock(&dev_mutex);
2259         list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2260                 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2261                 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2262                 list_del(&u_ctx->entry);
2263                 kfree(u_ctx);
2264         }
2265         mutex_unlock(&dev_mutex);
2266         cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2267 }
2268
2269 module_init(chcr_ktls_init);
2270 module_exit(chcr_ktls_exit);
2271
2272 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2273 MODULE_LICENSE("GPL");
2274 MODULE_AUTHOR("Chelsio Communications");
2275 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);