drivers/crypto/chelsio/chcr_ktls.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /* Copyright (C) 2020 Chelsio Communications.  All rights reserved. */
   3
   4 #ifdef CONFIG_CHELSIO_TLS_DEVICE
   5 #include <linux/highmem.h>
   6 #include "chcr_ktls.h"
   7 #include "clip_tbl.h"
   8
   9 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
  10 /*
  11  * chcr_ktls_save_keys: calculate and save crypto keys.
  12  * @tx_info - driver specific tls info.
  13  * @crypto_info - tls crypto information.
  14  * @direction - TX/RX direction.
  15  * return - SUCCESS/FAILURE.
  16  */
  17 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
  18                                struct tls_crypto_info *crypto_info,
  19                                enum tls_offload_ctx_dir direction)
  20 {
  21         int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
  22         unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
  23         struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
  24         struct ktls_key_ctx *kctx = &tx_info->key_ctx;
  25         struct crypto_cipher *cipher;
  26         unsigned char *key, *salt;
  27
  28         switch (crypto_info->cipher_type) {
  29         case TLS_CIPHER_AES_GCM_128:
  30                 info_128_gcm =
  31                         (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
  32                 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
  33                 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
  34                 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
  35                 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
  36                 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
  37                 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
  38
  39                 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
  40                 key = info_128_gcm->key;
  41                 salt = info_128_gcm->salt;
  42                 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
  43
  44                 /* The SCMD fields used when encrypting a full TLS
  45                  * record. Its a one time calculation till the
  46                  * connection exists.
  47                  */
  48                 tx_info->scmd0_seqno_numivs =
  49                         SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
  50                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
  51                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
  52                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
  53                         SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
  54                         SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
  55                         SCMD_NUM_IVS_V(1);
  56
  57                 /* keys will be sent inline. */
  58                 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
  59
  60                 /* The SCMD fields used when encrypting a partial TLS
  61                  * record (no trailer and possibly a truncated payload).
  62                  */
  63                 tx_info->scmd0_short_seqno_numivs =
  64                         SCMD_CIPH_AUTH_SEQ_CTRL_F |
  65                         SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
  66                         SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
  67                         SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
  68
  69                 tx_info->scmd0_short_ivgen_hdrlen =
  70                         tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
  71
  72                 break;
  73
  74         default:
  75                 pr_err("GCM: cipher type 0x%x not supported\n",
  76                        crypto_info->cipher_type);
  77                 ret = -EINVAL;
  78                 goto out;
  79         }
  80
  81         key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
  82                        roundup(keylen, 16) + ghash_size;
  83         /* Calculate the H = CIPH(K, 0 repeated 16 times).
  84          * It will go in key context
  85          */
  86         cipher = crypto_alloc_cipher("aes", 0, 0);
  87         if (IS_ERR(cipher)) {
  88                 ret = -ENOMEM;
  89                 goto out;
  90         }
  91
  92         ret = crypto_cipher_setkey(cipher, key, keylen);
  93         if (ret)
  94                 goto out1;
  95
  96         memset(ghash_h, 0, ghash_size);
  97         crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h);
  98
  99         /* fill the Key context */
 100         if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
 101                 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
 102                                                  mac_key_size,
 103                                                  key_ctx_size >> 4);
 104         } else {
 105                 ret = -EINVAL;
 106                 goto out1;
 107         }
 108
 109         memcpy(kctx->salt, salt, tx_info->salt_size);
 110         memcpy(kctx->key, key, keylen);
 111         memcpy(kctx->key + keylen, ghash_h, ghash_size);
 112         tx_info->key_ctx_len = key_ctx_size;
 113
 114 out1:
 115         crypto_free_cipher(cipher);
 116 out:
 117         return ret;
 118 }
 119
 120 static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info,
 121                                              int new_state)
 122 {
 123         /* This function can be called from both rx (interrupt context) and tx
 124          * queue contexts.
 125          */
 126         spin_lock_bh(&tx_info->lock);
 127         switch (tx_info->connection_state) {
 128         case KTLS_CONN_CLOSED:
 129                 tx_info->connection_state = new_state;
 130                 break;
 131
 132         case KTLS_CONN_ACT_OPEN_REQ:
 133                 /* only go forward if state is greater than current state. */
 134                 if (new_state <= tx_info->connection_state)
 135                         break;
 136                 /* update to the next state and also initialize TCB */
 137                 tx_info->connection_state = new_state;
 138                 /* FALLTHRU */
 139         case KTLS_CONN_ACT_OPEN_RPL:
 140                 /* if we are stuck in this state, means tcb init might not
 141                  * received by HW, try sending it again.
 142                  */
 143                 if (!chcr_init_tcb_fields(tx_info))
 144                         tx_info->connection_state = KTLS_CONN_SET_TCB_REQ;
 145                 break;
 146
 147         case KTLS_CONN_SET_TCB_REQ:
 148                 /* only go forward if state is greater than current state. */
 149                 if (new_state <= tx_info->connection_state)
 150                         break;
 151                 /* update to the next state and check if l2t_state is valid  */
 152                 tx_info->connection_state = new_state;
 153                 /* FALLTHRU */
 154         case KTLS_CONN_SET_TCB_RPL:
 155                 /* Check if l2t state is valid, then move to ready state. */
 156                 if (cxgb4_check_l2t_valid(tx_info->l2te)) {
 157                         tx_info->connection_state = KTLS_CONN_TX_READY;
 158                         atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx);
 159                 }
 160                 break;
 161
 162         case KTLS_CONN_TX_READY:
 163                 /* nothing to be done here */
 164                 break;
 165
 166         default:
 167                 pr_err("unknown KTLS connection state\n");
 168                 break;
 169         }
 170         spin_unlock_bh(&tx_info->lock);
 171
 172         return tx_info->connection_state;
 173 }
 174 /*
 175  * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
 176  * @sk - tcp socket.
 177  * @tx_info - driver specific tls info.
 178  * @atid - connection active tid.
 179  * return - send success/failure.
 180  */
 181 static int chcr_ktls_act_open_req(struct sock *sk,
 182                                   struct chcr_ktls_info *tx_info,
 183                                   int atid)
 184 {
 185         struct inet_sock *inet = inet_sk(sk);
 186         struct cpl_t6_act_open_req *cpl6;
 187         struct cpl_act_open_req *cpl;
 188         struct sk_buff *skb;
 189         unsigned int len;
 190         int qid_atid;
 191         u64 options;
 192
 193         len = sizeof(*cpl6);
 194         skb = alloc_skb(len, GFP_KERNEL);
 195         if (unlikely(!skb))
 196                 return -ENOMEM;
 197         /* mark it a control pkt */
 198         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 199
 200         cpl6 = __skb_put_zero(skb, len);
 201         cpl = (struct cpl_act_open_req *)cpl6;
 202         INIT_TP_WR(cpl6, 0);
 203         qid_atid = TID_QID_V(tx_info->rx_qid) |
 204                    TID_TID_V(atid);
 205         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
 206         cpl->local_port = inet->inet_sport;
 207         cpl->peer_port = inet->inet_dport;
 208         cpl->local_ip = inet->inet_rcv_saddr;
 209         cpl->peer_ip = inet->inet_daddr;
 210
 211         /* fill first 64 bit option field. */
 212         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 213                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 214         cpl->opt0 = cpu_to_be64(options);
 215
 216         /* next 64 bit option field. */
 217         options =
 218                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 219         cpl->opt2 = htonl(options);
 220
 221         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 222 }
 223
 224 #if IS_ENABLED(CONFIG_IPV6)
 225 /*
 226  * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
 227  * @sk - tcp socket.
 228  * @tx_info - driver specific tls info.
 229  * @atid - connection active tid.
 230  * return - send success/failure.
 231  */
 232 static int chcr_ktls_act_open_req6(struct sock *sk,
 233                                    struct chcr_ktls_info *tx_info,
 234                                    int atid)
 235 {
 236         struct inet_sock *inet = inet_sk(sk);
 237         struct cpl_t6_act_open_req6 *cpl6;
 238         struct cpl_act_open_req6 *cpl;
 239         struct sk_buff *skb;
 240         unsigned int len;
 241         int qid_atid;
 242         u64 options;
 243
 244         len = sizeof(*cpl6);
 245         skb = alloc_skb(len, GFP_KERNEL);
 246         if (unlikely(!skb))
 247                 return -ENOMEM;
 248         /* mark it a control pkt */
 249         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 250
 251         cpl6 = __skb_put_zero(skb, len);
 252         cpl = (struct cpl_act_open_req6 *)cpl6;
 253         INIT_TP_WR(cpl6, 0);
 254         qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
 255         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
 256         cpl->local_port = inet->inet_sport;
 257         cpl->peer_port = inet->inet_dport;
 258         cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
 259         cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
 260         cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
 261         cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
 262
 263         /* first 64 bit option field. */
 264         options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
 265                   SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
 266         cpl->opt0 = cpu_to_be64(options);
 267         /* next 64 bit option field. */
 268         options =
 269                 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
 270         cpl->opt2 = htonl(options);
 271
 272         return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
 273 }
 274 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
 275
 276 /*
 277  * chcr_setup_connection:  create a TCB entry so that TP will form tcp packets.
 278  * @sk - tcp socket.
 279  * @tx_info - driver specific tls info.
 280  * return: NET_TX_OK/NET_XMIT_DROP
 281  */
 282 static int chcr_setup_connection(struct sock *sk,
 283                                  struct chcr_ktls_info *tx_info)
 284 {
 285         struct tid_info *t = &tx_info->adap->tids;
 286         int atid, ret = 0;
 287
 288         atid = cxgb4_alloc_atid(t, tx_info);
 289         if (atid == -1)
 290                 return -EINVAL;
 291
 292         tx_info->atid = atid;
 293         tx_info->ip_family = sk->sk_family;
 294
 295         if (sk->sk_family == AF_INET) {
 296                 tx_info->ip_family = AF_INET;
 297                 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
 298 #if IS_ENABLED(CONFIG_IPV6)
 299         } else {
 300                 if (!sk->sk_ipv6only &&
 301                     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
 302                         tx_info->ip_family = AF_INET;
 303                         ret = chcr_ktls_act_open_req(sk, tx_info, atid);
 304                 } else {
 305                         tx_info->ip_family = AF_INET6;
 306                         ret = cxgb4_clip_get(tx_info->netdev,
 307                                              (const u32 *)
 308                                              &sk->sk_v6_rcv_saddr.s6_addr,
 309                                              1);
 310                         if (ret)
 311                                 goto out;
 312                         ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
 313                 }
 314 #endif
 315         }
 316
 317         /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
 318          * success, if any other return type clear atid and return that failure.
 319          */
 320         if (ret) {
 321                 if (ret == NET_XMIT_CN)
 322                         ret = 0;
 323                 else
 324                         cxgb4_free_atid(t, atid);
 325                 goto out;
 326         }
 327
 328         /* update the connection state */
 329         chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ);
 330 out:
 331         return ret;
 332 }
 333
 334 /*
 335  * chcr_set_tcb_field: update tcb fields.
 336  * @tx_info - driver specific tls info.
 337  * @word - TCB word.
 338  * @mask - TCB word related mask.
 339  * @val - TCB word related value.
 340  * @no_reply - set 1 if not looking for TP response.
 341  */
 342 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
 343                               u64 mask, u64 val, int no_reply)
 344 {
 345         struct cpl_set_tcb_field *req;
 346         struct sk_buff *skb;
 347
 348         skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
 349         if (!skb)
 350                 return -ENOMEM;
 351
 352         req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
 353         INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
 354         req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 355                                 NO_REPLY_V(no_reply));
 356         req->word_cookie = htons(TCB_WORD_V(word));
 357         req->mask = cpu_to_be64(mask);
 358         req->val = cpu_to_be64(val);
 359
 360         set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
 361         return cxgb4_ofld_send(tx_info->netdev, skb);
 362 }
 363
 364 /*
 365  * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE
 366  * @tx_info - driver specific tls info.
 367  * return: NET_TX_OK/NET_XMIT_DROP.
 368  */
 369 static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info)
 370 {
 371         return chcr_set_tcb_field(tx_info, TCB_T_STATE_W,
 372                                   TCB_T_STATE_V(TCB_T_STATE_M),
 373                                   CHCR_TCB_STATE_CLOSED, 1);
 374 }
 375
 376 /*
 377  * chcr_ktls_dev_del:  call back for tls_dev_del.
 378  * Remove the tid and l2t entry and close the connection.
 379  * it per connection basis.
 380  * @netdev - net device.
 381  * @tls_cts - tls context.
 382  * @direction - TX/RX crypto direction
 383  */
 384 void chcr_ktls_dev_del(struct net_device *netdev,
 385                        struct tls_context *tls_ctx,
 386                        enum tls_offload_ctx_dir direction)
 387 {
 388         struct chcr_ktls_ofld_ctx_tx *tx_ctx =
 389                                 chcr_get_ktls_tx_context(tls_ctx);
 390         struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
 391         struct sock *sk;
 392
 393         if (!tx_info)
 394                 return;
 395         sk = tx_info->sk;
 396
 397         spin_lock(&tx_info->lock);
 398         tx_info->connection_state = KTLS_CONN_CLOSED;
 399         spin_unlock(&tx_info->lock);
 400
 401         /* clear l2t entry */
 402         if (tx_info->l2te)
 403                 cxgb4_l2t_release(tx_info->l2te);
 404
 405 #if IS_ENABLED(CONFIG_IPV6)
 406         /* clear clip entry */
 407         if (tx_info->ip_family == AF_INET6)
 408                 cxgb4_clip_release(netdev,
 409                                    (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8,
 410                                    1);
 411 #endif
 412
 413         /* clear tid */
 414         if (tx_info->tid != -1) {
 415                 /* clear tcb state and then release tid */
 416                 chcr_ktls_mark_tcb_close(tx_info);
 417                 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
 418                                  tx_info->tid, tx_info->ip_family);
 419         }
 420
 421         atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close);
 422         kvfree(tx_info);
 423         tx_ctx->chcr_info = NULL;
 424         /* release module refcount */
 425         module_put(THIS_MODULE);
 426 }
 427
 428 /*
 429  * chcr_ktls_dev_add:  call back for tls_dev_add.
 430  * Create a tcb entry for TP. Also add l2t entry for the connection. And
 431  * generate keys & save those keys locally.
 432  * @netdev - net device.
 433  * @tls_cts - tls context.
 434  * @direction - TX/RX crypto direction
 435  * return: SUCCESS/FAILURE.
 436  */
 437 int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
 438                       enum tls_offload_ctx_dir direction,
 439                       struct tls_crypto_info *crypto_info,
 440                       u32 start_offload_tcp_sn)
 441 {
 442         struct tls_context *tls_ctx = tls_get_ctx(sk);
 443         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
 444         struct chcr_ktls_info *tx_info;
 445         struct dst_entry *dst;
 446         struct adapter *adap;
 447         struct port_info *pi;
 448         struct neighbour *n;
 449         u8 daaddr[16];
 450         int ret = -1;
 451
 452         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
 453
 454         pi = netdev_priv(netdev);
 455         adap = pi->adapter;
 456         if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
 457                 pr_err("not expecting for RX direction\n");
 458                 ret = -EINVAL;
 459                 goto out;
 460         }
 461         if (tx_ctx->chcr_info) {
 462                 ret = -EINVAL;
 463                 goto out;
 464         }
 465
 466         tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
 467         if (!tx_info) {
 468                 ret = -ENOMEM;
 469                 goto out;
 470         }
 471
 472         spin_lock_init(&tx_info->lock);
 473
 474         /* clear connection state */
 475         spin_lock(&tx_info->lock);
 476         tx_info->connection_state = KTLS_CONN_CLOSED;
 477         spin_unlock(&tx_info->lock);
 478
 479         tx_info->sk = sk;
 480         /* initialize tid and atid to -1, 0 is a also a valid id. */
 481         tx_info->tid = -1;
 482         tx_info->atid = -1;
 483
 484         tx_info->adap = adap;
 485         tx_info->netdev = netdev;
 486         tx_info->first_qset = pi->first_qset;
 487         tx_info->tx_chan = pi->tx_chan;
 488         tx_info->smt_idx = pi->smt_idx;
 489         tx_info->port_id = pi->port_id;
 490
 491         tx_info->rx_qid = chcr_get_first_rx_qid(adap);
 492         if (unlikely(tx_info->rx_qid < 0))
 493                 goto out2;
 494
 495         tx_info->prev_seq = start_offload_tcp_sn;
 496         tx_info->tcp_start_seq_number = start_offload_tcp_sn;
 497
 498         /* save crypto keys */
 499         ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
 500         if (ret < 0)
 501                 goto out2;
 502
 503         /* get peer ip */
 504         if (sk->sk_family == AF_INET) {
 505                 memcpy(daaddr, &sk->sk_daddr, 4);
 506 #if IS_ENABLED(CONFIG_IPV6)
 507         } else {
 508                 if (!sk->sk_ipv6only &&
 509                     ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED)
 510                         memcpy(daaddr, &sk->sk_daddr, 4);
 511                 else
 512                         memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
 513 #endif
 514         }
 515
 516         /* get the l2t index */
 517         dst = sk_dst_get(sk);
 518         if (!dst) {
 519                 pr_err("DST entry not found\n");
 520                 goto out2;
 521         }
 522         n = dst_neigh_lookup(dst, daaddr);
 523         if (!n || !n->dev) {
 524                 pr_err("neighbour not found\n");
 525                 dst_release(dst);
 526                 goto out2;
 527         }
 528         tx_info->l2te  = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
 529
 530         neigh_release(n);
 531         dst_release(dst);
 532
 533         if (!tx_info->l2te) {
 534                 pr_err("l2t entry not found\n");
 535                 goto out2;
 536         }
 537
 538         tx_ctx->chcr_info = tx_info;
 539
 540         /* create a filter and call cxgb4_l2t_send to send the packet out, which
 541          * will take care of updating l2t entry in hw if not already done.
 542          */
 543         ret = chcr_setup_connection(sk, tx_info);
 544         if (ret)
 545                 goto out2;
 546
 547         /* Driver shouldn't be removed until any single connection exists */
 548         if (!try_module_get(THIS_MODULE)) {
 549                 ret = -EINVAL;
 550                 goto out2;
 551         }
 552
 553         atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open);
 554         return 0;
 555 out2:
 556         kvfree(tx_info);
 557 out:
 558         atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail);
 559         return ret;
 560 }
 561
 562 /*
 563  * chcr_init_tcb_fields:  Initialize tcb fields to handle TCP seq number
 564  *                        handling.
 565  * @tx_info - driver specific tls info.
 566  * return: NET_TX_OK/NET_XMIT_DROP
 567  */
 568 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
 569 {
 570         int  ret = 0;
 571
 572         /* set tcb in offload and bypass */
 573         ret =
 574         chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
 575                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
 576                            TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
 577         if (ret)
 578                 return ret;
 579         /* reset snd_una and snd_next fields in tcb */
 580         ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
 581                                  TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
 582                                  TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
 583                                  0, 1);
 584         if (ret)
 585                 return ret;
 586
 587         /* reset send max */
 588         ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
 589                                  TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
 590                                  0, 1);
 591         if (ret)
 592                 return ret;
 593
 594         /* update l2t index and request for tp reply to confirm tcb is
 595          * initialised to handle tx traffic.
 596          */
 597         ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
 598                                  TCB_L2T_IX_V(TCB_L2T_IX_M),
 599                                  TCB_L2T_IX_V(tx_info->l2te->idx), 0);
 600         return ret;
 601 }
 602
 603 /*
 604  * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
 605  */
 606 int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input)
 607 {
 608         const struct cpl_act_open_rpl *p = (void *)input;
 609         struct chcr_ktls_info *tx_info = NULL;
 610         unsigned int atid, tid, status;
 611         struct tid_info *t;
 612
 613         tid = GET_TID(p);
 614         status = AOPEN_STATUS_G(ntohl(p->atid_status));
 615         atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
 616
 617         t = &adap->tids;
 618         tx_info = lookup_atid(t, atid);
 619
 620         if (!tx_info || tx_info->atid != atid) {
 621                 pr_err("tx_info or atid is not correct\n");
 622                 return -1;
 623         }
 624
 625         if (!status) {
 626                 tx_info->tid = tid;
 627                 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
 628
 629                 cxgb4_free_atid(t, atid);
 630                 tx_info->atid = -1;
 631                 /* update the connection state */
 632                 chcr_ktls_update_connection_state(tx_info,
 633                                                   KTLS_CONN_ACT_OPEN_RPL);
 634         }
 635         return 0;
 636 }
 637
 638 /*
 639  * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
 640  */
 641 int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
 642 {
 643         const struct cpl_set_tcb_rpl *p = (void *)input;
 644         struct chcr_ktls_info *tx_info = NULL;
 645         struct tid_info *t;
 646         u32 tid;
 647
 648         tid = GET_TID(p);
 649
 650         t = &adap->tids;
 651         tx_info = lookup_tid(t, tid);
 652         if (!tx_info || tx_info->tid != tid) {
 653                 pr_err("tx_info or atid is not correct\n");
 654                 return -1;
 655         }
 656         /* update the connection state */
 657         chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL);
 658         return 0;
 659 }
 660
 661 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 662                                         u32 tid, void *pos, u16 word, u64 mask,
 663                                         u64 val, u32 reply)
 664 {
 665         struct cpl_set_tcb_field_core *cpl;
 666         struct ulptx_idata *idata;
 667         struct ulp_txpkt *txpkt;
 668
 669         /* ULP_TXPKT */
 670         txpkt = pos;
 671         txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0));
 672         txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
 673
 674         /* ULPTX_IDATA sub-command */
 675         idata = (struct ulptx_idata *)(txpkt + 1);
 676         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
 677         idata->len = htonl(sizeof(*cpl));
 678         pos = idata + 1;
 679
 680         cpl = pos;
 681         /* CPL_SET_TCB_FIELD */
 682         OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
 683         cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
 684                         NO_REPLY_V(!reply));
 685         cpl->word_cookie = htons(TCB_WORD_V(word));
 686         cpl->mask = cpu_to_be64(mask);
 687         cpl->val = cpu_to_be64(val);
 688
 689         /* ULPTX_NOOP */
 690         idata = (struct ulptx_idata *)(cpl + 1);
 691         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
 692         idata->len = htonl(0);
 693         pos = idata + 1;
 694
 695         return pos;
 696 }
 697
 698
 699 /*
 700  * chcr_write_cpl_set_tcb_ulp: update tcb values.
 701  * TCB is responsible to create tcp headers, so all the related values
 702  * should be correctly updated.
 703  * @tx_info - driver specific tls info.
 704  * @q - tx queue on which packet is going out.
 705  * @tid - TCB identifier.
 706  * @pos - current index where should we start writing.
 707  * @word - TCB word.
 708  * @mask - TCB word related mask.
 709  * @val - TCB word related value.
 710  * @reply - set 1 if looking for TP response.
 711  * return - next position to write.
 712  */
 713 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
 714                                         struct sge_eth_txq *q, u32 tid,
 715                                         void *pos, u16 word, u64 mask,
 716                                         u64 val, u32 reply)
 717 {
 718         int left = (void *)q->q.stat - pos;
 719
 720         if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
 721                 if (!left) {
 722                         pos = q->q.desc;
 723                 } else {
 724                         u8 buf[48] = {0};
 725
 726                         __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word,
 727                                                      mask, val, reply);
 728
 729                         return chcr_copy_to_txd(buf, &q->q, pos,
 730                                                 CHCR_SET_TCB_FIELD_LEN);
 731                 }
 732         }
 733
 734         pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word,
 735                                            mask, val, reply);
 736
 737         /* check again if we are at the end of the queue */
 738         if (left == CHCR_SET_TCB_FIELD_LEN)
 739                 pos = q->q.desc;
 740
 741         return pos;
 742 }
 743
 744 /*
 745  * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
 746  * with updated values like tcp seq, ack, window etc.
 747  * @tx_info - driver specific tls info.
 748  * @q - TX queue.
 749  * @tcp_seq
 750  * @tcp_ack
 751  * @tcp_win
 752  * return: NETDEV_TX_BUSY/NET_TX_OK.
 753  */
 754 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
 755                                    struct sge_eth_txq *q, u64 tcp_seq,
 756                                    u64 tcp_ack, u64 tcp_win)
 757 {
 758         bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
 759         u32 len, cpl = 0, ndesc, wr_len;
 760         struct fw_ulptx_wr *wr;
 761         int credits;
 762         void *pos;
 763
 764         wr_len = sizeof(*wr);
 765         /* there can be max 4 cpls, check if we have enough credits */
 766         len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 767         ndesc = DIV_ROUND_UP(len, 64);
 768
 769         credits = chcr_txq_avail(&q->q) - ndesc;
 770         if (unlikely(credits < 0)) {
 771                 chcr_eth_txq_stop(q);
 772                 return NETDEV_TX_BUSY;
 773         }
 774
 775         pos = &q->q.desc[q->q.pidx];
 776         /* make space for WR, we'll fill it later when we know all the cpls
 777          * being sent out and have complete length.
 778          */
 779         wr = pos;
 780         pos += wr_len;
 781         /* update tx_max if its a re-transmit or the first wr */
 782         if (first_wr || tcp_seq != tx_info->prev_seq) {
 783                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 784                                                  TCB_TX_MAX_W,
 785                                                  TCB_TX_MAX_V(TCB_TX_MAX_M),
 786                                                  TCB_TX_MAX_V(tcp_seq), 0);
 787                 cpl++;
 788         }
 789         /* reset snd una if it's a re-transmit pkt */
 790         if (tcp_seq != tx_info->prev_seq) {
 791                 /* reset snd_una */
 792                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 793                                                  TCB_SND_UNA_RAW_W,
 794                                                  TCB_SND_UNA_RAW_V
 795                                                  (TCB_SND_UNA_RAW_M),
 796                                                  TCB_SND_UNA_RAW_V(0), 0);
 797                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo);
 798                 cpl++;
 799         }
 800         /* update ack */
 801         if (first_wr || tx_info->prev_ack != tcp_ack) {
 802                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 803                                                  TCB_RCV_NXT_W,
 804                                                  TCB_RCV_NXT_V(TCB_RCV_NXT_M),
 805                                                  TCB_RCV_NXT_V(tcp_ack), 0);
 806                 tx_info->prev_ack = tcp_ack;
 807                 cpl++;
 808         }
 809         /* update receive window */
 810         if (first_wr || tx_info->prev_win != tcp_win) {
 811                 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
 812                                                  TCB_RCV_WND_W,
 813                                                  TCB_RCV_WND_V(TCB_RCV_WND_M),
 814                                                  TCB_RCV_WND_V(tcp_win), 0);
 815                 tx_info->prev_win = tcp_win;
 816                 cpl++;
 817         }
 818
 819         if (cpl) {
 820                 /* get the actual length */
 821                 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
 822                 /* ULPTX wr */
 823                 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
 824                 wr->cookie = 0;
 825                 /* fill len in wr field */
 826                 wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
 827
 828                 ndesc = DIV_ROUND_UP(len, 64);
 829                 chcr_txq_advance(&q->q, ndesc);
 830                 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
 831         }
 832         return 0;
 833 }
 834
 835 /*
 836  * chcr_ktls_skb_copy
 837  * @nskb - new skb where the frags to be added.
 838  * @skb - old skb from which frags will be copied.
 839  */
 840 static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb)
 841 {
 842         int i;
 843
 844         for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) {
 845                 skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i];
 846                 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
 847         }
 848
 849         skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags;
 850         nskb->len += skb->data_len;
 851         nskb->data_len = skb->data_len;
 852         nskb->truesize += skb->data_len;
 853 }
 854
 855 /*
 856  * chcr_ktls_get_tx_flits
 857  * returns number of flits to be sent out, it includes key context length, WR
 858  * size and skb fragments.
 859  */
 860 static unsigned int
 861 chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len)
 862 {
 863         return chcr_sgl_len(skb_shinfo(skb)->nr_frags) +
 864                DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
 865 }
 866
 867 /*
 868  * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale
 869  * other than timestamp.
 870  * @skb - skb contains partial record..
 871  * return: 1 / 0
 872  */
 873 static int
 874 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
 875 {
 876         int cnt, opt, optlen;
 877         u_char *cp;
 878
 879         cp = (u_char *)(tcp + 1);
 880         cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
 881         for (; cnt > 0; cnt -= optlen, cp += optlen) {
 882                 opt = cp[0];
 883                 if (opt == TCPOPT_EOL)
 884                         break;
 885                 if (opt == TCPOPT_NOP) {
 886                         optlen = 1;
 887                 } else {
 888                         if (cnt < 2)
 889                                 break;
 890                         optlen = cp[1];
 891                         if (optlen < 2 || optlen > cnt)
 892                                 break;
 893                 }
 894                 switch (opt) {
 895                 case TCPOPT_NOP:
 896                         break;
 897                 default:
 898                         return 1;
 899                 }
 900         }
 901         return 0;
 902 }
 903
 904 /*
 905  * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
 906  * send out separately.
 907  * @tx_info - driver specific tls info.
 908  * @skb - skb contains partial record..
 909  * @q - TX queue.
 910  * @tx_chan - channel number.
 911  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
 912  */
 913 static int
 914 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
 915                             struct sge_eth_txq *q, uint32_t tx_chan)
 916 {
 917         struct fw_eth_tx_pkt_wr *wr;
 918         struct cpl_tx_pkt_core *cpl;
 919         u32 ctrl, iplen, maclen;
 920 #if IS_ENABLED(CONFIG_IPV6)
 921         struct ipv6hdr *ip6;
 922 #endif
 923         unsigned int ndesc;
 924         struct tcphdr *tcp;
 925         int len16, pktlen;
 926         struct iphdr *ip;
 927         int credits;
 928         u8 buf[150];
 929         void *pos;
 930
 931         iplen = skb_network_header_len(skb);
 932         maclen = skb_mac_header_len(skb);
 933
 934         /* packet length = eth hdr len + ip hdr len + tcp hdr len
 935          * (including options).
 936          */
 937         pktlen = skb->len - skb->data_len;
 938
 939         ctrl = sizeof(*cpl) + pktlen;
 940         len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
 941         /* check how many descriptors needed */
 942         ndesc = DIV_ROUND_UP(len16, 4);
 943
 944         credits = chcr_txq_avail(&q->q) - ndesc;
 945         if (unlikely(credits < 0)) {
 946                 chcr_eth_txq_stop(q);
 947                 return NETDEV_TX_BUSY;
 948         }
 949
 950         pos = &q->q.desc[q->q.pidx];
 951         wr = pos;
 952
 953         /* Firmware work request header */
 954         wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
 955                                FW_WR_IMMDLEN_V(ctrl));
 956
 957         wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16));
 958         wr->r3 = 0;
 959
 960         cpl = (void *)(wr + 1);
 961
 962         /* CPL header */
 963         cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
 964                            TXPKT_PF_V(tx_info->adap->pf));
 965         cpl->pack = 0;
 966         cpl->len = htons(pktlen);
 967         /* checksum offload */
 968         cpl->ctrl1 = 0;
 969
 970         pos = cpl + 1;
 971
 972         memcpy(buf, skb->data, pktlen);
 973         if (tx_info->ip_family == AF_INET) {
 974                 /* we need to correct ip header len */
 975                 ip = (struct iphdr *)(buf + maclen);
 976                 ip->tot_len = htons(pktlen - maclen);
 977 #if IS_ENABLED(CONFIG_IPV6)
 978         } else {
 979                 ip6 = (struct ipv6hdr *)(buf + maclen);
 980                 ip6->payload_len = htons(pktlen - maclen - iplen);
 981 #endif
 982         }
 983         /* now take care of the tcp header, if fin is not set then clear push
 984          * bit as well, and if fin is set, it will be sent at the last so we
 985          * need to update the tcp sequence number as per the last packet.
 986          */
 987         tcp = (struct tcphdr *)(buf + maclen + iplen);
 988
 989         if (!tcp->fin)
 990                 tcp->psh = 0;
 991         else
 992                 tcp->seq = htonl(tx_info->prev_seq);
 993
 994         chcr_copy_to_txd(buf, &q->q, pos, pktlen);
 995
 996         chcr_txq_advance(&q->q, ndesc);
 997         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
 998         return 0;
 999 }
1000
1001 /* chcr_ktls_skb_shift - Shifts request length paged data from skb to another.
1002  * @tgt- buffer into which tail data gets added
1003  * @skb- buffer from which the paged data comes from
1004  * @shiftlen- shift up to this many bytes
1005  */
1006 static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb,
1007                                int shiftlen)
1008 {
1009         skb_frag_t *fragfrom, *fragto;
1010         int from, to, todo;
1011
1012         WARN_ON(shiftlen > skb->data_len);
1013
1014         todo = shiftlen;
1015         from = 0;
1016         to = 0;
1017         fragfrom = &skb_shinfo(skb)->frags[from];
1018
1019         while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) {
1020                 fragfrom = &skb_shinfo(skb)->frags[from];
1021                 fragto = &skb_shinfo(tgt)->frags[to];
1022
1023                 if (todo >= skb_frag_size(fragfrom)) {
1024                         *fragto = *fragfrom;
1025                         todo -= skb_frag_size(fragfrom);
1026                         from++;
1027                         to++;
1028
1029                 } else {
1030                         __skb_frag_ref(fragfrom);
1031                         skb_frag_page_copy(fragto, fragfrom);
1032                         skb_frag_off_copy(fragto, fragfrom);
1033                         skb_frag_size_set(fragto, todo);
1034
1035                         skb_frag_off_add(fragfrom, todo);
1036                         skb_frag_size_sub(fragfrom, todo);
1037                         todo = 0;
1038
1039                         to++;
1040                         break;
1041                 }
1042         }
1043
1044         /* Ready to "commit" this state change to tgt */
1045         skb_shinfo(tgt)->nr_frags = to;
1046
1047         /* Reposition in the original skb */
1048         to = 0;
1049         while (from < skb_shinfo(skb)->nr_frags)
1050                 skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++];
1051
1052         skb_shinfo(skb)->nr_frags = to;
1053
1054         WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags);
1055
1056         skb->len -= shiftlen;
1057         skb->data_len -= shiftlen;
1058         skb->truesize -= shiftlen;
1059         tgt->len += shiftlen;
1060         tgt->data_len += shiftlen;
1061         tgt->truesize += shiftlen;
1062
1063         return shiftlen;
1064 }
1065
1066 /*
1067  * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1068  * received has partial end part of the record, send out the complete record, so
1069  * that crypto block will be able to generate TAG/HASH.
1070  * @skb - segment which has complete or partial end part.
1071  * @tx_info - driver specific tls info.
1072  * @q - TX queue.
1073  * @tcp_seq
1074  * @tcp_push - tcp push bit.
1075  * @mss - segment size.
1076  * return: NETDEV_TX_BUSY/NET_TX_OK.
1077  */
1078 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1079                                       struct chcr_ktls_info *tx_info,
1080                                       struct sge_eth_txq *q, u32 tcp_seq,
1081                                       bool tcp_push, u32 mss)
1082 {
1083         u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1084         struct adapter *adap = tx_info->adap;
1085         int credits, left, last_desc;
1086         struct tx_sw_desc *sgl_sdesc;
1087         struct cpl_tx_data *tx_data;
1088         struct cpl_tx_sec_pdu *cpl;
1089         struct ulptx_idata *idata;
1090         struct ulp_txpkt *ulptx;
1091         struct fw_ulptx_wr *wr;
1092         void *pos;
1093         u64 *end;
1094
1095         /* get the number of flits required */
1096         flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len);
1097         /* number of descriptors */
1098         ndesc = chcr_flits_to_desc(flits);
1099         /* check if enough credits available */
1100         credits = chcr_txq_avail(&q->q) - ndesc;
1101         if (unlikely(credits < 0)) {
1102                 chcr_eth_txq_stop(q);
1103                 return NETDEV_TX_BUSY;
1104         }
1105
1106         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1107                 /* Credits are below the threshold vaues, stop the queue after
1108                  * injecting the Work Request for this packet.
1109                  */
1110                 chcr_eth_txq_stop(q);
1111                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1112         }
1113
1114         last_desc = q->q.pidx + ndesc - 1;
1115         if (last_desc >= q->q.size)
1116                 last_desc -= q->q.size;
1117         sgl_sdesc = &q->q.sdesc[last_desc];
1118
1119         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1120                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1121                 q->mapping_err++;
1122                 return NETDEV_TX_BUSY;
1123         }
1124
1125         pos = &q->q.desc[q->q.pidx];
1126         end = (u64 *)pos + flits;
1127         /* FW_ULPTX_WR */
1128         wr = pos;
1129         /* WR will need len16 */
1130         len16 = DIV_ROUND_UP(flits, 2);
1131         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1132         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1133         wr->cookie = 0;
1134         pos += sizeof(*wr);
1135         /* ULP_TXPKT */
1136         ulptx = pos;
1137         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1138                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1139                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1140                                 ULP_TXPKT_RO_F);
1141         ulptx->len = htonl(len16 - 1);
1142         /* ULPTX_IDATA sub-command */
1143         idata = (struct ulptx_idata *)(ulptx + 1);
1144         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1145         /* idata length will include cpl_tx_sec_pdu + key context size +
1146          * cpl_tx_data header.
1147          */
1148         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1149                            sizeof(*tx_data));
1150         /* SEC CPL */
1151         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1152         cpl->op_ivinsrtofst =
1153                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1154                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1155                       CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1156                       CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1157         cpl->pldlen = htonl(skb->data_len);
1158
1159         /* encryption should start after tls header size + iv size */
1160         cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1161
1162         cpl->aadstart_cipherstop_hi =
1163                 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1164                       CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1165                       CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1166
1167         /* authentication will also start after tls header + iv size */
1168         cpl->cipherstop_lo_authinsert =
1169         htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1170               CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1171               CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1172
1173         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1174         cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1175         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1176         cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1177
1178         pos = cpl + 1;
1179         /* check if space left to fill the keys */
1180         left = (void *)q->q.stat - pos;
1181         if (!left) {
1182                 left = (void *)end - (void *)q->q.stat;
1183                 pos = q->q.desc;
1184                 end = pos + left;
1185         }
1186
1187         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1188                                tx_info->key_ctx_len);
1189         left = (void *)q->q.stat - pos;
1190
1191         if (!left) {
1192                 left = (void *)end - (void *)q->q.stat;
1193                 pos = q->q.desc;
1194                 end = pos + left;
1195         }
1196         /* CPL_TX_DATA */
1197         tx_data = (void *)pos;
1198         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1199         tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len));
1200
1201         tx_data->rsvd = htonl(tcp_seq);
1202
1203         tx_data->flags = htonl(TX_BYPASS_F);
1204         if (tcp_push)
1205                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1206
1207         /* check left again, it might go beyond queue limit */
1208         pos = tx_data + 1;
1209         left = (void *)q->q.stat - pos;
1210
1211         /* check the position again */
1212         if (!left) {
1213                 left = (void *)end - (void *)q->q.stat;
1214                 pos = q->q.desc;
1215                 end = pos + left;
1216         }
1217
1218         /* send the complete packet except the header */
1219         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1220                         sgl_sdesc->addr);
1221         sgl_sdesc->skb = skb;
1222
1223         chcr_txq_advance(&q->q, ndesc);
1224         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1225         atomic64_inc(&adap->chcr_stats.ktls_tx_send_records);
1226
1227         return 0;
1228 }
1229
1230 /*
1231  * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1232  * a middle part of a record, fetch the prior data to make it 16 byte aligned
1233  * and then only send it out.
1234  *
1235  * @skb - skb contains partial record..
1236  * @tx_info - driver specific tls info.
1237  * @q - TX queue.
1238  * @tcp_seq
1239  * @tcp_push - tcp push bit.
1240  * @mss - segment size.
1241  * @tls_rec_offset - offset from start of the tls record.
1242  * @perior_data - data before the current segment, required to make this record
1243  *                16 byte aligned.
1244  * @prior_data_len - prior_data length (less than 16)
1245  * return: NETDEV_TX_BUSY/NET_TX_OK.
1246  */
1247 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1248                                    struct chcr_ktls_info *tx_info,
1249                                    struct sge_eth_txq *q,
1250                                    u32 tcp_seq, bool tcp_push, u32 mss,
1251                                    u32 tls_rec_offset, u8 *prior_data,
1252                                    u32 prior_data_len)
1253 {
1254         struct adapter *adap = tx_info->adap;
1255         u32 len16, wr_mid = 0, cipher_start;
1256         unsigned int flits = 0, ndesc;
1257         int credits, left, last_desc;
1258         struct tx_sw_desc *sgl_sdesc;
1259         struct cpl_tx_data *tx_data;
1260         struct cpl_tx_sec_pdu *cpl;
1261         struct ulptx_idata *idata;
1262         struct ulp_txpkt *ulptx;
1263         struct fw_ulptx_wr *wr;
1264         __be64 iv_record;
1265         void *pos;
1266         u64 *end;
1267
1268         /* get the number of flits required, it's a partial record so 2 flits
1269          * (AES_BLOCK_SIZE) will be added.
1270          */
1271         flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2;
1272         /* get the correct 8 byte IV of this record */
1273         iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1274         /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1275          * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1276          * data will be added.
1277          */
1278         if (prior_data_len)
1279                 flits += 2;
1280         /* number of descriptors */
1281         ndesc = chcr_flits_to_desc(flits);
1282         /* check if enough credits available */
1283         credits = chcr_txq_avail(&q->q) - ndesc;
1284         if (unlikely(credits < 0)) {
1285                 chcr_eth_txq_stop(q);
1286                 return NETDEV_TX_BUSY;
1287         }
1288
1289         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1290                 chcr_eth_txq_stop(q);
1291                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1292         }
1293
1294         last_desc = q->q.pidx + ndesc - 1;
1295         if (last_desc >= q->q.size)
1296                 last_desc -= q->q.size;
1297         sgl_sdesc = &q->q.sdesc[last_desc];
1298
1299         if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1300                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1301                 q->mapping_err++;
1302                 return NETDEV_TX_BUSY;
1303         }
1304
1305         pos = &q->q.desc[q->q.pidx];
1306         end = (u64 *)pos + flits;
1307         /* FW_ULPTX_WR */
1308         wr = pos;
1309         /* WR will need len16 */
1310         len16 = DIV_ROUND_UP(flits, 2);
1311         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1312         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1313         wr->cookie = 0;
1314         pos += sizeof(*wr);
1315         /* ULP_TXPKT */
1316         ulptx = pos;
1317         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1318                                 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1319                                 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1320                                 ULP_TXPKT_RO_F);
1321         ulptx->len = htonl(len16 - 1);
1322         /* ULPTX_IDATA sub-command */
1323         idata = (struct ulptx_idata *)(ulptx + 1);
1324         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1325         /* idata length will include cpl_tx_sec_pdu + key context size +
1326          * cpl_tx_data header.
1327          */
1328         idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1329                            sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1330         /* SEC CPL */
1331         cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1332         /* cipher start will have tls header + iv size extra if its a header
1333          * part of tls record. else only 16 byte IV will be added.
1334          */
1335         cipher_start =
1336                 AES_BLOCK_LEN + 1 +
1337                 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1338
1339         cpl->op_ivinsrtofst =
1340                 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1341                       CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1342                       CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1343         cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len);
1344         cpl->aadstart_cipherstop_hi =
1345                 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1346         cpl->cipherstop_lo_authinsert = 0;
1347         /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1348         cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1349         cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1350         cpl->scmd1 = 0;
1351
1352         pos = cpl + 1;
1353         /* check if space left to fill the keys */
1354         left = (void *)q->q.stat - pos;
1355         if (!left) {
1356                 left = (void *)end - (void *)q->q.stat;
1357                 pos = q->q.desc;
1358                 end = pos + left;
1359         }
1360
1361         pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1362                                tx_info->key_ctx_len);
1363         left = (void *)q->q.stat - pos;
1364
1365         if (!left) {
1366                 left = (void *)end - (void *)q->q.stat;
1367                 pos = q->q.desc;
1368                 end = pos + left;
1369         }
1370         /* CPL_TX_DATA */
1371         tx_data = (void *)pos;
1372         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1373         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1374                         TX_LENGTH_V(skb->data_len + prior_data_len));
1375         tx_data->rsvd = htonl(tcp_seq);
1376         tx_data->flags = htonl(TX_BYPASS_F);
1377         if (tcp_push)
1378                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1379
1380         /* check left again, it might go beyond queue limit */
1381         pos = tx_data + 1;
1382         left = (void *)q->q.stat - pos;
1383
1384         /* check the position again */
1385         if (!left) {
1386                 left = (void *)end - (void *)q->q.stat;
1387                 pos = q->q.desc;
1388                 end = pos + left;
1389         }
1390         /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1391          * bytes of actual IV and 4 bytes of 16 byte-sequence.
1392          */
1393         memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1394         memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1395         *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1396                 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1397                 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1398
1399         pos += 16;
1400         /* Prior_data_len will always be less than 16 bytes, fill the
1401          * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1402          * to 0.
1403          */
1404         if (prior_data_len)
1405                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1406         /* send the complete packet except the header */
1407         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1408                         sgl_sdesc->addr);
1409         sgl_sdesc->skb = skb;
1410
1411         chcr_txq_advance(&q->q, ndesc);
1412         cxgb4_ring_tx_db(adap, &q->q, ndesc);
1413
1414         return 0;
1415 }
1416
1417 /*
1418  * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1419  * only plain text (only tls header and iv)
1420  * @tx_info - driver specific tls info.
1421  * @skb - skb contains partial record..
1422  * @tcp_seq
1423  * @mss - segment size.
1424  * @tcp_push - tcp push bit.
1425  * @q - TX queue.
1426  * @port_id : port number
1427  * @perior_data - data before the current segment, required to make this record
1428  *               16 byte aligned.
1429  * @prior_data_len - prior_data length (less than 16)
1430  * return: NETDEV_TX_BUSY/NET_TX_OK.
1431  */
1432 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1433                                  struct sk_buff *skb, u32 tcp_seq, u32 mss,
1434                                  bool tcp_push, struct sge_eth_txq *q,
1435                                  u32 port_id, u8 *prior_data,
1436                                  u32 prior_data_len)
1437 {
1438         int credits, left, len16, last_desc;
1439         unsigned int flits = 0, ndesc;
1440         struct tx_sw_desc *sgl_sdesc;
1441         struct cpl_tx_data *tx_data;
1442         struct ulptx_idata *idata;
1443         struct ulp_txpkt *ulptx;
1444         struct fw_ulptx_wr *wr;
1445         u32 wr_mid = 0;
1446         void *pos;
1447         u64 *end;
1448
1449         flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1450         flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags);
1451         if (prior_data_len)
1452                 flits += 2;
1453         /* WR will need len16 */
1454         len16 = DIV_ROUND_UP(flits, 2);
1455         /* check how many descriptors needed */
1456         ndesc = DIV_ROUND_UP(flits, 8);
1457
1458         credits = chcr_txq_avail(&q->q) - ndesc;
1459         if (unlikely(credits < 0)) {
1460                 chcr_eth_txq_stop(q);
1461                 return NETDEV_TX_BUSY;
1462         }
1463
1464         if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1465                 chcr_eth_txq_stop(q);
1466                 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1467         }
1468
1469         last_desc = q->q.pidx + ndesc - 1;
1470         if (last_desc >= q->q.size)
1471                 last_desc -= q->q.size;
1472         sgl_sdesc = &q->q.sdesc[last_desc];
1473
1474         if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1475                                    sgl_sdesc->addr) < 0)) {
1476                 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1477                 q->mapping_err++;
1478                 return NETDEV_TX_BUSY;
1479         }
1480
1481         pos = &q->q.desc[q->q.pidx];
1482         end = (u64 *)pos + flits;
1483         /* FW_ULPTX_WR */
1484         wr = pos;
1485         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1486         wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1487         wr->cookie = 0;
1488         pos += sizeof(*wr);
1489         /* ULP_TXPKT */
1490         ulptx = (struct ulp_txpkt *)(wr + 1);
1491         ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1492                         ULP_TXPKT_DATAMODIFY_V(0) |
1493                         ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1494                         ULP_TXPKT_DEST_V(0) |
1495                         ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1496         ulptx->len = htonl(len16 - 1);
1497         /* ULPTX_IDATA sub-command */
1498         idata = (struct ulptx_idata *)(ulptx + 1);
1499         idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1500         idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1501         /* CPL_TX_DATA */
1502         tx_data = (struct cpl_tx_data *)(idata + 1);
1503         OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1504         tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1505                         TX_LENGTH_V(skb->data_len + prior_data_len));
1506         /* set tcp seq number */
1507         tx_data->rsvd = htonl(tcp_seq);
1508         tx_data->flags = htonl(TX_BYPASS_F);
1509         if (tcp_push)
1510                 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1511
1512         pos = tx_data + 1;
1513         /* apart from prior_data_len, we should set remaining part of 16 bytes
1514          * to be zero.
1515          */
1516         if (prior_data_len)
1517                 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1518
1519         /* check left again, it might go beyond queue limit */
1520         left = (void *)q->q.stat - pos;
1521
1522         /* check the position again */
1523         if (!left) {
1524                 left = (void *)end - (void *)q->q.stat;
1525                 pos = q->q.desc;
1526                 end = pos + left;
1527         }
1528         /* send the complete packet including the header */
1529         cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len,
1530                         sgl_sdesc->addr);
1531         sgl_sdesc->skb = skb;
1532
1533         chcr_txq_advance(&q->q, ndesc);
1534         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1535         return 0;
1536 }
1537
1538 /*
1539  * chcr_ktls_copy_record_in_skb
1540  * @nskb - new skb where the frags to be added.
1541  * @record - specific record which has complete 16k record in frags.
1542  */
1543 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1544                                          struct tls_record_info *record)
1545 {
1546         int i = 0;
1547
1548         for (i = 0; i < record->num_frags; i++) {
1549                 skb_shinfo(nskb)->frags[i] = record->frags[i];
1550                 /* increase the frag ref count */
1551                 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1552         }
1553
1554         skb_shinfo(nskb)->nr_frags = record->num_frags;
1555         nskb->data_len = record->len;
1556         nskb->len += record->len;
1557         nskb->truesize += record->len;
1558 }
1559
1560 /*
1561  * chcr_ktls_update_snd_una:  Reset the SEND_UNA. It will be done to avoid
1562  * sending the same segment again. It will discard the segment which is before
1563  * the current tx max.
1564  * @tx_info - driver specific tls info.
1565  * @q - TX queue.
1566  * return: NET_TX_OK/NET_XMIT_DROP.
1567  */
1568 static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info,
1569                                     struct sge_eth_txq *q)
1570 {
1571         struct fw_ulptx_wr *wr;
1572         unsigned int ndesc;
1573         int credits;
1574         void *pos;
1575         u32 len;
1576
1577         len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16);
1578         ndesc = DIV_ROUND_UP(len, 64);
1579
1580         credits = chcr_txq_avail(&q->q) - ndesc;
1581         if (unlikely(credits < 0)) {
1582                 chcr_eth_txq_stop(q);
1583                 return NETDEV_TX_BUSY;
1584         }
1585
1586         pos = &q->q.desc[q->q.pidx];
1587
1588         wr = pos;
1589         /* ULPTX wr */
1590         wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1591         wr->cookie = 0;
1592         /* fill len in wr field */
1593         wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
1594
1595         pos += sizeof(*wr);
1596
1597         pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
1598                                          TCB_SND_UNA_RAW_W,
1599                                          TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
1600                                          TCB_SND_UNA_RAW_V(0), 0);
1601
1602         chcr_txq_advance(&q->q, ndesc);
1603         cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1604
1605         return 0;
1606 }
1607
1608 /*
1609  * chcr_end_part_handler: This handler will handle the record which
1610  * is complete or if record's end part is received. T6 adapter has a issue that
1611  * it can't send out TAG with partial record so if its an end part then we have
1612  * to send TAG as well and for which we need to fetch the complete record and
1613  * send it to crypto module.
1614  * @tx_info - driver specific tls info.
1615  * @skb - skb contains partial record.
1616  * @record - complete record of 16K size.
1617  * @tcp_seq
1618  * @mss - segment size in which TP needs to chop a packet.
1619  * @tcp_push_no_fin - tcp push if fin is not set.
1620  * @q - TX queue.
1621  * @tls_end_offset - offset from end of the record.
1622  * @last wr : check if this is the last part of the skb going out.
1623  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1624  */
1625 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1626                                  struct sk_buff *skb,
1627                                  struct tls_record_info *record,
1628                                  u32 tcp_seq, int mss, bool tcp_push_no_fin,
1629                                  struct sge_eth_txq *q,
1630                                  u32 tls_end_offset, bool last_wr)
1631 {
1632         struct sk_buff *nskb = NULL;
1633         /* check if it is a complete record */
1634         if (tls_end_offset == record->len) {
1635                 nskb = skb;
1636                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts);
1637         } else {
1638                 dev_kfree_skb_any(skb);
1639
1640                 nskb = alloc_skb(0, GFP_KERNEL);
1641                 if (!nskb)
1642                         return NETDEV_TX_BUSY;
1643                 /* copy complete record in skb */
1644                 chcr_ktls_copy_record_in_skb(nskb, record);
1645                 /* packet is being sent from the beginning, update the tcp_seq
1646                  * accordingly.
1647                  */
1648                 tcp_seq = tls_record_start_seq(record);
1649                 /* reset snd una, so the middle record won't send the already
1650                  * sent part.
1651                  */
1652                 if (chcr_ktls_update_snd_una(tx_info, q))
1653                         goto out;
1654                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts);
1655         }
1656
1657         if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1658                                        (last_wr && tcp_push_no_fin),
1659                                        mss)) {
1660                 goto out;
1661         }
1662         return 0;
1663 out:
1664         dev_kfree_skb_any(nskb);
1665         return NETDEV_TX_BUSY;
1666 }
1667
1668 /*
1669  * chcr_short_record_handler: This handler will take care of the records which
1670  * doesn't have end part (1st part or the middle part(/s) of a record). In such
1671  * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1672  * This partial record might be the first part of the record, or the middle
1673  * part. In case of middle record we should fetch the prior data to make it 16
1674  * byte aligned. If it has a partial tls header or iv then get to the start of
1675  * tls header. And if it has partial TAG, then remove the complete TAG and send
1676  * only the payload.
1677  * There is one more possibility that it gets a partial header, send that
1678  * portion as a plaintext.
1679  * @tx_info - driver specific tls info.
1680  * @skb - skb contains partial record..
1681  * @record - complete record of 16K size.
1682  * @tcp_seq
1683  * @mss - segment size in which TP needs to chop a packet.
1684  * @tcp_push_no_fin - tcp push if fin is not set.
1685  * @q - TX queue.
1686  * @tls_end_offset - offset from end of the record.
1687  * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1688  */
1689 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1690                                      struct sk_buff *skb,
1691                                      struct tls_record_info *record,
1692                                      u32 tcp_seq, int mss, bool tcp_push_no_fin,
1693                                      struct sge_eth_txq *q, u32 tls_end_offset)
1694 {
1695         u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1696         u8 prior_data[16] = {0};
1697         u32 prior_data_len = 0;
1698         u32 data_len;
1699
1700         /* check if the skb is ending in middle of tag/HASH, its a big
1701          * trouble, send the packet before the HASH.
1702          */
1703         int remaining_record = tls_end_offset - skb->data_len;
1704
1705         if (remaining_record > 0 &&
1706             remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1707                 int trimmed_len = skb->data_len -
1708                         (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record);
1709                 struct sk_buff *tmp_skb = NULL;
1710                 /* don't process the pkt if it is only a partial tag */
1711                 if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1712                         goto out;
1713
1714                 WARN_ON(trimmed_len > skb->data_len);
1715
1716                 /* shift to those many bytes */
1717                 tmp_skb = alloc_skb(0, GFP_KERNEL);
1718                 if (unlikely(!tmp_skb))
1719                         goto out;
1720
1721                 chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len);
1722                 /* free the last trimmed portion */
1723                 dev_kfree_skb_any(skb);
1724                 skb = tmp_skb;
1725                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts);
1726         }
1727         data_len = skb->data_len;
1728         /* check if the middle record's start point is 16 byte aligned. CTR
1729          * needs 16 byte aligned start point to start encryption.
1730          */
1731         if (tls_rec_offset) {
1732                 /* there is an offset from start, means its a middle record */
1733                 int remaining = 0;
1734
1735                 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1736                         prior_data_len = tls_rec_offset;
1737                         tls_rec_offset = 0;
1738                         remaining = 0;
1739                 } else {
1740                         prior_data_len =
1741                                 (tls_rec_offset -
1742                                 (TLS_HEADER_SIZE + tx_info->iv_size))
1743                                 % AES_BLOCK_LEN;
1744                         remaining = tls_rec_offset - prior_data_len;
1745                 }
1746
1747                 /* if prior_data_len is not zero, means we need to fetch prior
1748                  * data to make this record 16 byte aligned, or we need to reach
1749                  * to start offset.
1750                  */
1751                 if (prior_data_len) {
1752                         int i = 0;
1753                         u8 *data = NULL;
1754                         skb_frag_t *f;
1755                         u8 *vaddr;
1756                         int frag_size = 0, frag_delta = 0;
1757
1758                         while (remaining > 0) {
1759                                 frag_size = skb_frag_size(&record->frags[i]);
1760                                 if (remaining < frag_size)
1761                                         break;
1762
1763                                 remaining -= frag_size;
1764                                 i++;
1765                         }
1766                         f = &record->frags[i];
1767                         vaddr = kmap_atomic(skb_frag_page(f));
1768
1769                         data = vaddr + skb_frag_off(f)  + remaining;
1770                         frag_delta = skb_frag_size(f) - remaining;
1771
1772                         if (frag_delta >= prior_data_len) {
1773                                 memcpy(prior_data, data, prior_data_len);
1774                                 kunmap_atomic(vaddr);
1775                         } else {
1776                                 memcpy(prior_data, data, frag_delta);
1777                                 kunmap_atomic(vaddr);
1778                                 /* get the next page */
1779                                 f = &record->frags[i + 1];
1780                                 vaddr = kmap_atomic(skb_frag_page(f));
1781                                 data = vaddr + skb_frag_off(f);
1782                                 memcpy(prior_data + frag_delta,
1783                                        data, (prior_data_len - frag_delta));
1784                                 kunmap_atomic(vaddr);
1785                         }
1786                         /* reset tcp_seq as per the prior_data_required len */
1787                         tcp_seq -= prior_data_len;
1788                         /* include prio_data_len for  further calculation.
1789                          */
1790                         data_len += prior_data_len;
1791                 }
1792                 /* reset snd una, so the middle record won't send the already
1793                  * sent part.
1794                  */
1795                 if (chcr_ktls_update_snd_una(tx_info, q))
1796                         goto out;
1797                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts);
1798         } else {
1799                 /* Else means, its a partial first part of the record. Check if
1800                  * its only the header, don't need to send for encryption then.
1801                  */
1802                 if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) {
1803                         if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1804                                                   tcp_push_no_fin, q,
1805                                                   tx_info->port_id,
1806                                                   prior_data,
1807                                                   prior_data_len)) {
1808                                 goto out;
1809                         }
1810                         return 0;
1811                 }
1812                 atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts);
1813         }
1814
1815         if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1816                                     mss, tls_rec_offset, prior_data,
1817                                     prior_data_len)) {
1818                 goto out;
1819         }
1820
1821         return 0;
1822 out:
1823         dev_kfree_skb_any(skb);
1824         return NETDEV_TX_BUSY;
1825 }
1826
1827 /* nic tls TX handler */
1828 int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1829 {
1830         struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1831         struct tcphdr *th = tcp_hdr(skb);
1832         int data_len, qidx, ret = 0, mss;
1833         struct tls_record_info *record;
1834         struct chcr_stats_debug *stats;
1835         struct chcr_ktls_info *tx_info;
1836         u32 tls_end_offset, tcp_seq;
1837         struct tls_context *tls_ctx;
1838         struct sk_buff *local_skb;
1839         int new_connection_state;
1840         struct sge_eth_txq *q;
1841         struct adapter *adap;
1842         unsigned long flags;
1843
1844         tcp_seq = ntohl(th->seq);
1845
1846         mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len;
1847
1848         /* check if we haven't set it for ktls offload */
1849         if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk))
1850                 goto out;
1851
1852         tls_ctx = tls_get_ctx(skb->sk);
1853         if (unlikely(tls_ctx->netdev != dev))
1854                 goto out;
1855
1856         tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1857         tx_info = tx_ctx->chcr_info;
1858
1859         if (unlikely(!tx_info))
1860                 goto out;
1861
1862         /* check the connection state, we don't need to pass new connection
1863          * state, state machine will check and update the new state if it is
1864          * stuck due to responses not received from HW.
1865          * Start the tx handling only if state is KTLS_CONN_TX_READY.
1866          */
1867         new_connection_state = chcr_ktls_update_connection_state(tx_info, 0);
1868         if (new_connection_state != KTLS_CONN_TX_READY)
1869                 goto out;
1870
1871         /* don't touch the original skb, make a new skb to extract each records
1872          * and send them separately.
1873          */
1874         local_skb = alloc_skb(0, GFP_KERNEL);
1875
1876         if (unlikely(!local_skb))
1877                 return NETDEV_TX_BUSY;
1878
1879         adap = tx_info->adap;
1880         stats = &adap->chcr_stats;
1881
1882         qidx = skb->queue_mapping;
1883         q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1884         cxgb4_reclaim_completed_tx(adap, &q->q, true);
1885         /* if tcp options are set but finish is not send the options first */
1886         if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1887                 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1888                                                   tx_info->tx_chan);
1889                 if (ret)
1890                         return NETDEV_TX_BUSY;
1891         }
1892         /* update tcb */
1893         ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq),
1894                                       ntohl(th->ack_seq),
1895                                       ntohs(th->window));
1896         if (ret) {
1897                 dev_kfree_skb_any(local_skb);
1898                 return NETDEV_TX_BUSY;
1899         }
1900
1901         /* copy skb contents into local skb */
1902         chcr_ktls_skb_copy(skb, local_skb);
1903
1904         /* go through the skb and send only one record at a time. */
1905         data_len = skb->data_len;
1906         /* TCP segments can be in received either complete or partial.
1907          * chcr_end_part_handler will handle cases if complete record or end
1908          * part of the record is received. Incase of partial end part of record,
1909          * we will send the complete record again.
1910          */
1911
1912         do {
1913                 int i;
1914
1915                 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1916                 /* lock taken */
1917                 spin_lock_irqsave(&tx_ctx->base.lock, flags);
1918                 /* fetch the tls record */
1919                 record = tls_get_record(&tx_ctx->base, tcp_seq,
1920                                         &tx_info->record_no);
1921                 /* By the time packet reached to us, ACK is received, and record
1922                  * won't be found in that case, handle it gracefully.
1923                  */
1924                 if (unlikely(!record)) {
1925                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1926                         atomic64_inc(&stats->ktls_tx_drop_no_sync_data);
1927                         goto out;
1928                 }
1929
1930                 if (unlikely(tls_record_is_start_marker(record))) {
1931                         spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1932                         atomic64_inc(&stats->ktls_tx_skip_no_sync_data);
1933                         goto out;
1934                 }
1935
1936                 /* increase page reference count of the record, so that there
1937                  * won't be any chance of page free in middle if in case stack
1938                  * receives ACK and try to delete the record.
1939                  */
1940                 for (i = 0; i < record->num_frags; i++)
1941                         __skb_frag_ref(&record->frags[i]);
1942                 /* lock cleared */
1943                 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1944
1945                 tls_end_offset = record->end_seq - tcp_seq;
1946
1947                 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1948                          tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1949                 /* if a tls record is finishing in this SKB */
1950                 if (tls_end_offset <= data_len) {
1951                         struct sk_buff *nskb = NULL;
1952
1953                         if (tls_end_offset < data_len) {
1954                                 nskb = alloc_skb(0, GFP_KERNEL);
1955                                 if (unlikely(!nskb)) {
1956                                         ret = -ENOMEM;
1957                                         goto clear_ref;
1958                                 }
1959
1960                                 chcr_ktls_skb_shift(nskb, local_skb,
1961                                                     tls_end_offset);
1962                         } else {
1963                                 /* its the only record in this skb, directly
1964                                  * point it.
1965                                  */
1966                                 nskb = local_skb;
1967                         }
1968                         ret = chcr_end_part_handler(tx_info, nskb, record,
1969                                                     tcp_seq, mss,
1970                                                     (!th->fin && th->psh), q,
1971                                                     tls_end_offset,
1972                                                     (nskb == local_skb));
1973
1974                         if (ret && nskb != local_skb)
1975                                 dev_kfree_skb_any(local_skb);
1976
1977                         data_len -= tls_end_offset;
1978                         /* tcp_seq increment is required to handle next record.
1979                          */
1980                         tcp_seq += tls_end_offset;
1981                 } else {
1982                         ret = chcr_short_record_handler(tx_info, local_skb,
1983                                                         record, tcp_seq, mss,
1984                                                         (!th->fin && th->psh),
1985                                                         q, tls_end_offset);
1986                         data_len = 0;
1987                 }
1988 clear_ref:
1989                 /* clear the frag ref count which increased locally before */
1990                 for (i = 0; i < record->num_frags; i++) {
1991                         /* clear the frag ref count */
1992                         __skb_frag_unref(&record->frags[i]);
1993                 }
1994                 /* if any failure, come out from the loop. */
1995                 if (ret)
1996                         goto out;
1997                 /* length should never be less than 0 */
1998                 WARN_ON(data_len < 0);
1999
2000         } while (data_len > 0);
2001
2002         tx_info->prev_seq = ntohl(th->seq) + skb->data_len;
2003
2004         atomic64_inc(&stats->ktls_tx_encrypted_packets);
2005         atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes);
2006
2007         /* tcp finish is set, send a separate tcp msg including all the options
2008          * as well.
2009          */
2010         if (th->fin)
2011                 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2012
2013 out:
2014         dev_kfree_skb_any(skb);
2015         return NETDEV_TX_OK;
2016 }
2017 #endif /* CONFIG_CHELSIO_TLS_DEVICE */