From: Rohit Maheshwari Date: Thu, 10 Sep 2020 14:21:47 +0000 (+0530) Subject: crypto/chcr: move nic TLS functionality to drivers/net X-Git-Tag: microblaze-v5.11~110^2~299 X-Git-Url: http://git.monstr.eu/?a=commitdiff_plain;h=a8c16e8ed624f24b2b082fb9a193e0132a5fd108;p=linux-2.6-microblaze.git crypto/chcr: move nic TLS functionality to drivers/net This patch moves complete nic tls offload (kTLS) code from crypto directory to drivers/net/ethernet/chelsio/inline_crypto/ch_ktls directory. nic TLS is made a separate ULD of cxgb4. Signed-off-by: Rohit Maheshwari Signed-off-by: David S. Miller --- diff --git a/drivers/crypto/chelsio/Kconfig b/drivers/crypto/chelsio/Kconfig index af161dab49bd..f886401af13e 100644 --- a/drivers/crypto/chelsio/Kconfig +++ b/drivers/crypto/chelsio/Kconfig @@ -21,14 +21,3 @@ config CRYPTO_DEV_CHELSIO To compile this driver as a module, choose M here: the module will be called chcr. - -config CHELSIO_TLS_DEVICE - bool "Chelsio Inline KTLS Offload" - depends on CHELSIO_T4 - depends on TLS_DEVICE - select CRYPTO_DEV_CHELSIO - default y - help - This flag enables support for kernel tls offload over Chelsio T6 - crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled - only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled. diff --git a/drivers/crypto/chelsio/Makefile b/drivers/crypto/chelsio/Makefile index f2e8e2fb4e60..2e5df484ab01 100644 --- a/drivers/crypto/chelsio/Makefile +++ b/drivers/crypto/chelsio/Makefile @@ -3,6 +3,3 @@ ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 obj-$(CONFIG_CRYPTO_DEV_CHELSIO) += chcr.o chcr-objs := chcr_core.o chcr_algo.o -#ifdef CONFIG_CHELSIO_TLS_DEVICE -chcr-objs += chcr_ktls.o -#endif diff --git a/drivers/crypto/chelsio/chcr_common.h b/drivers/crypto/chelsio/chcr_common.h deleted file mode 100644 index 33f589cbfba1..000000000000 --- a/drivers/crypto/chelsio/chcr_common.h +++ /dev/null @@ -1,135 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifndef __CHCR_COMMON_H__ -#define __CHCR_COMMON_H__ - -#include "cxgb4.h" - -#define CHCR_MAX_SALT 4 -#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 -#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 -#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 -#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 -#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT 2 -#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 -#define CHCR_SCMD_PROTO_VERSION_TLS 0 -#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 -#define CHCR_SCMD_AUTH_MODE_GHASH 4 -#define AES_BLOCK_LEN 16 - -enum chcr_state { - CHCR_INIT = 0, - CHCR_ATTACH, - CHCR_DETACH, -}; - -struct chcr_dev { - spinlock_t lock_chcr_dev; /* chcr dev structure lock */ - enum chcr_state state; - atomic_t inflight; - int wqretry; - struct delayed_work detach_work; - struct completion detach_comp; - unsigned char tx_channel_id; -}; - -struct uld_ctx { - struct list_head entry; - struct cxgb4_lld_info lldi; - struct chcr_dev dev; -}; - -struct ktls_key_ctx { - __be32 ctx_hdr; - u8 salt[CHCR_MAX_SALT]; - __be64 iv_to_auth; - unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE + - TLS_CIPHER_AES_GCM_256_TAG_SIZE]; -}; - -/* Crypto key context */ -#define KEY_CONTEXT_CTX_LEN_S 24 -#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) - -#define KEY_CONTEXT_SALT_PRESENT_S 10 -#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) -#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) - -#define KEY_CONTEXT_VALID_S 0 -#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) -#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) - -#define KEY_CONTEXT_CK_SIZE_S 6 -#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) - -#define KEY_CONTEXT_MK_SIZE_S 2 -#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) - -#define KEY_CONTEXT_OPAD_PRESENT_S 11 -#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) -#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) - -#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \ - htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \ - KEY_CONTEXT_CK_SIZE_V(ck_size) | \ - KEY_CONTEXT_VALID_F | \ - KEY_CONTEXT_SALT_PRESENT_F | \ - KEY_CONTEXT_CTX_LEN_V((ctx_len))) - -struct uld_ctx *assign_chcr_device(void); - -static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q, - void *pos, int length) -{ - int left = (void *)q->stat - pos; - u64 *p; - - if (likely(length <= left)) { - memcpy(pos, src, length); - pos += length; - } else { - memcpy(pos, src, left); - memcpy(q->desc, src + left, length - left); - pos = (void *)q->desc + (length - left); - } - /* 0-pad to multiple of 16 */ - p = PTR_ALIGN(pos, 8); - if ((uintptr_t)p & 8) { - *p = 0; - return p + 1; - } - return p; -} - -static inline unsigned int chcr_txq_avail(const struct sge_txq *q) -{ - return q->size - 1 - q->in_use; -} - -static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n) -{ - q->in_use += n; - q->pidx += n; - if (q->pidx >= q->size) - q->pidx -= q->size; -} - -static inline void chcr_eth_txq_stop(struct sge_eth_txq *q) -{ - netif_tx_stop_queue(q->txq); - q->q.stops++; -} - -static inline unsigned int chcr_sgl_len(unsigned int n) -{ - n--; - return (3 * n) / 2 + (n & 1) + 2; -} - -static inline unsigned int chcr_flits_to_desc(unsigned int n) -{ - WARN_ON(n > SGE_MAX_WR_LEN / 8); - return DIV_ROUND_UP(n, 8); -} -#endif /* __CHCR_COMMON_H__ */ diff --git a/drivers/crypto/chelsio/chcr_core.c b/drivers/crypto/chelsio/chcr_core.c index b3570b41a737..40d51d2bd935 100644 --- a/drivers/crypto/chelsio/chcr_core.c +++ b/drivers/crypto/chelsio/chcr_core.c @@ -33,19 +33,8 @@ static int cpl_fw6_pld_handler(struct adapter *adap, unsigned char *input); static void *chcr_uld_add(const struct cxgb4_lld_info *lld); static int chcr_uld_state_change(void *handle, enum cxgb4_state state); -#if defined(CONFIG_CHELSIO_TLS_DEVICE) -static const struct tlsdev_ops chcr_ktls_ops = { - .tls_dev_add = chcr_ktls_dev_add, - .tls_dev_del = chcr_ktls_dev_del, -}; -#endif - static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { [CPL_FW6_PLD] = cpl_fw6_pld_handler, -#ifdef CONFIG_CHELSIO_TLS_DEVICE - [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, - [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, -#endif }; static struct cxgb4_uld_info chcr_uld_info = { @@ -56,10 +45,6 @@ static struct cxgb4_uld_info chcr_uld_info = { .add = chcr_uld_add, .state_change = chcr_uld_state_change, .rx_handler = chcr_uld_rx_handler, -#if defined(CONFIG_CHELSIO_TLS_DEVICE) - .tx_handler = chcr_uld_tx_handler, - .tlsdev_ops = &chcr_ktls_ops, -#endif }; static void detach_work_fn(struct work_struct *work) @@ -235,15 +220,6 @@ int chcr_uld_rx_handler(void *handle, const __be64 *rsp, return 0; } -#if defined(CONFIG_CHELSIO_TLS_DEVICE) -int chcr_uld_tx_handler(struct sk_buff *skb, struct net_device *dev) -{ - if (skb->decrypted) - return chcr_ktls_xmit(skb, dev); - return 0; -} -#endif /* CONFIG_CHELSIO_IPSEC_INLINE || CONFIG_CHELSIO_TLS_DEVICE */ - static void chcr_detach_device(struct uld_ctx *u_ctx) { struct chcr_dev *dev = &u_ctx->dev; diff --git a/drivers/crypto/chelsio/chcr_core.h b/drivers/crypto/chelsio/chcr_core.h index 81f6e61401e5..bb092b6b36b2 100644 --- a/drivers/crypto/chelsio/chcr_core.h +++ b/drivers/crypto/chelsio/chcr_core.h @@ -139,16 +139,4 @@ int chcr_handle_resp(struct crypto_async_request *req, unsigned char *input, int err); int chcr_ipsec_xmit(struct sk_buff *skb, struct net_device *dev); void chcr_add_xfrmops(const struct cxgb4_lld_info *lld); -#ifdef CONFIG_CHELSIO_TLS_DEVICE -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); -extern int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn); -extern void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction); -#endif #endif /* __CHCR_CORE_H__ */ diff --git a/drivers/crypto/chelsio/chcr_ktls.c b/drivers/crypto/chelsio/chcr_ktls.c deleted file mode 100644 index c5cce024886a..000000000000 --- a/drivers/crypto/chelsio/chcr_ktls.c +++ /dev/null @@ -1,2017 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0-only -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifdef CONFIG_CHELSIO_TLS_DEVICE -#include -#include "chcr_ktls.h" -#include "clip_tbl.h" - -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); -/* - * chcr_ktls_save_keys: calculate and save crypto keys. - * @tx_info - driver specific tls info. - * @crypto_info - tls crypto information. - * @direction - TX/RX direction. - * return - SUCCESS/FAILURE. - */ -static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, - struct tls_crypto_info *crypto_info, - enum tls_offload_ctx_dir direction) -{ - int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; - unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; - struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; - struct ktls_key_ctx *kctx = &tx_info->key_ctx; - struct crypto_cipher *cipher; - unsigned char *key, *salt; - - switch (crypto_info->cipher_type) { - case TLS_CIPHER_AES_GCM_128: - info_128_gcm = - (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; - keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; - ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; - tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; - mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; - tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; - tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); - - ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; - key = info_128_gcm->key; - salt = info_128_gcm->salt; - tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; - - /* The SCMD fields used when encrypting a full TLS - * record. Its a one time calculation till the - * connection exists. - */ - tx_info->scmd0_seqno_numivs = - SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | - SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | - SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | - SCMD_NUM_IVS_V(1); - - /* keys will be sent inline. */ - tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; - - /* The SCMD fields used when encrypting a partial TLS - * record (no trailer and possibly a truncated payload). - */ - tx_info->scmd0_short_seqno_numivs = - SCMD_CIPH_AUTH_SEQ_CTRL_F | - SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | - SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | - SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); - - tx_info->scmd0_short_ivgen_hdrlen = - tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; - - break; - - default: - pr_err("GCM: cipher type 0x%x not supported\n", - crypto_info->cipher_type); - ret = -EINVAL; - goto out; - } - - key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + - roundup(keylen, 16) + ghash_size; - /* Calculate the H = CIPH(K, 0 repeated 16 times). - * It will go in key context - */ - cipher = crypto_alloc_cipher("aes", 0, 0); - if (IS_ERR(cipher)) { - ret = -ENOMEM; - goto out; - } - - ret = crypto_cipher_setkey(cipher, key, keylen); - if (ret) - goto out1; - - memset(ghash_h, 0, ghash_size); - crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); - - /* fill the Key context */ - if (direction == TLS_OFFLOAD_CTX_DIR_TX) { - kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, - mac_key_size, - key_ctx_size >> 4); - } else { - ret = -EINVAL; - goto out1; - } - - memcpy(kctx->salt, salt, tx_info->salt_size); - memcpy(kctx->key, key, keylen); - memcpy(kctx->key + keylen, ghash_h, ghash_size); - tx_info->key_ctx_len = key_ctx_size; - -out1: - crypto_free_cipher(cipher); -out: - return ret; -} - -static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, - int new_state) -{ - /* This function can be called from both rx (interrupt context) and tx - * queue contexts. - */ - spin_lock_bh(&tx_info->lock); - switch (tx_info->connection_state) { - case KTLS_CONN_CLOSED: - tx_info->connection_state = new_state; - break; - - case KTLS_CONN_ACT_OPEN_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and also initialize TCB */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_ACT_OPEN_RPL: - /* if we are stuck in this state, means tcb init might not - * received by HW, try sending it again. - */ - if (!chcr_init_tcb_fields(tx_info)) - tx_info->connection_state = KTLS_CONN_SET_TCB_REQ; - break; - - case KTLS_CONN_SET_TCB_REQ: - /* only go forward if state is greater than current state. */ - if (new_state <= tx_info->connection_state) - break; - /* update to the next state and check if l2t_state is valid */ - tx_info->connection_state = new_state; - fallthrough; - case KTLS_CONN_SET_TCB_RPL: - /* Check if l2t state is valid, then move to ready state. */ - if (cxgb4_check_l2t_valid(tx_info->l2te)) { - tx_info->connection_state = KTLS_CONN_TX_READY; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ctx); - } - break; - - case KTLS_CONN_TX_READY: - /* nothing to be done here */ - break; - - default: - pr_err("unknown KTLS connection state\n"); - break; - } - spin_unlock_bh(&tx_info->lock); - - return tx_info->connection_state; -} -/* - * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req *cpl6; - struct cpl_act_open_req *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | - TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip = inet->inet_rcv_saddr; - cpl->peer_ip = inet->inet_daddr; - - /* fill first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} - -#if IS_ENABLED(CONFIG_IPV6) -/* - * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * @atid - connection active tid. - * return - send success/failure. - */ -static int chcr_ktls_act_open_req6(struct sock *sk, - struct chcr_ktls_info *tx_info, - int atid) -{ - struct inet_sock *inet = inet_sk(sk); - struct cpl_t6_act_open_req6 *cpl6; - struct cpl_act_open_req6 *cpl; - struct sk_buff *skb; - unsigned int len; - int qid_atid; - u64 options; - - len = sizeof(*cpl6); - skb = alloc_skb(len, GFP_KERNEL); - if (unlikely(!skb)) - return -ENOMEM; - /* mark it a control pkt */ - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - - cpl6 = __skb_put_zero(skb, len); - cpl = (struct cpl_act_open_req6 *)cpl6; - INIT_TP_WR(cpl6, 0); - qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); - cpl->local_port = inet->inet_sport; - cpl->peer_port = inet->inet_dport; - cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; - cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; - cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; - cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; - - /* first 64 bit option field. */ - options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | - SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); - cpl->opt0 = cpu_to_be64(options); - /* next 64 bit option field. */ - options = - TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); - cpl->opt2 = htonl(options); - - return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); -} -#endif /* #if IS_ENABLED(CONFIG_IPV6) */ - -/* - * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. - * @sk - tcp socket. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_setup_connection(struct sock *sk, - struct chcr_ktls_info *tx_info) -{ - struct tid_info *t = &tx_info->adap->tids; - int atid, ret = 0; - - atid = cxgb4_alloc_atid(t, tx_info); - if (atid == -1) - return -EINVAL; - - tx_info->atid = atid; - tx_info->ip_family = sk->sk_family; - - if (sk->sk_family == AF_INET) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { - tx_info->ip_family = AF_INET; - ret = chcr_ktls_act_open_req(sk, tx_info, atid); - } else { - tx_info->ip_family = AF_INET6; - ret = cxgb4_clip_get(tx_info->netdev, - (const u32 *) - &sk->sk_v6_rcv_saddr.s6_addr, - 1); - if (ret) - goto out; - ret = chcr_ktls_act_open_req6(sk, tx_info, atid); - } -#endif - } - - /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret - * success, if any other return type clear atid and return that failure. - */ - if (ret) { - if (ret == NET_XMIT_CN) - ret = 0; - else - cxgb4_free_atid(t, atid); - goto out; - } - - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ); -out: - return ret; -} - -/* - * chcr_set_tcb_field: update tcb fields. - * @tx_info - driver specific tls info. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @no_reply - set 1 if not looking for TP response. - */ -static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, - u64 mask, u64 val, int no_reply) -{ - struct cpl_set_tcb_field *req; - struct sk_buff *skb; - - skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); - if (!skb) - return -ENOMEM; - - req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); - INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); - req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(no_reply)); - req->word_cookie = htons(TCB_WORD_V(word)); - req->mask = cpu_to_be64(mask); - req->val = cpu_to_be64(val); - - set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); - return cxgb4_ofld_send(tx_info->netdev, skb); -} - -/* - * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP. - */ -static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info) -{ - return chcr_set_tcb_field(tx_info, TCB_T_STATE_W, - TCB_T_STATE_V(TCB_T_STATE_M), - CHCR_TCB_STATE_CLOSED, 1); -} - -/* - * chcr_ktls_dev_del: call back for tls_dev_del. - * Remove the tid and l2t entry and close the connection. - * it per connection basis. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - */ -void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction) -{ - struct chcr_ktls_ofld_ctx_tx *tx_ctx = - chcr_get_ktls_tx_context(tls_ctx); - struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; - struct sock *sk; - - if (!tx_info) - return; - sk = tx_info->sk; - - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - /* clear l2t entry */ - if (tx_info->l2te) - cxgb4_l2t_release(tx_info->l2te); - -#if IS_ENABLED(CONFIG_IPV6) - /* clear clip entry */ - if (tx_info->ip_family == AF_INET6) - cxgb4_clip_release(netdev, - (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, - 1); -#endif - - /* clear tid */ - if (tx_info->tid != -1) { - /* clear tcb state and then release tid */ - chcr_ktls_mark_tcb_close(tx_info); - cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, - tx_info->tid, tx_info->ip_family); - } - - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_connection_close); - kvfree(tx_info); - tx_ctx->chcr_info = NULL; - /* release module refcount */ - module_put(THIS_MODULE); -} - -/* - * chcr_ktls_dev_add: call back for tls_dev_add. - * Create a tcb entry for TP. Also add l2t entry for the connection. And - * generate keys & save those keys locally. - * @netdev - net device. - * @tls_cts - tls context. - * @direction - TX/RX crypto direction - * return: SUCCESS/FAILURE. - */ -int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn) -{ - struct tls_context *tls_ctx = tls_get_ctx(sk); - struct chcr_ktls_ofld_ctx_tx *tx_ctx; - struct chcr_ktls_info *tx_info; - struct dst_entry *dst; - struct adapter *adap; - struct port_info *pi; - struct neighbour *n; - u8 daaddr[16]; - int ret = -1; - - tx_ctx = chcr_get_ktls_tx_context(tls_ctx); - - pi = netdev_priv(netdev); - adap = pi->adapter; - if (direction == TLS_OFFLOAD_CTX_DIR_RX) { - pr_err("not expecting for RX direction\n"); - ret = -EINVAL; - goto out; - } - if (tx_ctx->chcr_info) { - ret = -EINVAL; - goto out; - } - - tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); - if (!tx_info) { - ret = -ENOMEM; - goto out; - } - - spin_lock_init(&tx_info->lock); - - /* clear connection state */ - spin_lock(&tx_info->lock); - tx_info->connection_state = KTLS_CONN_CLOSED; - spin_unlock(&tx_info->lock); - - tx_info->sk = sk; - /* initialize tid and atid to -1, 0 is a also a valid id. */ - tx_info->tid = -1; - tx_info->atid = -1; - - tx_info->adap = adap; - tx_info->netdev = netdev; - tx_info->first_qset = pi->first_qset; - tx_info->tx_chan = pi->tx_chan; - tx_info->smt_idx = pi->smt_idx; - tx_info->port_id = pi->port_id; - - tx_info->rx_qid = chcr_get_first_rx_qid(adap); - if (unlikely(tx_info->rx_qid < 0)) - goto out2; - - tx_info->prev_seq = start_offload_tcp_sn; - tx_info->tcp_start_seq_number = start_offload_tcp_sn; - - /* save crypto keys */ - ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); - if (ret < 0) - goto out2; - - /* get peer ip */ - if (sk->sk_family == AF_INET) { - memcpy(daaddr, &sk->sk_daddr, 4); -#if IS_ENABLED(CONFIG_IPV6) - } else { - if (!sk->sk_ipv6only && - ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) - memcpy(daaddr, &sk->sk_daddr, 4); - else - memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); -#endif - } - - /* get the l2t index */ - dst = sk_dst_get(sk); - if (!dst) { - pr_err("DST entry not found\n"); - goto out2; - } - n = dst_neigh_lookup(dst, daaddr); - if (!n || !n->dev) { - pr_err("neighbour not found\n"); - dst_release(dst); - goto out2; - } - tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); - - neigh_release(n); - dst_release(dst); - - if (!tx_info->l2te) { - pr_err("l2t entry not found\n"); - goto out2; - } - - tx_ctx->chcr_info = tx_info; - - /* create a filter and call cxgb4_l2t_send to send the packet out, which - * will take care of updating l2t entry in hw if not already done. - */ - ret = chcr_setup_connection(sk, tx_info); - if (ret) - goto out2; - - /* Driver shouldn't be removed until any single connection exists */ - if (!try_module_get(THIS_MODULE)) { - ret = -EINVAL; - goto out2; - } - - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_open); - return 0; -out2: - kvfree(tx_info); -out: - atomic64_inc(&adap->chcr_stats.ktls_tx_connection_fail); - return ret; -} - -/* - * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number - * handling. - * @tx_info - driver specific tls info. - * return: NET_TX_OK/NET_XMIT_DROP - */ -static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) -{ - int ret = 0; - - /* set tcb in offload and bypass */ - ret = - chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, - TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), - TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); - if (ret) - return ret; - /* reset snd_una and snd_next fields in tcb */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, - TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | - TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), - 0, 1); - if (ret) - return ret; - - /* reset send max */ - ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, - TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), - 0, 1); - if (ret) - return ret; - - /* update l2t index and request for tp reply to confirm tcb is - * initialised to handle tx traffic. - */ - ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, - TCB_L2T_IX_V(TCB_L2T_IX_M), - TCB_L2T_IX_V(tx_info->l2te->idx), 0); - return ret; -} - -/* - * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. - */ -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_act_open_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - unsigned int atid, tid, status; - struct tid_info *t; - - tid = GET_TID(p); - status = AOPEN_STATUS_G(ntohl(p->atid_status)); - atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); - - t = &adap->tids; - tx_info = lookup_atid(t, atid); - - if (!tx_info || tx_info->atid != atid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - - if (!status) { - tx_info->tid = tid; - cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); - - cxgb4_free_atid(t, atid); - tx_info->atid = -1; - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, - KTLS_CONN_ACT_OPEN_RPL); - } - return 0; -} - -/* - * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. - */ -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) -{ - const struct cpl_set_tcb_rpl *p = (void *)input; - struct chcr_ktls_info *tx_info = NULL; - struct tid_info *t; - u32 tid; - - tid = GET_TID(p); - - t = &adap->tids; - tx_info = lookup_tid(t, tid); - if (!tx_info || tx_info->tid != tid) { - pr_err("tx_info or atid is not correct\n"); - return -1; - } - /* update the connection state */ - chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL); - return 0; -} - -static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - u32 tid, void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - struct cpl_set_tcb_field_core *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *txpkt; - - /* ULP_TXPKT */ - txpkt = pos; - txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); - txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); - - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(txpkt + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); - idata->len = htonl(sizeof(*cpl)); - pos = idata + 1; - - cpl = pos; - /* CPL_SET_TCB_FIELD */ - OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); - cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | - NO_REPLY_V(!reply)); - cpl->word_cookie = htons(TCB_WORD_V(word)); - cpl->mask = cpu_to_be64(mask); - cpl->val = cpu_to_be64(val); - - /* ULPTX_NOOP */ - idata = (struct ulptx_idata *)(cpl + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); - idata->len = htonl(0); - pos = idata + 1; - - return pos; -} - - -/* - * chcr_write_cpl_set_tcb_ulp: update tcb values. - * TCB is responsible to create tcp headers, so all the related values - * should be correctly updated. - * @tx_info - driver specific tls info. - * @q - tx queue on which packet is going out. - * @tid - TCB identifier. - * @pos - current index where should we start writing. - * @word - TCB word. - * @mask - TCB word related mask. - * @val - TCB word related value. - * @reply - set 1 if looking for TP response. - * return - next position to write. - */ -static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tid, - void *pos, u16 word, u64 mask, - u64 val, u32 reply) -{ - int left = (void *)q->q.stat - pos; - - if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { - if (!left) { - pos = q->q.desc; - } else { - u8 buf[48] = {0}; - - __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, - mask, val, reply); - - return chcr_copy_to_txd(buf, &q->q, pos, - CHCR_SET_TCB_FIELD_LEN); - } - } - - pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, - mask, val, reply); - - /* check again if we are at the end of the queue */ - if (left == CHCR_SET_TCB_FIELD_LEN) - pos = q->q.desc; - - return pos; -} - -/* - * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header - * with updated values like tcp seq, ack, window etc. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_ack - * @tcp_win - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u64 tcp_seq, - u64 tcp_ack, u64 tcp_win) -{ - bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); - u32 len, cpl = 0, ndesc, wr_len; - struct fw_ulptx_wr *wr; - int credits; - void *pos; - - wr_len = sizeof(*wr); - /* there can be max 4 cpls, check if we have enough credits */ - len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); - ndesc = DIV_ROUND_UP(len, 64); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - /* make space for WR, we'll fill it later when we know all the cpls - * being sent out and have complete length. - */ - wr = pos; - pos += wr_len; - /* update tx_max if its a re-transmit or the first wr */ - if (first_wr || tcp_seq != tx_info->prev_seq) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_TX_MAX_W, - TCB_TX_MAX_V(TCB_TX_MAX_M), - TCB_TX_MAX_V(tcp_seq), 0); - cpl++; - } - /* reset snd una if it's a re-transmit pkt */ - if (tcp_seq != tx_info->prev_seq) { - /* reset snd_una */ - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_SND_UNA_RAW_W, - TCB_SND_UNA_RAW_V - (TCB_SND_UNA_RAW_M), - TCB_SND_UNA_RAW_V(0), 0); - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_ooo); - cpl++; - } - /* update ack */ - if (first_wr || tx_info->prev_ack != tcp_ack) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_NXT_W, - TCB_RCV_NXT_V(TCB_RCV_NXT_M), - TCB_RCV_NXT_V(tcp_ack), 0); - tx_info->prev_ack = tcp_ack; - cpl++; - } - /* update receive window */ - if (first_wr || tx_info->prev_win != tcp_win) { - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_RCV_WND_W, - TCB_RCV_WND_V(TCB_RCV_WND_M), - TCB_RCV_WND_V(tcp_win), 0); - tx_info->prev_win = tcp_win; - cpl++; - } - - if (cpl) { - /* get the actual length */ - len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16); - /* ULPTX wr */ - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->cookie = 0; - /* fill len in wr field */ - wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); - - ndesc = DIV_ROUND_UP(len, 64); - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - } - return 0; -} - -/* - * chcr_ktls_skb_copy - * @nskb - new skb where the frags to be added. - * @skb - old skb from which frags will be copied. - */ -static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb) -{ - int i; - - for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { - skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i]; - __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); - } - - skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags; - nskb->len += skb->data_len; - nskb->data_len = skb->data_len; - nskb->truesize += skb->data_len; -} - -/* - * chcr_ktls_get_tx_flits - * returns number of flits to be sent out, it includes key context length, WR - * size and skb fragments. - */ -static unsigned int -chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len) -{ - return chcr_sgl_len(skb_shinfo(skb)->nr_frags) + - DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8); -} - -/* - * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale - * other than timestamp. - * @skb - skb contains partial record.. - * return: 1 / 0 - */ -static int -chcr_ktls_check_tcp_options(struct tcphdr *tcp) -{ - int cnt, opt, optlen; - u_char *cp; - - cp = (u_char *)(tcp + 1); - cnt = (tcp->doff << 2) - sizeof(struct tcphdr); - for (; cnt > 0; cnt -= optlen, cp += optlen) { - opt = cp[0]; - if (opt == TCPOPT_EOL) - break; - if (opt == TCPOPT_NOP) { - optlen = 1; - } else { - if (cnt < 2) - break; - optlen = cp[1]; - if (optlen < 2 || optlen > cnt) - break; - } - switch (opt) { - case TCPOPT_NOP: - break; - default: - return 1; - } - } - return 0; -} - -/* - * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to - * send out separately. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @q - TX queue. - * @tx_chan - channel number. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int -chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, - struct sge_eth_txq *q, uint32_t tx_chan) -{ - struct fw_eth_tx_pkt_wr *wr; - struct cpl_tx_pkt_core *cpl; - u32 ctrl, iplen, maclen; -#if IS_ENABLED(CONFIG_IPV6) - struct ipv6hdr *ip6; -#endif - unsigned int ndesc; - struct tcphdr *tcp; - int len16, pktlen; - struct iphdr *ip; - int credits; - u8 buf[150]; - void *pos; - - iplen = skb_network_header_len(skb); - maclen = skb_mac_header_len(skb); - - /* packet length = eth hdr len + ip hdr len + tcp hdr len - * (including options). - */ - pktlen = skb->len - skb->data_len; - - ctrl = sizeof(*cpl) + pktlen; - len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); - /* check how many descriptors needed */ - ndesc = DIV_ROUND_UP(len16, 4); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - wr = pos; - - /* Firmware work request header */ - wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | - FW_WR_IMMDLEN_V(ctrl)); - - wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16)); - wr->r3 = 0; - - cpl = (void *)(wr + 1); - - /* CPL header */ - cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) | - TXPKT_PF_V(tx_info->adap->pf)); - cpl->pack = 0; - cpl->len = htons(pktlen); - /* checksum offload */ - cpl->ctrl1 = 0; - - pos = cpl + 1; - - memcpy(buf, skb->data, pktlen); - if (tx_info->ip_family == AF_INET) { - /* we need to correct ip header len */ - ip = (struct iphdr *)(buf + maclen); - ip->tot_len = htons(pktlen - maclen); -#if IS_ENABLED(CONFIG_IPV6) - } else { - ip6 = (struct ipv6hdr *)(buf + maclen); - ip6->payload_len = htons(pktlen - maclen - iplen); -#endif - } - /* now take care of the tcp header, if fin is not set then clear push - * bit as well, and if fin is set, it will be sent at the last so we - * need to update the tcp sequence number as per the last packet. - */ - tcp = (struct tcphdr *)(buf + maclen + iplen); - - if (!tcp->fin) - tcp->psh = 0; - else - tcp->seq = htonl(tx_info->prev_seq); - - chcr_copy_to_txd(buf, &q->q, pos, pktlen); - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - return 0; -} - -/* chcr_ktls_skb_shift - Shifts request length paged data from skb to another. - * @tgt- buffer into which tail data gets added - * @skb- buffer from which the paged data comes from - * @shiftlen- shift up to this many bytes - */ -static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb, - int shiftlen) -{ - skb_frag_t *fragfrom, *fragto; - int from, to, todo; - - WARN_ON(shiftlen > skb->data_len); - - todo = shiftlen; - from = 0; - to = 0; - fragfrom = &skb_shinfo(skb)->frags[from]; - - while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { - fragfrom = &skb_shinfo(skb)->frags[from]; - fragto = &skb_shinfo(tgt)->frags[to]; - - if (todo >= skb_frag_size(fragfrom)) { - *fragto = *fragfrom; - todo -= skb_frag_size(fragfrom); - from++; - to++; - - } else { - __skb_frag_ref(fragfrom); - skb_frag_page_copy(fragto, fragfrom); - skb_frag_off_copy(fragto, fragfrom); - skb_frag_size_set(fragto, todo); - - skb_frag_off_add(fragfrom, todo); - skb_frag_size_sub(fragfrom, todo); - todo = 0; - - to++; - break; - } - } - - /* Ready to "commit" this state change to tgt */ - skb_shinfo(tgt)->nr_frags = to; - - /* Reposition in the original skb */ - to = 0; - while (from < skb_shinfo(skb)->nr_frags) - skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; - - skb_shinfo(skb)->nr_frags = to; - - WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); - - skb->len -= shiftlen; - skb->data_len -= shiftlen; - skb->truesize -= shiftlen; - tgt->len += shiftlen; - tgt->data_len += shiftlen; - tgt->truesize += shiftlen; - - return shiftlen; -} - -/* - * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb - * received has partial end part of the record, send out the complete record, so - * that crypto block will be able to generate TAG/HASH. - * @skb - segment which has complete or partial end part. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_push - tcp push bit. - * @mss - segment size. - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb, - struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, u32 tcp_seq, - bool tcp_push, u32 mss) -{ - u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start; - struct adapter *adap = tx_info->adap; - int credits, left, last_desc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct cpl_tx_sec_pdu *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - void *pos; - u64 *end; - - /* get the number of flits required */ - flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len); - /* number of descriptors */ - ndesc = chcr_flits_to_desc(flits); - /* check if enough credits available */ - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - /* Credits are below the threshold vaues, stop the queue after - * injecting the Work Request for this packet. - */ - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = pos; - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | - ULP_TXPKT_RO_F); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - /* idata length will include cpl_tx_sec_pdu + key context size + - * cpl_tx_data header. - */ - idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + - sizeof(*tx_data)); - /* SEC CPL */ - cpl = (struct cpl_tx_sec_pdu *)(idata + 1); - cpl->op_ivinsrtofst = - htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | - CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | - CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | - CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1)); - cpl->pldlen = htonl(skb->data_len); - - /* encryption should start after tls header size + iv size */ - cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1; - - cpl->aadstart_cipherstop_hi = - htonl(CPL_TX_SEC_PDU_AADSTART_V(1) | - CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) | - CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); - - /* authentication will also start after tls header + iv size */ - cpl->cipherstop_lo_authinsert = - htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) | - CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) | - CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE)); - - /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ - cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs); - cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen); - cpl->scmd1 = cpu_to_be64(tx_info->record_no); - - pos = cpl + 1; - /* check if space left to fill the keys */ - left = (void *)q->q.stat - pos; - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, - tx_info->key_ctx_len); - left = (void *)q->q.stat - pos; - - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* CPL_TX_DATA */ - tx_data = (void *)pos; - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len)); - - tx_data->rsvd = htonl(tcp_seq); - - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - /* check left again, it might go beyond queue limit */ - pos = tx_data + 1; - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - /* send the complete packet except the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(adap, &q->q, ndesc); - atomic64_inc(&adap->chcr_stats.ktls_tx_send_records); - - return 0; -} - -/* - * chcr_ktls_xmit_wr_short: This is to send out partial records. If its - * a middle part of a record, fetch the prior data to make it 16 byte aligned - * and then only send it out. - * - * @skb - skb contains partial record.. - * @tx_info - driver specific tls info. - * @q - TX queue. - * @tcp_seq - * @tcp_push - tcp push bit. - * @mss - segment size. - * @tls_rec_offset - offset from start of the tls record. - * @perior_data - data before the current segment, required to make this record - * 16 byte aligned. - * @prior_data_len - prior_data length (less than 16) - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_xmit_wr_short(struct sk_buff *skb, - struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q, - u32 tcp_seq, bool tcp_push, u32 mss, - u32 tls_rec_offset, u8 *prior_data, - u32 prior_data_len) -{ - struct adapter *adap = tx_info->adap; - u32 len16, wr_mid = 0, cipher_start; - unsigned int flits = 0, ndesc; - int credits, left, last_desc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct cpl_tx_sec_pdu *cpl; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - __be64 iv_record; - void *pos; - u64 *end; - - /* get the number of flits required, it's a partial record so 2 flits - * (AES_BLOCK_SIZE) will be added. - */ - flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2; - /* get the correct 8 byte IV of this record */ - iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no); - /* If it's a middle record and not 16 byte aligned to run AES CTR, need - * to make it 16 byte aligned. So atleadt 2 extra flits of immediate - * data will be added. - */ - if (prior_data_len) - flits += 2; - /* number of descriptors */ - ndesc = chcr_flits_to_desc(flits); - /* check if enough credits available */ - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = pos; - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | - ULP_TXPKT_RO_F); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - /* idata length will include cpl_tx_sec_pdu + key context size + - * cpl_tx_data header. - */ - idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + - sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len); - /* SEC CPL */ - cpl = (struct cpl_tx_sec_pdu *)(idata + 1); - /* cipher start will have tls header + iv size extra if its a header - * part of tls record. else only 16 byte IV will be added. - */ - cipher_start = - AES_BLOCK_LEN + 1 + - (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0); - - cpl->op_ivinsrtofst = - htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | - CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | - CPL_TX_SEC_PDU_IVINSRTOFST_V(1)); - cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len); - cpl->aadstart_cipherstop_hi = - htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); - cpl->cipherstop_lo_authinsert = 0; - /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ - cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs); - cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen); - cpl->scmd1 = 0; - - pos = cpl + 1; - /* check if space left to fill the keys */ - left = (void *)q->q.stat - pos; - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - - pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, - tx_info->key_ctx_len); - left = (void *)q->q.stat - pos; - - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* CPL_TX_DATA */ - tx_data = (void *)pos; - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | - TX_LENGTH_V(skb->data_len + prior_data_len)); - tx_data->rsvd = htonl(tcp_seq); - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - /* check left again, it might go beyond queue limit */ - pos = tx_data + 1; - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8 - * bytes of actual IV and 4 bytes of 16 byte-sequence. - */ - memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size); - memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size); - *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) = - htonl(2 + (tls_rec_offset ? ((tls_rec_offset - - (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0)); - - pos += 16; - /* Prior_data_len will always be less than 16 bytes, fill the - * prio_data_len after AES_CTRL_BLOCK and clear the remaining length - * to 0. - */ - if (prior_data_len) - pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); - /* send the complete packet except the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(adap, &q->q, ndesc); - - return 0; -} - -/* - * chcr_ktls_tx_plaintxt: This handler will take care of the records which has - * only plain text (only tls header and iv) - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @tcp_seq - * @mss - segment size. - * @tcp_push - tcp push bit. - * @q - TX queue. - * @port_id : port number - * @perior_data - data before the current segment, required to make this record - * 16 byte aligned. - * @prior_data_len - prior_data length (less than 16) - * return: NETDEV_TX_BUSY/NET_TX_OK. - */ -static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, u32 tcp_seq, u32 mss, - bool tcp_push, struct sge_eth_txq *q, - u32 port_id, u8 *prior_data, - u32 prior_data_len) -{ - int credits, left, len16, last_desc; - unsigned int flits = 0, ndesc; - struct tx_sw_desc *sgl_sdesc; - struct cpl_tx_data *tx_data; - struct ulptx_idata *idata; - struct ulp_txpkt *ulptx; - struct fw_ulptx_wr *wr; - u32 wr_mid = 0; - void *pos; - u64 *end; - - flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8); - flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags); - if (prior_data_len) - flits += 2; - /* WR will need len16 */ - len16 = DIV_ROUND_UP(flits, 2); - /* check how many descriptors needed */ - ndesc = DIV_ROUND_UP(flits, 8); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - if (unlikely(credits < ETHTXQ_STOP_THRES)) { - chcr_eth_txq_stop(q); - wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; - } - - last_desc = q->q.pidx + ndesc - 1; - if (last_desc >= q->q.size) - last_desc -= q->q.size; - sgl_sdesc = &q->q.sdesc[last_desc]; - - if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, - sgl_sdesc->addr) < 0)) { - memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); - q->mapping_err++; - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - end = (u64 *)pos + flits; - /* FW_ULPTX_WR */ - wr = pos; - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); - wr->cookie = 0; - pos += sizeof(*wr); - /* ULP_TXPKT */ - ulptx = (struct ulp_txpkt *)(wr + 1); - ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | - ULP_TXPKT_DATAMODIFY_V(0) | - ULP_TXPKT_CHANNELID_V(tx_info->port_id) | - ULP_TXPKT_DEST_V(0) | - ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1)); - ulptx->len = htonl(len16 - 1); - /* ULPTX_IDATA sub-command */ - idata = (struct ulptx_idata *)(ulptx + 1); - idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); - idata->len = htonl(sizeof(*tx_data) + prior_data_len); - /* CPL_TX_DATA */ - tx_data = (struct cpl_tx_data *)(idata + 1); - OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); - tx_data->len = htonl(TX_DATA_MSS_V(mss) | - TX_LENGTH_V(skb->data_len + prior_data_len)); - /* set tcp seq number */ - tx_data->rsvd = htonl(tcp_seq); - tx_data->flags = htonl(TX_BYPASS_F); - if (tcp_push) - tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); - - pos = tx_data + 1; - /* apart from prior_data_len, we should set remaining part of 16 bytes - * to be zero. - */ - if (prior_data_len) - pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); - - /* check left again, it might go beyond queue limit */ - left = (void *)q->q.stat - pos; - - /* check the position again */ - if (!left) { - left = (void *)end - (void *)q->q.stat; - pos = q->q.desc; - end = pos + left; - } - /* send the complete packet including the header */ - cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, - sgl_sdesc->addr); - sgl_sdesc->skb = skb; - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - return 0; -} - -/* - * chcr_ktls_copy_record_in_skb - * @nskb - new skb where the frags to be added. - * @record - specific record which has complete 16k record in frags. - */ -static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb, - struct tls_record_info *record) -{ - int i = 0; - - for (i = 0; i < record->num_frags; i++) { - skb_shinfo(nskb)->frags[i] = record->frags[i]; - /* increase the frag ref count */ - __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); - } - - skb_shinfo(nskb)->nr_frags = record->num_frags; - nskb->data_len = record->len; - nskb->len += record->len; - nskb->truesize += record->len; -} - -/* - * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid - * sending the same segment again. It will discard the segment which is before - * the current tx max. - * @tx_info - driver specific tls info. - * @q - TX queue. - * return: NET_TX_OK/NET_XMIT_DROP. - */ -static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info, - struct sge_eth_txq *q) -{ - struct fw_ulptx_wr *wr; - unsigned int ndesc; - int credits; - void *pos; - u32 len; - - len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16); - ndesc = DIV_ROUND_UP(len, 64); - - credits = chcr_txq_avail(&q->q) - ndesc; - if (unlikely(credits < 0)) { - chcr_eth_txq_stop(q); - return NETDEV_TX_BUSY; - } - - pos = &q->q.desc[q->q.pidx]; - - wr = pos; - /* ULPTX wr */ - wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); - wr->cookie = 0; - /* fill len in wr field */ - wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); - - pos += sizeof(*wr); - - pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, - TCB_SND_UNA_RAW_W, - TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), - TCB_SND_UNA_RAW_V(0), 0); - - chcr_txq_advance(&q->q, ndesc); - cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); - - return 0; -} - -/* - * chcr_end_part_handler: This handler will handle the record which - * is complete or if record's end part is received. T6 adapter has a issue that - * it can't send out TAG with partial record so if its an end part then we have - * to send TAG as well and for which we need to fetch the complete record and - * send it to crypto module. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record. - * @record - complete record of 16K size. - * @tcp_seq - * @mss - segment size in which TP needs to chop a packet. - * @tcp_push_no_fin - tcp push if fin is not set. - * @q - TX queue. - * @tls_end_offset - offset from end of the record. - * @last wr : check if this is the last part of the skb going out. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int chcr_end_part_handler(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, - struct tls_record_info *record, - u32 tcp_seq, int mss, bool tcp_push_no_fin, - struct sge_eth_txq *q, - u32 tls_end_offset, bool last_wr) -{ - struct sk_buff *nskb = NULL; - /* check if it is a complete record */ - if (tls_end_offset == record->len) { - nskb = skb; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_complete_pkts); - } else { - dev_kfree_skb_any(skb); - - nskb = alloc_skb(0, GFP_KERNEL); - if (!nskb) - return NETDEV_TX_BUSY; - /* copy complete record in skb */ - chcr_ktls_copy_record_in_skb(nskb, record); - /* packet is being sent from the beginning, update the tcp_seq - * accordingly. - */ - tcp_seq = tls_record_start_seq(record); - /* reset snd una, so the middle record won't send the already - * sent part. - */ - if (chcr_ktls_update_snd_una(tx_info, q)) - goto out; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_end_pkts); - } - - if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq, - (last_wr && tcp_push_no_fin), - mss)) { - goto out; - } - return 0; -out: - dev_kfree_skb_any(nskb); - return NETDEV_TX_BUSY; -} - -/* - * chcr_short_record_handler: This handler will take care of the records which - * doesn't have end part (1st part or the middle part(/s) of a record). In such - * cases, AES CTR will be used in place of AES GCM to send out partial packet. - * This partial record might be the first part of the record, or the middle - * part. In case of middle record we should fetch the prior data to make it 16 - * byte aligned. If it has a partial tls header or iv then get to the start of - * tls header. And if it has partial TAG, then remove the complete TAG and send - * only the payload. - * There is one more possibility that it gets a partial header, send that - * portion as a plaintext. - * @tx_info - driver specific tls info. - * @skb - skb contains partial record.. - * @record - complete record of 16K size. - * @tcp_seq - * @mss - segment size in which TP needs to chop a packet. - * @tcp_push_no_fin - tcp push if fin is not set. - * @q - TX queue. - * @tls_end_offset - offset from end of the record. - * return: NETDEV_TX_OK/NETDEV_TX_BUSY. - */ -static int chcr_short_record_handler(struct chcr_ktls_info *tx_info, - struct sk_buff *skb, - struct tls_record_info *record, - u32 tcp_seq, int mss, bool tcp_push_no_fin, - struct sge_eth_txq *q, u32 tls_end_offset) -{ - u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record); - u8 prior_data[16] = {0}; - u32 prior_data_len = 0; - u32 data_len; - - /* check if the skb is ending in middle of tag/HASH, its a big - * trouble, send the packet before the HASH. - */ - int remaining_record = tls_end_offset - skb->data_len; - - if (remaining_record > 0 && - remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) { - int trimmed_len = skb->data_len - - (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record); - struct sk_buff *tmp_skb = NULL; - /* don't process the pkt if it is only a partial tag */ - if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE) - goto out; - - WARN_ON(trimmed_len > skb->data_len); - - /* shift to those many bytes */ - tmp_skb = alloc_skb(0, GFP_KERNEL); - if (unlikely(!tmp_skb)) - goto out; - - chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len); - /* free the last trimmed portion */ - dev_kfree_skb_any(skb); - skb = tmp_skb; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_trimmed_pkts); - } - data_len = skb->data_len; - /* check if the middle record's start point is 16 byte aligned. CTR - * needs 16 byte aligned start point to start encryption. - */ - if (tls_rec_offset) { - /* there is an offset from start, means its a middle record */ - int remaining = 0; - - if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) { - prior_data_len = tls_rec_offset; - tls_rec_offset = 0; - remaining = 0; - } else { - prior_data_len = - (tls_rec_offset - - (TLS_HEADER_SIZE + tx_info->iv_size)) - % AES_BLOCK_LEN; - remaining = tls_rec_offset - prior_data_len; - } - - /* if prior_data_len is not zero, means we need to fetch prior - * data to make this record 16 byte aligned, or we need to reach - * to start offset. - */ - if (prior_data_len) { - int i = 0; - u8 *data = NULL; - skb_frag_t *f; - u8 *vaddr; - int frag_size = 0, frag_delta = 0; - - while (remaining > 0) { - frag_size = skb_frag_size(&record->frags[i]); - if (remaining < frag_size) - break; - - remaining -= frag_size; - i++; - } - f = &record->frags[i]; - vaddr = kmap_atomic(skb_frag_page(f)); - - data = vaddr + skb_frag_off(f) + remaining; - frag_delta = skb_frag_size(f) - remaining; - - if (frag_delta >= prior_data_len) { - memcpy(prior_data, data, prior_data_len); - kunmap_atomic(vaddr); - } else { - memcpy(prior_data, data, frag_delta); - kunmap_atomic(vaddr); - /* get the next page */ - f = &record->frags[i + 1]; - vaddr = kmap_atomic(skb_frag_page(f)); - data = vaddr + skb_frag_off(f); - memcpy(prior_data + frag_delta, - data, (prior_data_len - frag_delta)); - kunmap_atomic(vaddr); - } - /* reset tcp_seq as per the prior_data_required len */ - tcp_seq -= prior_data_len; - /* include prio_data_len for further calculation. - */ - data_len += prior_data_len; - } - /* reset snd una, so the middle record won't send the already - * sent part. - */ - if (chcr_ktls_update_snd_una(tx_info, q)) - goto out; - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_middle_pkts); - } else { - /* Else means, its a partial first part of the record. Check if - * its only the header, don't need to send for encryption then. - */ - if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) { - if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, - tcp_push_no_fin, q, - tx_info->port_id, - prior_data, - prior_data_len)) { - goto out; - } - return 0; - } - atomic64_inc(&tx_info->adap->chcr_stats.ktls_tx_start_pkts); - } - - if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin, - mss, tls_rec_offset, prior_data, - prior_data_len)) { - goto out; - } - - return 0; -out: - dev_kfree_skb_any(skb); - return NETDEV_TX_BUSY; -} - -/* nic tls TX handler */ -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) -{ - struct chcr_ktls_ofld_ctx_tx *tx_ctx; - struct tcphdr *th = tcp_hdr(skb); - int data_len, qidx, ret = 0, mss; - struct tls_record_info *record; - struct chcr_stats_debug *stats; - struct chcr_ktls_info *tx_info; - u32 tls_end_offset, tcp_seq; - struct tls_context *tls_ctx; - struct sk_buff *local_skb; - int new_connection_state; - struct sge_eth_txq *q; - struct adapter *adap; - unsigned long flags; - - tcp_seq = ntohl(th->seq); - - mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len; - - /* check if we haven't set it for ktls offload */ - if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) - goto out; - - tls_ctx = tls_get_ctx(skb->sk); - if (unlikely(tls_ctx->netdev != dev)) - goto out; - - tx_ctx = chcr_get_ktls_tx_context(tls_ctx); - tx_info = tx_ctx->chcr_info; - - if (unlikely(!tx_info)) - goto out; - - /* check the connection state, we don't need to pass new connection - * state, state machine will check and update the new state if it is - * stuck due to responses not received from HW. - * Start the tx handling only if state is KTLS_CONN_TX_READY. - */ - new_connection_state = chcr_ktls_update_connection_state(tx_info, 0); - if (new_connection_state != KTLS_CONN_TX_READY) - goto out; - - /* don't touch the original skb, make a new skb to extract each records - * and send them separately. - */ - local_skb = alloc_skb(0, GFP_KERNEL); - - if (unlikely(!local_skb)) - return NETDEV_TX_BUSY; - - adap = tx_info->adap; - stats = &adap->chcr_stats; - - qidx = skb->queue_mapping; - q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; - cxgb4_reclaim_completed_tx(adap, &q->q, true); - /* if tcp options are set but finish is not send the options first */ - if (!th->fin && chcr_ktls_check_tcp_options(th)) { - ret = chcr_ktls_write_tcp_options(tx_info, skb, q, - tx_info->tx_chan); - if (ret) - return NETDEV_TX_BUSY; - } - /* update tcb */ - ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq), - ntohl(th->ack_seq), - ntohs(th->window)); - if (ret) { - dev_kfree_skb_any(local_skb); - return NETDEV_TX_BUSY; - } - - /* copy skb contents into local skb */ - chcr_ktls_skb_copy(skb, local_skb); - - /* go through the skb and send only one record at a time. */ - data_len = skb->data_len; - /* TCP segments can be in received either complete or partial. - * chcr_end_part_handler will handle cases if complete record or end - * part of the record is received. Incase of partial end part of record, - * we will send the complete record again. - */ - - do { - int i; - - cxgb4_reclaim_completed_tx(adap, &q->q, true); - /* lock taken */ - spin_lock_irqsave(&tx_ctx->base.lock, flags); - /* fetch the tls record */ - record = tls_get_record(&tx_ctx->base, tcp_seq, - &tx_info->record_no); - /* By the time packet reached to us, ACK is received, and record - * won't be found in that case, handle it gracefully. - */ - if (unlikely(!record)) { - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_drop_no_sync_data); - goto out; - } - - if (unlikely(tls_record_is_start_marker(record))) { - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - atomic64_inc(&stats->ktls_tx_skip_no_sync_data); - goto out; - } - - /* increase page reference count of the record, so that there - * won't be any chance of page free in middle if in case stack - * receives ACK and try to delete the record. - */ - for (i = 0; i < record->num_frags; i++) - __skb_frag_ref(&record->frags[i]); - /* lock cleared */ - spin_unlock_irqrestore(&tx_ctx->base.lock, flags); - - tls_end_offset = record->end_seq - tcp_seq; - - pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n", - tcp_seq, record->end_seq, tx_info->prev_seq, data_len); - /* if a tls record is finishing in this SKB */ - if (tls_end_offset <= data_len) { - struct sk_buff *nskb = NULL; - - if (tls_end_offset < data_len) { - nskb = alloc_skb(0, GFP_KERNEL); - if (unlikely(!nskb)) { - ret = -ENOMEM; - goto clear_ref; - } - - chcr_ktls_skb_shift(nskb, local_skb, - tls_end_offset); - } else { - /* its the only record in this skb, directly - * point it. - */ - nskb = local_skb; - } - ret = chcr_end_part_handler(tx_info, nskb, record, - tcp_seq, mss, - (!th->fin && th->psh), q, - tls_end_offset, - (nskb == local_skb)); - - if (ret && nskb != local_skb) - dev_kfree_skb_any(local_skb); - - data_len -= tls_end_offset; - /* tcp_seq increment is required to handle next record. - */ - tcp_seq += tls_end_offset; - } else { - ret = chcr_short_record_handler(tx_info, local_skb, - record, tcp_seq, mss, - (!th->fin && th->psh), - q, tls_end_offset); - data_len = 0; - } -clear_ref: - /* clear the frag ref count which increased locally before */ - for (i = 0; i < record->num_frags; i++) { - /* clear the frag ref count */ - __skb_frag_unref(&record->frags[i]); - } - /* if any failure, come out from the loop. */ - if (ret) - goto out; - /* length should never be less than 0 */ - WARN_ON(data_len < 0); - - } while (data_len > 0); - - tx_info->prev_seq = ntohl(th->seq) + skb->data_len; - - atomic64_inc(&stats->ktls_tx_encrypted_packets); - atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes); - - /* tcp finish is set, send a separate tcp msg including all the options - * as well. - */ - if (th->fin) - chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan); - -out: - dev_kfree_skb_any(skb); - return NETDEV_TX_OK; -} -#endif /* CONFIG_CHELSIO_TLS_DEVICE */ diff --git a/drivers/crypto/chelsio/chcr_ktls.h b/drivers/crypto/chelsio/chcr_ktls.h deleted file mode 100644 index 5cbd84b1da05..000000000000 --- a/drivers/crypto/chelsio/chcr_ktls.h +++ /dev/null @@ -1,103 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0-only */ -/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ - -#ifndef __CHCR_KTLS_H__ -#define __CHCR_KTLS_H__ - -#ifdef CONFIG_CHELSIO_TLS_DEVICE -#include -#include "cxgb4.h" -#include "t4_msg.h" -#include "t4_tcb.h" -#include "l2t.h" -#include "chcr_common.h" -#include "cxgb4_uld.h" - -#define CHCR_TCB_STATE_CLOSED 0 -#define CHCR_KTLS_KEY_CTX_LEN 16 -#define CHCR_SET_TCB_FIELD_LEN sizeof(struct cpl_set_tcb_field) -#define CHCR_PLAIN_TX_DATA_LEN (sizeof(struct fw_ulptx_wr) +\ - sizeof(struct ulp_txpkt) +\ - sizeof(struct ulptx_idata) +\ - sizeof(struct cpl_tx_data)) - -#define CHCR_KTLS_WR_SIZE (CHCR_PLAIN_TX_DATA_LEN +\ - sizeof(struct cpl_tx_sec_pdu)) - -enum chcr_ktls_conn_state { - KTLS_CONN_CLOSED, - KTLS_CONN_ACT_OPEN_REQ, - KTLS_CONN_ACT_OPEN_RPL, - KTLS_CONN_SET_TCB_REQ, - KTLS_CONN_SET_TCB_RPL, - KTLS_CONN_TX_READY, -}; - -struct chcr_ktls_info { - struct sock *sk; - spinlock_t lock; /* state machine lock */ - struct ktls_key_ctx key_ctx; - struct adapter *adap; - struct l2t_entry *l2te; - struct net_device *netdev; - u64 iv; - u64 record_no; - int tid; - int atid; - int rx_qid; - u32 iv_size; - u32 prev_seq; - u32 prev_ack; - u32 salt_size; - u32 key_ctx_len; - u32 scmd0_seqno_numivs; - u32 scmd0_ivgen_hdrlen; - u32 tcp_start_seq_number; - u32 scmd0_short_seqno_numivs; - u32 scmd0_short_ivgen_hdrlen; - enum chcr_ktls_conn_state connection_state; - u16 prev_win; - u8 tx_chan; - u8 smt_idx; - u8 port_id; - u8 ip_family; - u8 first_qset; -}; - -struct chcr_ktls_ofld_ctx_tx { - struct tls_offload_context_tx base; - struct chcr_ktls_info *chcr_info; -}; - -static inline struct chcr_ktls_ofld_ctx_tx * -chcr_get_ktls_tx_context(struct tls_context *tls_ctx) -{ - BUILD_BUG_ON(sizeof(struct chcr_ktls_ofld_ctx_tx) > - TLS_OFFLOAD_CONTEXT_SIZE_TX); - return container_of(tls_offload_ctx_tx(tls_ctx), - struct chcr_ktls_ofld_ctx_tx, - base); -} - -static inline int chcr_get_first_rx_qid(struct adapter *adap) -{ - /* u_ctx is saved in adap, fetch it */ - struct uld_ctx *u_ctx = adap->uld[CXGB4_ULD_CRYPTO].handle; - - if (!u_ctx) - return -1; - return u_ctx->lldi.rxq_ids[0]; -} - -int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input); -int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev); -int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, - enum tls_offload_ctx_dir direction, - struct tls_crypto_info *crypto_info, - u32 start_offload_tcp_sn); -void chcr_ktls_dev_del(struct net_device *netdev, - struct tls_context *tls_ctx, - enum tls_offload_ctx_dir direction); -#endif /* CONFIG_CHELSIO_TLS_DEVICE */ -#endif /* __CHCR_KTLS_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h index abab82ad3f63..f55550d3a429 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4.h @@ -1201,6 +1201,9 @@ struct adapter { struct cxgb4_tc_u32_table *tc_u32; struct chcr_ktls chcr_ktls; struct chcr_stats_debug chcr_stats; +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) + struct ch_ktls_stats_debug ch_ktls_stats; +#endif #if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) struct ch_ipsec_stats_debug ch_ipsec_stats; #endif @@ -2177,7 +2180,7 @@ void cxgb4_enable_rx(struct adapter *adap, struct sge_rspq *q); void cxgb4_quiesce_rx(struct sge_rspq *q); int cxgb4_port_mirror_alloc(struct net_device *dev); void cxgb4_port_mirror_free(struct net_device *dev); -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) int cxgb4_set_ktls_feature(struct adapter *adap, bool enable); #endif #endif /* __CXGB4_H__ */ diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c index 42112e8ad687..5491a41fd1be 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_debugfs.c @@ -3553,44 +3553,43 @@ static int chcr_stats_show(struct seq_file *seq, void *v) seq_printf(seq, "IPSec PDU: %10u\n", atomic_read(&adap->ch_ipsec_stats.ipsec_cnt)); #endif -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) seq_puts(seq, "\nChelsio KTLS Crypto Accelerator Stats\n"); seq_printf(seq, "Tx TLS offload refcount: %20u\n", refcount_read(&adap->chcr_ktls.ktls_refcount)); seq_printf(seq, "Tx HW offload contexts added: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_ctx)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_ctx)); seq_printf(seq, "Tx connection created: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_connection_open)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_open)); seq_printf(seq, "Tx connection failed: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_connection_fail)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_fail)); seq_printf(seq, "Tx connection closed: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_connection_close)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_connection_close)); seq_printf(seq, "Packets passed for encryption : %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_encrypted_packets)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_encrypted_packets)); seq_printf(seq, "Bytes passed for encryption : %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_encrypted_bytes)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_encrypted_bytes)); seq_printf(seq, "Tx records send: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_send_records)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_send_records)); seq_printf(seq, "Tx partial start of records: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_start_pkts)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_start_pkts)); seq_printf(seq, "Tx partial middle of records: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_middle_pkts)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_middle_pkts)); seq_printf(seq, "Tx partial end of record: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_end_pkts)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_end_pkts)); seq_printf(seq, "Tx complete records: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_complete_pkts)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_complete_pkts)); seq_printf(seq, "TX trim pkts : %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_trimmed_pkts)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_trimmed_pkts)); seq_printf(seq, "Tx out of order packets: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_ooo)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_ooo)); seq_printf(seq, "Tx drop pkts before HW offload: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_skip_no_sync_data)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_skip_no_sync_data)); seq_printf(seq, "Tx drop not synced packets: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_drop_no_sync_data)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_drop_no_sync_data)); seq_printf(seq, "Tx drop bypass req: %20llu\n", - atomic64_read(&adap->chcr_stats.ktls_tx_drop_bypass_req)); + atomic64_read(&adap->ch_ktls_stats.ktls_tx_drop_bypass_req)); #endif - return 0; } DEFINE_SHOW_ATTRIBUTE(chcr_stats); diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c index 9f3173f86eed..8c6c9bedcba1 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_ethtool.c @@ -125,7 +125,7 @@ static char adapter_stats_strings[][ETH_GSTRING_LEN] = { "db_empty ", "write_coal_success ", "write_coal_fail ", -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) "tx_tls_encrypted_packets", "tx_tls_encrypted_bytes ", "tx_tls_ctx ", @@ -265,7 +265,7 @@ struct adapter_stats { u64 db_empty; u64 wc_success; u64 wc_fail; -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) u64 tx_tls_encrypted_packets; u64 tx_tls_encrypted_bytes; u64 tx_tls_ctx; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c index 27530c0e2d14..a952fe198eb9 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_main.c @@ -66,7 +66,7 @@ #include #include #include -#if defined(CONFIG_CHELSIO_TLS_DEVICE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) #include #endif @@ -6396,21 +6396,21 @@ static int cxgb4_iov_configure(struct pci_dev *pdev, int num_vfs) } #endif /* CONFIG_PCI_IOV */ -#if defined(CONFIG_CHELSIO_TLS_DEVICE) || IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) || IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) static int chcr_offload_state(struct adapter *adap, enum cxgb4_netdev_tls_ops op_val) { switch (op_val) { -#if defined(CONFIG_CHELSIO_TLS_DEVICE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) case CXGB4_TLSDEV_OPS: - if (!adap->uld[CXGB4_ULD_CRYPTO].handle) { - dev_dbg(adap->pdev_dev, "chcr driver is not loaded\n"); + if (!adap->uld[CXGB4_ULD_KTLS].handle) { + dev_dbg(adap->pdev_dev, "ch_ktls driver is not loaded\n"); return -EOPNOTSUPP; } - if (!adap->uld[CXGB4_ULD_CRYPTO].tlsdev_ops) { + if (!adap->uld[CXGB4_ULD_KTLS].tlsdev_ops) { dev_dbg(adap->pdev_dev, - "chcr driver has no registered tlsdev_ops\n"); + "ch_ktls driver has no registered tlsdev_ops\n"); return -EOPNOTSUPP; } break; @@ -6439,7 +6439,7 @@ static int chcr_offload_state(struct adapter *adap, #endif /* CONFIG_CHELSIO_TLS_DEVICE || CONFIG_CHELSIO_IPSEC_INLINE */ -#if defined(CONFIG_CHELSIO_TLS_DEVICE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) static int cxgb4_ktls_dev_add(struct net_device *netdev, struct sock *sk, enum tls_offload_ctx_dir direction, @@ -6458,10 +6458,10 @@ static int cxgb4_ktls_dev_add(struct net_device *netdev, struct sock *sk, if (ret) goto out_unlock; - ret = adap->uld[CXGB4_ULD_CRYPTO].tlsdev_ops->tls_dev_add(netdev, sk, - direction, - crypto_info, - tcp_sn); + ret = adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_add(netdev, sk, + direction, + crypto_info, + tcp_sn); /* if there is a failure, clear the refcount */ if (ret) cxgb4_set_ktls_feature(adap, @@ -6481,14 +6481,20 @@ static void cxgb4_ktls_dev_del(struct net_device *netdev, if (chcr_offload_state(adap, CXGB4_TLSDEV_OPS)) goto out_unlock; - adap->uld[CXGB4_ULD_CRYPTO].tlsdev_ops->tls_dev_del(netdev, tls_ctx, - direction); + adap->uld[CXGB4_ULD_KTLS].tlsdev_ops->tls_dev_del(netdev, tls_ctx, + direction); cxgb4_set_ktls_feature(adap, FW_PARAMS_PARAM_DEV_KTLS_HW_DISABLE); out_unlock: mutex_unlock(&uld_mutex); } +static const struct tlsdev_ops cxgb4_ktls_ops = { + .tls_dev_add = cxgb4_ktls_dev_add, + .tls_dev_del = cxgb4_ktls_dev_del, +}; +#endif /* CONFIG_CHELSIO_TLS_DEVICE */ + #if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) static int cxgb4_xfrm_add_state(struct xfrm_state *x) @@ -6597,12 +6603,6 @@ static const struct xfrmdev_ops cxgb4_xfrmdev_ops = { #endif /* CONFIG_CHELSIO_IPSEC_INLINE */ -static const struct tlsdev_ops cxgb4_ktls_ops = { - .tls_dev_add = cxgb4_ktls_dev_add, - .tls_dev_del = cxgb4_ktls_dev_del, -}; -#endif /* CONFIG_CHELSIO_TLS_DEVICE */ - static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) { struct net_device *netdev; @@ -6855,7 +6855,7 @@ static int init_one(struct pci_dev *pdev, const struct pci_device_id *ent) netdev->hw_features |= NETIF_F_HIGHDMA; netdev->features |= netdev->hw_features; netdev->vlan_features = netdev->features & VLAN_FEAT; -#if defined(CONFIG_CHELSIO_TLS_DEVICE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) if (pi->adapter->params.crypto & FW_CAPS_CONFIG_TLS_HW) { netdev->hw_features |= NETIF_F_HW_TLS_TX; netdev->tlsdev_ops = &cxgb4_ktls_ops; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c index 08439e215efe..b154190e1ee2 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.c @@ -663,7 +663,7 @@ static int uld_attach(struct adapter *adap, unsigned int uld) return 0; } -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) static bool cxgb4_uld_in_use(struct adapter *adap) { const struct tid_info *t = &adap->tids; diff --git a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h index 421ae87aa7db..ea2fabbdd934 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h +++ b/drivers/net/ethernet/chelsio/cxgb4/cxgb4_uld.h @@ -40,6 +40,7 @@ #include #include #include +#include #include "cxgb4.h" #define MAX_ULD_QSETS 16 @@ -304,6 +305,7 @@ enum cxgb4_uld { CXGB4_ULD_CRYPTO, CXGB4_ULD_IPSEC, CXGB4_ULD_TLS, + CXGB4_ULD_KTLS, CXGB4_ULD_MAX }; @@ -362,17 +364,8 @@ struct cxgb4_virt_res { /* virtualized HW resources */ struct cxgb4_range ppod_edram; }; -struct chcr_stats_debug { - atomic_t cipher_rqst; - atomic_t digest_rqst; - atomic_t aead_rqst; - atomic_t complete; - atomic_t error; - atomic_t fallback; - atomic_t tls_pdu_tx; - atomic_t tls_pdu_rx; - atomic_t tls_key; -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) +struct ch_ktls_stats_debug { atomic64_t ktls_tx_connection_open; atomic64_t ktls_tx_connection_fail; atomic64_t ktls_tx_connection_close; @@ -390,8 +383,19 @@ struct chcr_stats_debug { atomic64_t ktls_tx_skip_no_sync_data; atomic64_t ktls_tx_drop_no_sync_data; atomic64_t ktls_tx_drop_bypass_req; - +}; #endif + +struct chcr_stats_debug { + atomic_t cipher_rqst; + atomic_t digest_rqst; + atomic_t aead_rqst; + atomic_t complete; + atomic_t error; + atomic_t fallback; + atomic_t tls_pdu_tx; + atomic_t tls_pdu_rx; + atomic_t tls_key; }; #if IS_ENABLED(CONFIG_CHELSIO_IPSEC_INLINE) @@ -476,7 +480,7 @@ struct cxgb4_uld_info { struct napi_struct *napi); void (*lro_flush)(struct t4_lro_mgr *); int (*tx_handler)(struct sk_buff *skb, struct net_device *dev); -#if IS_ENABLED(CONFIG_TLS_DEVICE) +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) const struct tlsdev_ops *tlsdev_ops; #endif #if IS_ENABLED(CONFIG_XFRM_OFFLOAD) diff --git a/drivers/net/ethernet/chelsio/cxgb4/sge.c b/drivers/net/ethernet/chelsio/cxgb4/sge.c index fddd70ee6436..437c054ef749 100644 --- a/drivers/net/ethernet/chelsio/cxgb4/sge.c +++ b/drivers/net/ethernet/chelsio/cxgb4/sge.c @@ -1421,9 +1421,9 @@ static netdev_tx_t cxgb4_eth_xmit(struct sk_buff *skb, struct net_device *dev) return adap->uld[CXGB4_ULD_IPSEC].tx_handler(skb, dev); #endif /* CHELSIO_IPSEC_INLINE */ -#ifdef CONFIG_CHELSIO_TLS_DEVICE +#if IS_ENABLED(CONFIG_CHELSIO_TLS_DEVICE) if (skb->decrypted) - return adap->uld[CXGB4_ULD_CRYPTO].tx_handler(skb, dev); + return adap->uld[CXGB4_ULD_KTLS].tx_handler(skb, dev); #endif /* CHELSIO_TLS_DEVICE */ qidx = skb_get_queue_mapping(skb); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig index be70b59b6f80..1923e713b53a 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/Kconfig +++ b/drivers/net/ethernet/chelsio/inline_crypto/Kconfig @@ -34,4 +34,17 @@ config CHELSIO_IPSEC_INLINE To compile this driver as a module, choose M here: the module will be called ch_ipsec. +config CHELSIO_TLS_DEVICE + tristate "Chelsio Inline KTLS Offload" + depends on CHELSIO_T4 + depends on TLS + depends on TLS_DEVICE + help + This flag enables support for kernel tls offload over Chelsio T6 + crypto accelerator. CONFIG_CHELSIO_TLS_DEVICE flag can be enabled + only if CONFIG_TLS and CONFIG_TLS_DEVICE flags are enabled. + + To compile this driver as a module, choose M here: the module + will be called ch_ktls. + endif # CHELSIO_INLINE_CRYPTO diff --git a/drivers/net/ethernet/chelsio/inline_crypto/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/Makefile index 9a86ee8f1f38..27e6d7e2f1eb 100644 --- a/drivers/net/ethernet/chelsio/inline_crypto/Makefile +++ b/drivers/net/ethernet/chelsio/inline_crypto/Makefile @@ -1,3 +1,4 @@ # SPDX-License-Identifier: GPL-2.0-only obj-$(CONFIG_CRYPTO_DEV_CHELSIO_TLS) += chtls/ obj-$(CONFIG_CHELSIO_IPSEC_INLINE) += ch_ipsec/ +obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls/ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile new file mode 100644 index 000000000000..5e7d161c3199 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0-only +ccflags-y := -I $(srctree)/drivers/net/ethernet/chelsio/cxgb4 + +obj-$(CONFIG_CHELSIO_TLS_DEVICE) += ch_ktls.o +ch_ktls-objs := chcr_ktls.o diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h new file mode 100644 index 000000000000..38319f4c3121 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_common.h @@ -0,0 +1,111 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#ifndef __CHCR_COMMON_H__ +#define __CHCR_COMMON_H__ + +#include "cxgb4.h" + +#define CHCR_MAX_SALT 4 +#define CHCR_KEYCTX_MAC_KEY_SIZE_128 0 +#define CHCR_KEYCTX_CIPHER_KEY_SIZE_128 0 +#define CHCR_SCMD_CIPHER_MODE_AES_GCM 2 +#define CHCR_SCMD_CIPHER_MODE_AES_CTR 3 +#define CHCR_CPL_TX_SEC_PDU_LEN_64BIT 2 +#define CHCR_SCMD_SEQ_NO_CTRL_64BIT 3 +#define CHCR_SCMD_PROTO_VERSION_TLS 0 +#define CHCR_SCMD_PROTO_VERSION_GENERIC 4 +#define CHCR_SCMD_AUTH_MODE_GHASH 4 +#define AES_BLOCK_LEN 16 + +struct ktls_key_ctx { + __be32 ctx_hdr; + u8 salt[CHCR_MAX_SALT]; + __be64 iv_to_auth; + unsigned char key[TLS_CIPHER_AES_GCM_128_KEY_SIZE + + TLS_CIPHER_AES_GCM_256_TAG_SIZE]; +}; + +/* Crypto key context */ +#define KEY_CONTEXT_CTX_LEN_S 24 +#define KEY_CONTEXT_CTX_LEN_V(x) ((x) << KEY_CONTEXT_CTX_LEN_S) + +#define KEY_CONTEXT_SALT_PRESENT_S 10 +#define KEY_CONTEXT_SALT_PRESENT_V(x) ((x) << KEY_CONTEXT_SALT_PRESENT_S) +#define KEY_CONTEXT_SALT_PRESENT_F KEY_CONTEXT_SALT_PRESENT_V(1U) + +#define KEY_CONTEXT_VALID_S 0 +#define KEY_CONTEXT_VALID_V(x) ((x) << KEY_CONTEXT_VALID_S) +#define KEY_CONTEXT_VALID_F KEY_CONTEXT_VALID_V(1U) + +#define KEY_CONTEXT_CK_SIZE_S 6 +#define KEY_CONTEXT_CK_SIZE_V(x) ((x) << KEY_CONTEXT_CK_SIZE_S) + +#define KEY_CONTEXT_MK_SIZE_S 2 +#define KEY_CONTEXT_MK_SIZE_V(x) ((x) << KEY_CONTEXT_MK_SIZE_S) + +#define KEY_CONTEXT_OPAD_PRESENT_S 11 +#define KEY_CONTEXT_OPAD_PRESENT_V(x) ((x) << KEY_CONTEXT_OPAD_PRESENT_S) +#define KEY_CONTEXT_OPAD_PRESENT_F KEY_CONTEXT_OPAD_PRESENT_V(1U) + +#define FILL_KEY_CTX_HDR(ck_size, mk_size, ctx_len) \ + htonl(KEY_CONTEXT_MK_SIZE_V(mk_size) | \ + KEY_CONTEXT_CK_SIZE_V(ck_size) | \ + KEY_CONTEXT_VALID_F | \ + KEY_CONTEXT_SALT_PRESENT_F | \ + KEY_CONTEXT_CTX_LEN_V((ctx_len))) + +static inline void *chcr_copy_to_txd(const void *src, const struct sge_txq *q, + void *pos, int length) +{ + int left = (void *)q->stat - pos; + u64 *p; + + if (likely(length <= left)) { + memcpy(pos, src, length); + pos += length; + } else { + memcpy(pos, src, left); + memcpy(q->desc, src + left, length - left); + pos = (void *)q->desc + (length - left); + } + /* 0-pad to multiple of 16 */ + p = PTR_ALIGN(pos, 8); + if ((uintptr_t)p & 8) { + *p = 0; + return p + 1; + } + return p; +} + +static inline unsigned int chcr_txq_avail(const struct sge_txq *q) +{ + return q->size - 1 - q->in_use; +} + +static inline void chcr_txq_advance(struct sge_txq *q, unsigned int n) +{ + q->in_use += n; + q->pidx += n; + if (q->pidx >= q->size) + q->pidx -= q->size; +} + +static inline void chcr_eth_txq_stop(struct sge_eth_txq *q) +{ + netif_tx_stop_queue(q->txq); + q->q.stops++; +} + +static inline unsigned int chcr_sgl_len(unsigned int n) +{ + n--; + return (3 * n) / 2 + (n & 1) + 2; +} + +static inline unsigned int chcr_flits_to_desc(unsigned int n) +{ + WARN_ON(n > SGE_MAX_WR_LEN / 8); + return DIV_ROUND_UP(n, 8); +} +#endif /* __CHCR_COMMON_H__ */ diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c new file mode 100644 index 000000000000..4609f1f78426 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.c @@ -0,0 +1,2139 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include "chcr_ktls.h" + +static LIST_HEAD(uld_ctx_list); +static DEFINE_MUTEX(dev_mutex); + +static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info); +/* + * chcr_ktls_save_keys: calculate and save crypto keys. + * @tx_info - driver specific tls info. + * @crypto_info - tls crypto information. + * @direction - TX/RX direction. + * return - SUCCESS/FAILURE. + */ +static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info, + struct tls_crypto_info *crypto_info, + enum tls_offload_ctx_dir direction) +{ + int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret; + unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE]; + struct tls12_crypto_info_aes_gcm_128 *info_128_gcm; + struct ktls_key_ctx *kctx = &tx_info->key_ctx; + struct crypto_cipher *cipher; + unsigned char *key, *salt; + + switch (crypto_info->cipher_type) { + case TLS_CIPHER_AES_GCM_128: + info_128_gcm = + (struct tls12_crypto_info_aes_gcm_128 *)crypto_info; + keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE; + ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128; + tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE; + mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128; + tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE; + tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv); + + ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE; + key = info_128_gcm->key; + salt = info_128_gcm->salt; + tx_info->record_no = *(u64 *)info_128_gcm->rec_seq; + + /* The SCMD fields used when encrypting a full TLS + * record. Its a one time calculation till the + * connection exists. + */ + tx_info->scmd0_seqno_numivs = + SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) | + SCMD_CIPH_AUTH_SEQ_CTRL_F | + SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) | + SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) | + SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) | + SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) | + SCMD_NUM_IVS_V(1); + + /* keys will be sent inline. */ + tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F; + + /* The SCMD fields used when encrypting a partial TLS + * record (no trailer and possibly a truncated payload). + */ + tx_info->scmd0_short_seqno_numivs = + SCMD_CIPH_AUTH_SEQ_CTRL_F | + SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) | + SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) | + SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1); + + tx_info->scmd0_short_ivgen_hdrlen = + tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F; + + break; + + default: + pr_err("GCM: cipher type 0x%x not supported\n", + crypto_info->cipher_type); + ret = -EINVAL; + goto out; + } + + key_ctx_size = CHCR_KTLS_KEY_CTX_LEN + + roundup(keylen, 16) + ghash_size; + /* Calculate the H = CIPH(K, 0 repeated 16 times). + * It will go in key context + */ + cipher = crypto_alloc_cipher("aes", 0, 0); + if (IS_ERR(cipher)) { + ret = -ENOMEM; + goto out; + } + + ret = crypto_cipher_setkey(cipher, key, keylen); + if (ret) + goto out1; + + memset(ghash_h, 0, ghash_size); + crypto_cipher_encrypt_one(cipher, ghash_h, ghash_h); + + /* fill the Key context */ + if (direction == TLS_OFFLOAD_CTX_DIR_TX) { + kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size, + mac_key_size, + key_ctx_size >> 4); + } else { + ret = -EINVAL; + goto out1; + } + + memcpy(kctx->salt, salt, tx_info->salt_size); + memcpy(kctx->key, key, keylen); + memcpy(kctx->key + keylen, ghash_h, ghash_size); + tx_info->key_ctx_len = key_ctx_size; + +out1: + crypto_free_cipher(cipher); +out: + return ret; +} + +static int chcr_ktls_update_connection_state(struct chcr_ktls_info *tx_info, + int new_state) +{ + /* This function can be called from both rx (interrupt context) and tx + * queue contexts. + */ + spin_lock_bh(&tx_info->lock); + switch (tx_info->connection_state) { + case KTLS_CONN_CLOSED: + tx_info->connection_state = new_state; + break; + + case KTLS_CONN_ACT_OPEN_REQ: + /* only go forward if state is greater than current state. */ + if (new_state <= tx_info->connection_state) + break; + /* update to the next state and also initialize TCB */ + tx_info->connection_state = new_state; + fallthrough; + case KTLS_CONN_ACT_OPEN_RPL: + /* if we are stuck in this state, means tcb init might not + * received by HW, try sending it again. + */ + if (!chcr_init_tcb_fields(tx_info)) + tx_info->connection_state = KTLS_CONN_SET_TCB_REQ; + break; + + case KTLS_CONN_SET_TCB_REQ: + /* only go forward if state is greater than current state. */ + if (new_state <= tx_info->connection_state) + break; + /* update to the next state and check if l2t_state is valid */ + tx_info->connection_state = new_state; + fallthrough; + case KTLS_CONN_SET_TCB_RPL: + /* Check if l2t state is valid, then move to ready state. */ + if (cxgb4_check_l2t_valid(tx_info->l2te)) { + tx_info->connection_state = KTLS_CONN_TX_READY; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_ctx); + } + break; + + case KTLS_CONN_TX_READY: + /* nothing to be done here */ + break; + + default: + pr_err("unknown KTLS connection state\n"); + break; + } + spin_unlock_bh(&tx_info->lock); + + return tx_info->connection_state; +} +/* + * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * @atid - connection active tid. + * return - send success/failure. + */ +static int chcr_ktls_act_open_req(struct sock *sk, + struct chcr_ktls_info *tx_info, + int atid) +{ + struct inet_sock *inet = inet_sk(sk); + struct cpl_t6_act_open_req *cpl6; + struct cpl_act_open_req *cpl; + struct sk_buff *skb; + unsigned int len; + int qid_atid; + u64 options; + + len = sizeof(*cpl6); + skb = alloc_skb(len, GFP_KERNEL); + if (unlikely(!skb)) + return -ENOMEM; + /* mark it a control pkt */ + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + + cpl6 = __skb_put_zero(skb, len); + cpl = (struct cpl_act_open_req *)cpl6; + INIT_TP_WR(cpl6, 0); + qid_atid = TID_QID_V(tx_info->rx_qid) | + TID_TID_V(atid); + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid)); + cpl->local_port = inet->inet_sport; + cpl->peer_port = inet->inet_dport; + cpl->local_ip = inet->inet_rcv_saddr; + cpl->peer_ip = inet->inet_daddr; + + /* fill first 64 bit option field. */ + options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | + SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); + cpl->opt0 = cpu_to_be64(options); + + /* next 64 bit option field. */ + options = + TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); + cpl->opt2 = htonl(options); + + return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); +} + +#if IS_ENABLED(CONFIG_IPV6) +/* + * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * @atid - connection active tid. + * return - send success/failure. + */ +static int chcr_ktls_act_open_req6(struct sock *sk, + struct chcr_ktls_info *tx_info, + int atid) +{ + struct inet_sock *inet = inet_sk(sk); + struct cpl_t6_act_open_req6 *cpl6; + struct cpl_act_open_req6 *cpl; + struct sk_buff *skb; + unsigned int len; + int qid_atid; + u64 options; + + len = sizeof(*cpl6); + skb = alloc_skb(len, GFP_KERNEL); + if (unlikely(!skb)) + return -ENOMEM; + /* mark it a control pkt */ + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + + cpl6 = __skb_put_zero(skb, len); + cpl = (struct cpl_act_open_req6 *)cpl6; + INIT_TP_WR(cpl6, 0); + qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid); + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid)); + cpl->local_port = inet->inet_sport; + cpl->peer_port = inet->inet_dport; + cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0]; + cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8]; + cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0]; + cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8]; + + /* first 64 bit option field. */ + options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F | + SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan); + cpl->opt0 = cpu_to_be64(options); + /* next 64 bit option field. */ + options = + TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]); + cpl->opt2 = htonl(options); + + return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te); +} +#endif /* #if IS_ENABLED(CONFIG_IPV6) */ + +/* + * chcr_setup_connection: create a TCB entry so that TP will form tcp packets. + * @sk - tcp socket. + * @tx_info - driver specific tls info. + * return: NET_TX_OK/NET_XMIT_DROP + */ +static int chcr_setup_connection(struct sock *sk, + struct chcr_ktls_info *tx_info) +{ + struct tid_info *t = &tx_info->adap->tids; + int atid, ret = 0; + + atid = cxgb4_alloc_atid(t, tx_info); + if (atid == -1) + return -EINVAL; + + tx_info->atid = atid; + tx_info->ip_family = sk->sk_family; + + if (sk->sk_family == AF_INET) { + tx_info->ip_family = AF_INET; + ret = chcr_ktls_act_open_req(sk, tx_info, atid); +#if IS_ENABLED(CONFIG_IPV6) + } else { + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) { + tx_info->ip_family = AF_INET; + ret = chcr_ktls_act_open_req(sk, tx_info, atid); + } else { + tx_info->ip_family = AF_INET6; + ret = cxgb4_clip_get(tx_info->netdev, + (const u32 *) + &sk->sk_v6_rcv_saddr.s6_addr, + 1); + if (ret) + goto out; + ret = chcr_ktls_act_open_req6(sk, tx_info, atid); + } +#endif + } + + /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret + * success, if any other return type clear atid and return that failure. + */ + if (ret) { + if (ret == NET_XMIT_CN) + ret = 0; + else + cxgb4_free_atid(t, atid); + goto out; + } + + /* update the connection state */ + chcr_ktls_update_connection_state(tx_info, KTLS_CONN_ACT_OPEN_REQ); +out: + return ret; +} + +/* + * chcr_set_tcb_field: update tcb fields. + * @tx_info - driver specific tls info. + * @word - TCB word. + * @mask - TCB word related mask. + * @val - TCB word related value. + * @no_reply - set 1 if not looking for TP response. + */ +static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word, + u64 mask, u64 val, int no_reply) +{ + struct cpl_set_tcb_field *req; + struct sk_buff *skb; + + skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC); + if (!skb) + return -ENOMEM; + + req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req)); + INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid); + req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | + NO_REPLY_V(no_reply)); + req->word_cookie = htons(TCB_WORD_V(word)); + req->mask = cpu_to_be64(mask); + req->val = cpu_to_be64(val); + + set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id); + return cxgb4_ofld_send(tx_info->netdev, skb); +} + +/* + * chcr_ktls_mark_tcb_close: mark tcb state to CLOSE + * @tx_info - driver specific tls info. + * return: NET_TX_OK/NET_XMIT_DROP. + */ +static int chcr_ktls_mark_tcb_close(struct chcr_ktls_info *tx_info) +{ + return chcr_set_tcb_field(tx_info, TCB_T_STATE_W, + TCB_T_STATE_V(TCB_T_STATE_M), + CHCR_TCB_STATE_CLOSED, 1); +} + +/* + * chcr_ktls_dev_del: call back for tls_dev_del. + * Remove the tid and l2t entry and close the connection. + * it per connection basis. + * @netdev - net device. + * @tls_cts - tls context. + * @direction - TX/RX crypto direction + */ +static void chcr_ktls_dev_del(struct net_device *netdev, + struct tls_context *tls_ctx, + enum tls_offload_ctx_dir direction) +{ + struct chcr_ktls_ofld_ctx_tx *tx_ctx = + chcr_get_ktls_tx_context(tls_ctx); + struct chcr_ktls_info *tx_info = tx_ctx->chcr_info; + struct sock *sk; + + if (!tx_info) + return; + sk = tx_info->sk; + + spin_lock(&tx_info->lock); + tx_info->connection_state = KTLS_CONN_CLOSED; + spin_unlock(&tx_info->lock); + + /* clear l2t entry */ + if (tx_info->l2te) + cxgb4_l2t_release(tx_info->l2te); + +#if IS_ENABLED(CONFIG_IPV6) + /* clear clip entry */ + if (tx_info->ip_family == AF_INET6) + cxgb4_clip_release(netdev, + (const u32 *)&sk->sk_v6_daddr.in6_u.u6_addr8, + 1); +#endif + + /* clear tid */ + if (tx_info->tid != -1) { + /* clear tcb state and then release tid */ + chcr_ktls_mark_tcb_close(tx_info); + cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan, + tx_info->tid, tx_info->ip_family); + } + + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_connection_close); + kvfree(tx_info); + tx_ctx->chcr_info = NULL; + /* release module refcount */ + module_put(THIS_MODULE); +} + +/* + * chcr_ktls_dev_add: call back for tls_dev_add. + * Create a tcb entry for TP. Also add l2t entry for the connection. And + * generate keys & save those keys locally. + * @netdev - net device. + * @tls_cts - tls context. + * @direction - TX/RX crypto direction + * return: SUCCESS/FAILURE. + */ +static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk, + enum tls_offload_ctx_dir direction, + struct tls_crypto_info *crypto_info, + u32 start_offload_tcp_sn) +{ + struct tls_context *tls_ctx = tls_get_ctx(sk); + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct chcr_ktls_info *tx_info; + struct dst_entry *dst; + struct adapter *adap; + struct port_info *pi; + struct neighbour *n; + u8 daaddr[16]; + int ret = -1; + + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + + pi = netdev_priv(netdev); + adap = pi->adapter; + if (direction == TLS_OFFLOAD_CTX_DIR_RX) { + pr_err("not expecting for RX direction\n"); + ret = -EINVAL; + goto out; + } + if (tx_ctx->chcr_info) { + ret = -EINVAL; + goto out; + } + + tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL); + if (!tx_info) { + ret = -ENOMEM; + goto out; + } + + spin_lock_init(&tx_info->lock); + + /* clear connection state */ + spin_lock(&tx_info->lock); + tx_info->connection_state = KTLS_CONN_CLOSED; + spin_unlock(&tx_info->lock); + + tx_info->sk = sk; + /* initialize tid and atid to -1, 0 is a also a valid id. */ + tx_info->tid = -1; + tx_info->atid = -1; + + tx_info->adap = adap; + tx_info->netdev = netdev; + tx_info->first_qset = pi->first_qset; + tx_info->tx_chan = pi->tx_chan; + tx_info->smt_idx = pi->smt_idx; + tx_info->port_id = pi->port_id; + + tx_info->rx_qid = chcr_get_first_rx_qid(adap); + if (unlikely(tx_info->rx_qid < 0)) + goto out2; + + tx_info->prev_seq = start_offload_tcp_sn; + tx_info->tcp_start_seq_number = start_offload_tcp_sn; + + /* save crypto keys */ + ret = chcr_ktls_save_keys(tx_info, crypto_info, direction); + if (ret < 0) + goto out2; + + /* get peer ip */ + if (sk->sk_family == AF_INET) { + memcpy(daaddr, &sk->sk_daddr, 4); +#if IS_ENABLED(CONFIG_IPV6) + } else { + if (!sk->sk_ipv6only && + ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) + memcpy(daaddr, &sk->sk_daddr, 4); + else + memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16); +#endif + } + + /* get the l2t index */ + dst = sk_dst_get(sk); + if (!dst) { + pr_err("DST entry not found\n"); + goto out2; + } + n = dst_neigh_lookup(dst, daaddr); + if (!n || !n->dev) { + pr_err("neighbour not found\n"); + dst_release(dst); + goto out2; + } + tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0); + + neigh_release(n); + dst_release(dst); + + if (!tx_info->l2te) { + pr_err("l2t entry not found\n"); + goto out2; + } + + tx_ctx->chcr_info = tx_info; + + /* create a filter and call cxgb4_l2t_send to send the packet out, which + * will take care of updating l2t entry in hw if not already done. + */ + ret = chcr_setup_connection(sk, tx_info); + if (ret) + goto out2; + + /* Driver shouldn't be removed until any single connection exists */ + if (!try_module_get(THIS_MODULE)) { + ret = -EINVAL; + goto out2; + } + + atomic64_inc(&adap->ch_ktls_stats.ktls_tx_connection_open); + return 0; +out2: + kvfree(tx_info); +out: + atomic64_inc(&adap->ch_ktls_stats.ktls_tx_connection_fail); + return ret; +} + +/* + * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number + * handling. + * @tx_info - driver specific tls info. + * return: NET_TX_OK/NET_XMIT_DROP + */ +static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info) +{ + int ret = 0; + + /* set tcb in offload and bypass */ + ret = + chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W, + TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F), + TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1); + if (ret) + return ret; + /* reset snd_una and snd_next fields in tcb */ + ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W, + TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) | + TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), + 0, 1); + if (ret) + return ret; + + /* reset send max */ + ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W, + TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M), + 0, 1); + if (ret) + return ret; + + /* update l2t index and request for tp reply to confirm tcb is + * initialised to handle tx traffic. + */ + ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W, + TCB_L2T_IX_V(TCB_L2T_IX_M), + TCB_L2T_IX_V(tx_info->l2te->idx), 0); + return ret; +} + +/* + * chcr_ktls_cpl_act_open_rpl: connection reply received from TP. + */ +static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap, + unsigned char *input) +{ + const struct cpl_act_open_rpl *p = (void *)input; + struct chcr_ktls_info *tx_info = NULL; + unsigned int atid, tid, status; + struct tid_info *t; + + tid = GET_TID(p); + status = AOPEN_STATUS_G(ntohl(p->atid_status)); + atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status))); + + t = &adap->tids; + tx_info = lookup_atid(t, atid); + + if (!tx_info || tx_info->atid != atid) { + pr_err("tx_info or atid is not correct\n"); + return -1; + } + + if (!status) { + tx_info->tid = tid; + cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family); + + cxgb4_free_atid(t, atid); + tx_info->atid = -1; + /* update the connection state */ + chcr_ktls_update_connection_state(tx_info, + KTLS_CONN_ACT_OPEN_RPL); + } + return 0; +} + +/* + * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP. + */ +static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input) +{ + const struct cpl_set_tcb_rpl *p = (void *)input; + struct chcr_ktls_info *tx_info = NULL; + struct tid_info *t; + u32 tid; + + tid = GET_TID(p); + + t = &adap->tids; + tx_info = lookup_tid(t, tid); + if (!tx_info || tx_info->tid != tid) { + pr_err("tx_info or atid is not correct\n"); + return -1; + } + /* update the connection state */ + chcr_ktls_update_connection_state(tx_info, KTLS_CONN_SET_TCB_RPL); + return 0; +} + +static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + u32 tid, void *pos, u16 word, u64 mask, + u64 val, u32 reply) +{ + struct cpl_set_tcb_field_core *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *txpkt; + + /* ULP_TXPKT */ + txpkt = pos; + txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | ULP_TXPKT_DEST_V(0)); + txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16)); + + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(txpkt + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM)); + idata->len = htonl(sizeof(*cpl)); + pos = idata + 1; + + cpl = pos; + /* CPL_SET_TCB_FIELD */ + OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid)); + cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) | + NO_REPLY_V(!reply)); + cpl->word_cookie = htons(TCB_WORD_V(word)); + cpl->mask = cpu_to_be64(mask); + cpl->val = cpu_to_be64(val); + + /* ULPTX_NOOP */ + idata = (struct ulptx_idata *)(cpl + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP)); + idata->len = htonl(0); + pos = idata + 1; + + return pos; +} + + +/* + * chcr_write_cpl_set_tcb_ulp: update tcb values. + * TCB is responsible to create tcp headers, so all the related values + * should be correctly updated. + * @tx_info - driver specific tls info. + * @q - tx queue on which packet is going out. + * @tid - TCB identifier. + * @pos - current index where should we start writing. + * @word - TCB word. + * @mask - TCB word related mask. + * @val - TCB word related value. + * @reply - set 1 if looking for TP response. + * return - next position to write. + */ +static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u32 tid, + void *pos, u16 word, u64 mask, + u64 val, u32 reply) +{ + int left = (void *)q->q.stat - pos; + + if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) { + if (!left) { + pos = q->q.desc; + } else { + u8 buf[48] = {0}; + + __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, + mask, val, reply); + + return chcr_copy_to_txd(buf, &q->q, pos, + CHCR_SET_TCB_FIELD_LEN); + } + } + + pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, + mask, val, reply); + + /* check again if we are at the end of the queue */ + if (left == CHCR_SET_TCB_FIELD_LEN) + pos = q->q.desc; + + return pos; +} + +/* + * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header + * with updated values like tcp seq, ack, window etc. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_ack + * @tcp_win + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u64 tcp_seq, + u64 tcp_ack, u64 tcp_win) +{ + bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0)); + u32 len, cpl = 0, ndesc, wr_len; + struct fw_ulptx_wr *wr; + int credits; + void *pos; + + wr_len = sizeof(*wr); + /* there can be max 4 cpls, check if we have enough credits */ + len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16); + ndesc = DIV_ROUND_UP(len, 64); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + /* make space for WR, we'll fill it later when we know all the cpls + * being sent out and have complete length. + */ + wr = pos; + pos += wr_len; + /* update tx_max if its a re-transmit or the first wr */ + if (first_wr || tcp_seq != tx_info->prev_seq) { + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_TX_MAX_W, + TCB_TX_MAX_V(TCB_TX_MAX_M), + TCB_TX_MAX_V(tcp_seq), 0); + cpl++; + } + /* reset snd una if it's a re-transmit pkt */ + if (tcp_seq != tx_info->prev_seq) { + /* reset snd_una */ + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_SND_UNA_RAW_W, + TCB_SND_UNA_RAW_V + (TCB_SND_UNA_RAW_M), + TCB_SND_UNA_RAW_V(0), 0); + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_ooo); + cpl++; + } + /* update ack */ + if (first_wr || tx_info->prev_ack != tcp_ack) { + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_RCV_NXT_W, + TCB_RCV_NXT_V(TCB_RCV_NXT_M), + TCB_RCV_NXT_V(tcp_ack), 0); + tx_info->prev_ack = tcp_ack; + cpl++; + } + /* update receive window */ + if (first_wr || tx_info->prev_win != tcp_win) { + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_RCV_WND_W, + TCB_RCV_WND_V(TCB_RCV_WND_M), + TCB_RCV_WND_V(tcp_win), 0); + tx_info->prev_win = tcp_win; + cpl++; + } + + if (cpl) { + /* get the actual length */ + len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16); + /* ULPTX wr */ + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->cookie = 0; + /* fill len in wr field */ + wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); + + ndesc = DIV_ROUND_UP(len, 64); + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + } + return 0; +} + +/* + * chcr_ktls_skb_copy + * @nskb - new skb where the frags to be added. + * @skb - old skb from which frags will be copied. + */ +static void chcr_ktls_skb_copy(struct sk_buff *skb, struct sk_buff *nskb) +{ + int i; + + for (i = 0; i < skb_shinfo(skb)->nr_frags; i++) { + skb_shinfo(nskb)->frags[i] = skb_shinfo(skb)->frags[i]; + __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); + } + + skb_shinfo(nskb)->nr_frags = skb_shinfo(skb)->nr_frags; + nskb->len += skb->data_len; + nskb->data_len = skb->data_len; + nskb->truesize += skb->data_len; +} + +/* + * chcr_ktls_get_tx_flits + * returns number of flits to be sent out, it includes key context length, WR + * size and skb fragments. + */ +static unsigned int +chcr_ktls_get_tx_flits(const struct sk_buff *skb, unsigned int key_ctx_len) +{ + return chcr_sgl_len(skb_shinfo(skb)->nr_frags) + + DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8); +} + +/* + * chcr_ktls_check_tcp_options: To check if there is any TCP option availbale + * other than timestamp. + * @skb - skb contains partial record.. + * return: 1 / 0 + */ +static int +chcr_ktls_check_tcp_options(struct tcphdr *tcp) +{ + int cnt, opt, optlen; + u_char *cp; + + cp = (u_char *)(tcp + 1); + cnt = (tcp->doff << 2) - sizeof(struct tcphdr); + for (; cnt > 0; cnt -= optlen, cp += optlen) { + opt = cp[0]; + if (opt == TCPOPT_EOL) + break; + if (opt == TCPOPT_NOP) { + optlen = 1; + } else { + if (cnt < 2) + break; + optlen = cp[1]; + if (optlen < 2 || optlen > cnt) + break; + } + switch (opt) { + case TCPOPT_NOP: + break; + default: + return 1; + } + } + return 0; +} + +/* + * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to + * send out separately. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @q - TX queue. + * @tx_chan - channel number. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int +chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb, + struct sge_eth_txq *q, uint32_t tx_chan) +{ + struct fw_eth_tx_pkt_wr *wr; + struct cpl_tx_pkt_core *cpl; + u32 ctrl, iplen, maclen; +#if IS_ENABLED(CONFIG_IPV6) + struct ipv6hdr *ip6; +#endif + unsigned int ndesc; + struct tcphdr *tcp; + int len16, pktlen; + struct iphdr *ip; + int credits; + u8 buf[150]; + void *pos; + + iplen = skb_network_header_len(skb); + maclen = skb_mac_header_len(skb); + + /* packet length = eth hdr len + ip hdr len + tcp hdr len + * (including options). + */ + pktlen = skb->len - skb->data_len; + + ctrl = sizeof(*cpl) + pktlen; + len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16); + /* check how many descriptors needed */ + ndesc = DIV_ROUND_UP(len16, 4); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + wr = pos; + + /* Firmware work request header */ + wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) | + FW_WR_IMMDLEN_V(ctrl)); + + wr->equiq_to_len16 = htonl(FW_WR_LEN16_V(len16)); + wr->r3 = 0; + + cpl = (void *)(wr + 1); + + /* CPL header */ + cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) | + TXPKT_PF_V(tx_info->adap->pf)); + cpl->pack = 0; + cpl->len = htons(pktlen); + /* checksum offload */ + cpl->ctrl1 = 0; + + pos = cpl + 1; + + memcpy(buf, skb->data, pktlen); + if (tx_info->ip_family == AF_INET) { + /* we need to correct ip header len */ + ip = (struct iphdr *)(buf + maclen); + ip->tot_len = htons(pktlen - maclen); +#if IS_ENABLED(CONFIG_IPV6) + } else { + ip6 = (struct ipv6hdr *)(buf + maclen); + ip6->payload_len = htons(pktlen - maclen - iplen); +#endif + } + /* now take care of the tcp header, if fin is not set then clear push + * bit as well, and if fin is set, it will be sent at the last so we + * need to update the tcp sequence number as per the last packet. + */ + tcp = (struct tcphdr *)(buf + maclen + iplen); + + if (!tcp->fin) + tcp->psh = 0; + else + tcp->seq = htonl(tx_info->prev_seq); + + chcr_copy_to_txd(buf, &q->q, pos, pktlen); + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + return 0; +} + +/* chcr_ktls_skb_shift - Shifts request length paged data from skb to another. + * @tgt- buffer into which tail data gets added + * @skb- buffer from which the paged data comes from + * @shiftlen- shift up to this many bytes + */ +static int chcr_ktls_skb_shift(struct sk_buff *tgt, struct sk_buff *skb, + int shiftlen) +{ + skb_frag_t *fragfrom, *fragto; + int from, to, todo; + + WARN_ON(shiftlen > skb->data_len); + + todo = shiftlen; + from = 0; + to = 0; + fragfrom = &skb_shinfo(skb)->frags[from]; + + while ((todo > 0) && (from < skb_shinfo(skb)->nr_frags)) { + fragfrom = &skb_shinfo(skb)->frags[from]; + fragto = &skb_shinfo(tgt)->frags[to]; + + if (todo >= skb_frag_size(fragfrom)) { + *fragto = *fragfrom; + todo -= skb_frag_size(fragfrom); + from++; + to++; + + } else { + __skb_frag_ref(fragfrom); + skb_frag_page_copy(fragto, fragfrom); + skb_frag_off_copy(fragto, fragfrom); + skb_frag_size_set(fragto, todo); + + skb_frag_off_add(fragfrom, todo); + skb_frag_size_sub(fragfrom, todo); + todo = 0; + + to++; + break; + } + } + + /* Ready to "commit" this state change to tgt */ + skb_shinfo(tgt)->nr_frags = to; + + /* Reposition in the original skb */ + to = 0; + while (from < skb_shinfo(skb)->nr_frags) + skb_shinfo(skb)->frags[to++] = skb_shinfo(skb)->frags[from++]; + + skb_shinfo(skb)->nr_frags = to; + + WARN_ON(todo > 0 && !skb_shinfo(skb)->nr_frags); + + skb->len -= shiftlen; + skb->data_len -= shiftlen; + skb->truesize -= shiftlen; + tgt->len += shiftlen; + tgt->data_len += shiftlen; + tgt->truesize += shiftlen; + + return shiftlen; +} + +/* + * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb + * received has partial end part of the record, send out the complete record, so + * that crypto block will be able to generate TAG/HASH. + * @skb - segment which has complete or partial end part. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_push - tcp push bit. + * @mss - segment size. + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb, + struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, u32 tcp_seq, + bool tcp_push, u32 mss) +{ + u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start; + struct adapter *adap = tx_info->adap; + int credits, left, last_desc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct cpl_tx_sec_pdu *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + void *pos; + u64 *end; + + /* get the number of flits required */ + flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len); + /* number of descriptors */ + ndesc = chcr_flits_to_desc(flits); + /* check if enough credits available */ + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + /* Credits are below the threshold vaues, stop the queue after + * injecting the Work Request for this packet. + */ + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + pos += sizeof(*wr); + /* ULP_TXPKT */ + ulptx = pos; + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | + ULP_TXPKT_RO_F); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + /* idata length will include cpl_tx_sec_pdu + key context size + + * cpl_tx_data header. + */ + idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + + sizeof(*tx_data)); + /* SEC CPL */ + cpl = (struct cpl_tx_sec_pdu *)(idata + 1); + cpl->op_ivinsrtofst = + htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | + CPL_TX_SEC_PDU_PLACEHOLDER_V(1) | + CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1)); + cpl->pldlen = htonl(skb->data_len); + + /* encryption should start after tls header size + iv size */ + cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1; + + cpl->aadstart_cipherstop_hi = + htonl(CPL_TX_SEC_PDU_AADSTART_V(1) | + CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) | + CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); + + /* authentication will also start after tls header + iv size */ + cpl->cipherstop_lo_authinsert = + htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) | + CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) | + CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE)); + + /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ + cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs); + cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen); + cpl->scmd1 = cpu_to_be64(tx_info->record_no); + + pos = cpl + 1; + /* check if space left to fill the keys */ + left = (void *)q->q.stat - pos; + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, + tx_info->key_ctx_len); + left = (void *)q->q.stat - pos; + + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* CPL_TX_DATA */ + tx_data = (void *)pos; + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(skb->data_len)); + + tx_data->rsvd = htonl(tcp_seq); + + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + /* check left again, it might go beyond queue limit */ + pos = tx_data + 1; + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + /* send the complete packet except the header */ + cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, + sgl_sdesc->addr); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); + atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records); + + return 0; +} + +/* + * chcr_ktls_xmit_wr_short: This is to send out partial records. If its + * a middle part of a record, fetch the prior data to make it 16 byte aligned + * and then only send it out. + * + * @skb - skb contains partial record.. + * @tx_info - driver specific tls info. + * @q - TX queue. + * @tcp_seq + * @tcp_push - tcp push bit. + * @mss - segment size. + * @tls_rec_offset - offset from start of the tls record. + * @perior_data - data before the current segment, required to make this record + * 16 byte aligned. + * @prior_data_len - prior_data length (less than 16) + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_xmit_wr_short(struct sk_buff *skb, + struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q, + u32 tcp_seq, bool tcp_push, u32 mss, + u32 tls_rec_offset, u8 *prior_data, + u32 prior_data_len) +{ + struct adapter *adap = tx_info->adap; + u32 len16, wr_mid = 0, cipher_start; + unsigned int flits = 0, ndesc; + int credits, left, last_desc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct cpl_tx_sec_pdu *cpl; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + __be64 iv_record; + void *pos; + u64 *end; + + /* get the number of flits required, it's a partial record so 2 flits + * (AES_BLOCK_SIZE) will be added. + */ + flits = chcr_ktls_get_tx_flits(skb, tx_info->key_ctx_len) + 2; + /* get the correct 8 byte IV of this record */ + iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no); + /* If it's a middle record and not 16 byte aligned to run AES CTR, need + * to make it 16 byte aligned. So atleadt 2 extra flits of immediate + * data will be added. + */ + if (prior_data_len) + flits += 2; + /* number of descriptors */ + ndesc = chcr_flits_to_desc(flits); + /* check if enough credits available */ + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + pos += sizeof(*wr); + /* ULP_TXPKT */ + ulptx = pos; + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | + ULP_TXPKT_RO_F); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + /* idata length will include cpl_tx_sec_pdu + key context size + + * cpl_tx_data header. + */ + idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len + + sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len); + /* SEC CPL */ + cpl = (struct cpl_tx_sec_pdu *)(idata + 1); + /* cipher start will have tls header + iv size extra if its a header + * part of tls record. else only 16 byte IV will be added. + */ + cipher_start = + AES_BLOCK_LEN + 1 + + (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0); + + cpl->op_ivinsrtofst = + htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) | + CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) | + CPL_TX_SEC_PDU_IVINSRTOFST_V(1)); + cpl->pldlen = htonl(skb->data_len + AES_BLOCK_LEN + prior_data_len); + cpl->aadstart_cipherstop_hi = + htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start)); + cpl->cipherstop_lo_authinsert = 0; + /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */ + cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs); + cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen); + cpl->scmd1 = 0; + + pos = cpl + 1; + /* check if space left to fill the keys */ + left = (void *)q->q.stat - pos; + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + + pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos, + tx_info->key_ctx_len); + left = (void *)q->q.stat - pos; + + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* CPL_TX_DATA */ + tx_data = (void *)pos; + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | + TX_LENGTH_V(skb->data_len + prior_data_len)); + tx_data->rsvd = htonl(tcp_seq); + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + /* check left again, it might go beyond queue limit */ + pos = tx_data + 1; + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8 + * bytes of actual IV and 4 bytes of 16 byte-sequence. + */ + memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size); + memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size); + *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) = + htonl(2 + (tls_rec_offset ? ((tls_rec_offset - + (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0)); + + pos += 16; + /* Prior_data_len will always be less than 16 bytes, fill the + * prio_data_len after AES_CTRL_BLOCK and clear the remaining length + * to 0. + */ + if (prior_data_len) + pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); + /* send the complete packet except the header */ + cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, + sgl_sdesc->addr); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(adap, &q->q, ndesc); + + return 0; +} + +/* + * chcr_ktls_tx_plaintxt: This handler will take care of the records which has + * only plain text (only tls header and iv) + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @tcp_seq + * @mss - segment size. + * @tcp_push - tcp push bit. + * @q - TX queue. + * @port_id : port number + * @perior_data - data before the current segment, required to make this record + * 16 byte aligned. + * @prior_data_len - prior_data length (less than 16) + * return: NETDEV_TX_BUSY/NET_TX_OK. + */ +static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, u32 tcp_seq, u32 mss, + bool tcp_push, struct sge_eth_txq *q, + u32 port_id, u8 *prior_data, + u32 prior_data_len) +{ + int credits, left, len16, last_desc; + unsigned int flits = 0, ndesc; + struct tx_sw_desc *sgl_sdesc; + struct cpl_tx_data *tx_data; + struct ulptx_idata *idata; + struct ulp_txpkt *ulptx; + struct fw_ulptx_wr *wr; + u32 wr_mid = 0; + void *pos; + u64 *end; + + flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8); + flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags); + if (prior_data_len) + flits += 2; + /* WR will need len16 */ + len16 = DIV_ROUND_UP(flits, 2); + /* check how many descriptors needed */ + ndesc = DIV_ROUND_UP(flits, 8); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + if (unlikely(credits < ETHTXQ_STOP_THRES)) { + chcr_eth_txq_stop(q); + wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F; + } + + last_desc = q->q.pidx + ndesc - 1; + if (last_desc >= q->q.size) + last_desc -= q->q.size; + sgl_sdesc = &q->q.sdesc[last_desc]; + + if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb, + sgl_sdesc->addr) < 0)) { + memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr)); + q->mapping_err++; + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + end = (u64 *)pos + flits; + /* FW_ULPTX_WR */ + wr = pos; + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16)); + wr->cookie = 0; + pos += sizeof(*wr); + /* ULP_TXPKT */ + ulptx = (struct ulp_txpkt *)(wr + 1); + ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) | + ULP_TXPKT_DATAMODIFY_V(0) | + ULP_TXPKT_CHANNELID_V(tx_info->port_id) | + ULP_TXPKT_DEST_V(0) | + ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1)); + ulptx->len = htonl(len16 - 1); + /* ULPTX_IDATA sub-command */ + idata = (struct ulptx_idata *)(ulptx + 1); + idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F); + idata->len = htonl(sizeof(*tx_data) + prior_data_len); + /* CPL_TX_DATA */ + tx_data = (struct cpl_tx_data *)(idata + 1); + OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid)); + tx_data->len = htonl(TX_DATA_MSS_V(mss) | + TX_LENGTH_V(skb->data_len + prior_data_len)); + /* set tcp seq number */ + tx_data->rsvd = htonl(tcp_seq); + tx_data->flags = htonl(TX_BYPASS_F); + if (tcp_push) + tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F); + + pos = tx_data + 1; + /* apart from prior_data_len, we should set remaining part of 16 bytes + * to be zero. + */ + if (prior_data_len) + pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16); + + /* check left again, it might go beyond queue limit */ + left = (void *)q->q.stat - pos; + + /* check the position again */ + if (!left) { + left = (void *)end - (void *)q->q.stat; + pos = q->q.desc; + end = pos + left; + } + /* send the complete packet including the header */ + cxgb4_write_sgl(skb, &q->q, pos, end, skb->len - skb->data_len, + sgl_sdesc->addr); + sgl_sdesc->skb = skb; + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + return 0; +} + +/* + * chcr_ktls_copy_record_in_skb + * @nskb - new skb where the frags to be added. + * @record - specific record which has complete 16k record in frags. + */ +static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb, + struct tls_record_info *record) +{ + int i = 0; + + for (i = 0; i < record->num_frags; i++) { + skb_shinfo(nskb)->frags[i] = record->frags[i]; + /* increase the frag ref count */ + __skb_frag_ref(&skb_shinfo(nskb)->frags[i]); + } + + skb_shinfo(nskb)->nr_frags = record->num_frags; + nskb->data_len = record->len; + nskb->len += record->len; + nskb->truesize += record->len; +} + +/* + * chcr_ktls_update_snd_una: Reset the SEND_UNA. It will be done to avoid + * sending the same segment again. It will discard the segment which is before + * the current tx max. + * @tx_info - driver specific tls info. + * @q - TX queue. + * return: NET_TX_OK/NET_XMIT_DROP. + */ +static int chcr_ktls_update_snd_una(struct chcr_ktls_info *tx_info, + struct sge_eth_txq *q) +{ + struct fw_ulptx_wr *wr; + unsigned int ndesc; + int credits; + void *pos; + u32 len; + + len = sizeof(*wr) + roundup(CHCR_SET_TCB_FIELD_LEN, 16); + ndesc = DIV_ROUND_UP(len, 64); + + credits = chcr_txq_avail(&q->q) - ndesc; + if (unlikely(credits < 0)) { + chcr_eth_txq_stop(q); + return NETDEV_TX_BUSY; + } + + pos = &q->q.desc[q->q.pidx]; + + wr = pos; + /* ULPTX wr */ + wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR)); + wr->cookie = 0; + /* fill len in wr field */ + wr->flowid_len16 = htonl(FW_WR_LEN16_V(DIV_ROUND_UP(len, 16))); + + pos += sizeof(*wr); + + pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos, + TCB_SND_UNA_RAW_W, + TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M), + TCB_SND_UNA_RAW_V(0), 0); + + chcr_txq_advance(&q->q, ndesc); + cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc); + + return 0; +} + +/* + * chcr_end_part_handler: This handler will handle the record which + * is complete or if record's end part is received. T6 adapter has a issue that + * it can't send out TAG with partial record so if its an end part then we have + * to send TAG as well and for which we need to fetch the complete record and + * send it to crypto module. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record. + * @record - complete record of 16K size. + * @tcp_seq + * @mss - segment size in which TP needs to chop a packet. + * @tcp_push_no_fin - tcp push if fin is not set. + * @q - TX queue. + * @tls_end_offset - offset from end of the record. + * @last wr : check if this is the last part of the skb going out. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int chcr_end_part_handler(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, + struct tls_record_info *record, + u32 tcp_seq, int mss, bool tcp_push_no_fin, + struct sge_eth_txq *q, + u32 tls_end_offset, bool last_wr) +{ + struct sk_buff *nskb = NULL; + /* check if it is a complete record */ + if (tls_end_offset == record->len) { + nskb = skb; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts); + } else { + dev_kfree_skb_any(skb); + + nskb = alloc_skb(0, GFP_KERNEL); + if (!nskb) + return NETDEV_TX_BUSY; + /* copy complete record in skb */ + chcr_ktls_copy_record_in_skb(nskb, record); + /* packet is being sent from the beginning, update the tcp_seq + * accordingly. + */ + tcp_seq = tls_record_start_seq(record); + /* reset snd una, so the middle record won't send the already + * sent part. + */ + if (chcr_ktls_update_snd_una(tx_info, q)) + goto out; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts); + } + + if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq, + (last_wr && tcp_push_no_fin), + mss)) { + goto out; + } + return 0; +out: + dev_kfree_skb_any(nskb); + return NETDEV_TX_BUSY; +} + +/* + * chcr_short_record_handler: This handler will take care of the records which + * doesn't have end part (1st part or the middle part(/s) of a record). In such + * cases, AES CTR will be used in place of AES GCM to send out partial packet. + * This partial record might be the first part of the record, or the middle + * part. In case of middle record we should fetch the prior data to make it 16 + * byte aligned. If it has a partial tls header or iv then get to the start of + * tls header. And if it has partial TAG, then remove the complete TAG and send + * only the payload. + * There is one more possibility that it gets a partial header, send that + * portion as a plaintext. + * @tx_info - driver specific tls info. + * @skb - skb contains partial record.. + * @record - complete record of 16K size. + * @tcp_seq + * @mss - segment size in which TP needs to chop a packet. + * @tcp_push_no_fin - tcp push if fin is not set. + * @q - TX queue. + * @tls_end_offset - offset from end of the record. + * return: NETDEV_TX_OK/NETDEV_TX_BUSY. + */ +static int chcr_short_record_handler(struct chcr_ktls_info *tx_info, + struct sk_buff *skb, + struct tls_record_info *record, + u32 tcp_seq, int mss, bool tcp_push_no_fin, + struct sge_eth_txq *q, u32 tls_end_offset) +{ + u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record); + u8 prior_data[16] = {0}; + u32 prior_data_len = 0; + u32 data_len; + + /* check if the skb is ending in middle of tag/HASH, its a big + * trouble, send the packet before the HASH. + */ + int remaining_record = tls_end_offset - skb->data_len; + + if (remaining_record > 0 && + remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) { + int trimmed_len = skb->data_len - + (TLS_CIPHER_AES_GCM_128_TAG_SIZE - remaining_record); + struct sk_buff *tmp_skb = NULL; + /* don't process the pkt if it is only a partial tag */ + if (skb->data_len < TLS_CIPHER_AES_GCM_128_TAG_SIZE) + goto out; + + WARN_ON(trimmed_len > skb->data_len); + + /* shift to those many bytes */ + tmp_skb = alloc_skb(0, GFP_KERNEL); + if (unlikely(!tmp_skb)) + goto out; + + chcr_ktls_skb_shift(tmp_skb, skb, trimmed_len); + /* free the last trimmed portion */ + dev_kfree_skb_any(skb); + skb = tmp_skb; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts); + } + data_len = skb->data_len; + /* check if the middle record's start point is 16 byte aligned. CTR + * needs 16 byte aligned start point to start encryption. + */ + if (tls_rec_offset) { + /* there is an offset from start, means its a middle record */ + int remaining = 0; + + if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) { + prior_data_len = tls_rec_offset; + tls_rec_offset = 0; + remaining = 0; + } else { + prior_data_len = + (tls_rec_offset - + (TLS_HEADER_SIZE + tx_info->iv_size)) + % AES_BLOCK_LEN; + remaining = tls_rec_offset - prior_data_len; + } + + /* if prior_data_len is not zero, means we need to fetch prior + * data to make this record 16 byte aligned, or we need to reach + * to start offset. + */ + if (prior_data_len) { + int i = 0; + u8 *data = NULL; + skb_frag_t *f; + u8 *vaddr; + int frag_size = 0, frag_delta = 0; + + while (remaining > 0) { + frag_size = skb_frag_size(&record->frags[i]); + if (remaining < frag_size) + break; + + remaining -= frag_size; + i++; + } + f = &record->frags[i]; + vaddr = kmap_atomic(skb_frag_page(f)); + + data = vaddr + skb_frag_off(f) + remaining; + frag_delta = skb_frag_size(f) - remaining; + + if (frag_delta >= prior_data_len) { + memcpy(prior_data, data, prior_data_len); + kunmap_atomic(vaddr); + } else { + memcpy(prior_data, data, frag_delta); + kunmap_atomic(vaddr); + /* get the next page */ + f = &record->frags[i + 1]; + vaddr = kmap_atomic(skb_frag_page(f)); + data = vaddr + skb_frag_off(f); + memcpy(prior_data + frag_delta, + data, (prior_data_len - frag_delta)); + kunmap_atomic(vaddr); + } + /* reset tcp_seq as per the prior_data_required len */ + tcp_seq -= prior_data_len; + /* include prio_data_len for further calculation. + */ + data_len += prior_data_len; + } + /* reset snd una, so the middle record won't send the already + * sent part. + */ + if (chcr_ktls_update_snd_una(tx_info, q)) + goto out; + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts); + } else { + /* Else means, its a partial first part of the record. Check if + * its only the header, don't need to send for encryption then. + */ + if (data_len <= TLS_HEADER_SIZE + tx_info->iv_size) { + if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss, + tcp_push_no_fin, q, + tx_info->port_id, + prior_data, + prior_data_len)) { + goto out; + } + return 0; + } + atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts); + } + + if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin, + mss, tls_rec_offset, prior_data, + prior_data_len)) { + goto out; + } + + return 0; +out: + dev_kfree_skb_any(skb); + return NETDEV_TX_BUSY; +} + +/* nic tls TX handler */ +static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev) +{ + struct chcr_ktls_ofld_ctx_tx *tx_ctx; + struct ch_ktls_stats_debug *stats; + struct tcphdr *th = tcp_hdr(skb); + int data_len, qidx, ret = 0, mss; + struct tls_record_info *record; + struct chcr_ktls_info *tx_info; + u32 tls_end_offset, tcp_seq; + struct tls_context *tls_ctx; + struct sk_buff *local_skb; + int new_connection_state; + struct sge_eth_txq *q; + struct adapter *adap; + unsigned long flags; + + tcp_seq = ntohl(th->seq); + + mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : skb->data_len; + + /* check if we haven't set it for ktls offload */ + if (!skb->sk || !tls_is_sk_tx_device_offloaded(skb->sk)) + goto out; + + tls_ctx = tls_get_ctx(skb->sk); + if (unlikely(tls_ctx->netdev != dev)) + goto out; + + tx_ctx = chcr_get_ktls_tx_context(tls_ctx); + tx_info = tx_ctx->chcr_info; + + if (unlikely(!tx_info)) + goto out; + + /* check the connection state, we don't need to pass new connection + * state, state machine will check and update the new state if it is + * stuck due to responses not received from HW. + * Start the tx handling only if state is KTLS_CONN_TX_READY. + */ + new_connection_state = chcr_ktls_update_connection_state(tx_info, 0); + if (new_connection_state != KTLS_CONN_TX_READY) + goto out; + + /* don't touch the original skb, make a new skb to extract each records + * and send them separately. + */ + local_skb = alloc_skb(0, GFP_KERNEL); + + if (unlikely(!local_skb)) + return NETDEV_TX_BUSY; + + adap = tx_info->adap; + stats = &adap->ch_ktls_stats; + + qidx = skb->queue_mapping; + q = &adap->sge.ethtxq[qidx + tx_info->first_qset]; + cxgb4_reclaim_completed_tx(adap, &q->q, true); + /* if tcp options are set but finish is not send the options first */ + if (!th->fin && chcr_ktls_check_tcp_options(th)) { + ret = chcr_ktls_write_tcp_options(tx_info, skb, q, + tx_info->tx_chan); + if (ret) + return NETDEV_TX_BUSY; + } + /* update tcb */ + ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, ntohl(th->seq), + ntohl(th->ack_seq), + ntohs(th->window)); + if (ret) { + dev_kfree_skb_any(local_skb); + return NETDEV_TX_BUSY; + } + + /* copy skb contents into local skb */ + chcr_ktls_skb_copy(skb, local_skb); + + /* go through the skb and send only one record at a time. */ + data_len = skb->data_len; + /* TCP segments can be in received either complete or partial. + * chcr_end_part_handler will handle cases if complete record or end + * part of the record is received. Incase of partial end part of record, + * we will send the complete record again. + */ + + do { + int i; + + cxgb4_reclaim_completed_tx(adap, &q->q, true); + /* lock taken */ + spin_lock_irqsave(&tx_ctx->base.lock, flags); + /* fetch the tls record */ + record = tls_get_record(&tx_ctx->base, tcp_seq, + &tx_info->record_no); + /* By the time packet reached to us, ACK is received, and record + * won't be found in that case, handle it gracefully. + */ + if (unlikely(!record)) { + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + atomic64_inc(&stats->ktls_tx_drop_no_sync_data); + goto out; + } + + if (unlikely(tls_record_is_start_marker(record))) { + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + atomic64_inc(&stats->ktls_tx_skip_no_sync_data); + goto out; + } + + /* increase page reference count of the record, so that there + * won't be any chance of page free in middle if in case stack + * receives ACK and try to delete the record. + */ + for (i = 0; i < record->num_frags; i++) + __skb_frag_ref(&record->frags[i]); + /* lock cleared */ + spin_unlock_irqrestore(&tx_ctx->base.lock, flags); + + tls_end_offset = record->end_seq - tcp_seq; + + pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n", + tcp_seq, record->end_seq, tx_info->prev_seq, data_len); + /* if a tls record is finishing in this SKB */ + if (tls_end_offset <= data_len) { + struct sk_buff *nskb = NULL; + + if (tls_end_offset < data_len) { + nskb = alloc_skb(0, GFP_KERNEL); + if (unlikely(!nskb)) { + ret = -ENOMEM; + goto clear_ref; + } + + chcr_ktls_skb_shift(nskb, local_skb, + tls_end_offset); + } else { + /* its the only record in this skb, directly + * point it. + */ + nskb = local_skb; + } + ret = chcr_end_part_handler(tx_info, nskb, record, + tcp_seq, mss, + (!th->fin && th->psh), q, + tls_end_offset, + (nskb == local_skb)); + + if (ret && nskb != local_skb) + dev_kfree_skb_any(local_skb); + + data_len -= tls_end_offset; + /* tcp_seq increment is required to handle next record. + */ + tcp_seq += tls_end_offset; + } else { + ret = chcr_short_record_handler(tx_info, local_skb, + record, tcp_seq, mss, + (!th->fin && th->psh), + q, tls_end_offset); + data_len = 0; + } +clear_ref: + /* clear the frag ref count which increased locally before */ + for (i = 0; i < record->num_frags; i++) { + /* clear the frag ref count */ + __skb_frag_unref(&record->frags[i]); + } + /* if any failure, come out from the loop. */ + if (ret) + goto out; + /* length should never be less than 0 */ + WARN_ON(data_len < 0); + + } while (data_len > 0); + + tx_info->prev_seq = ntohl(th->seq) + skb->data_len; + + atomic64_inc(&stats->ktls_tx_encrypted_packets); + atomic64_add(skb->data_len, &stats->ktls_tx_encrypted_bytes); + + /* tcp finish is set, send a separate tcp msg including all the options + * as well. + */ + if (th->fin) + chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan); + +out: + dev_kfree_skb_any(skb); + return NETDEV_TX_OK; +} + +static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi) +{ + struct chcr_ktls_uld_ctx *u_ctx; + + pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC, + CHCR_KTLS_DRV_VERSION); + u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL); + if (!u_ctx) { + u_ctx = ERR_PTR(-ENOMEM); + goto out; + } + u_ctx->lldi = *lldi; +out: + return u_ctx; +} + +static const struct tlsdev_ops chcr_ktls_ops = { + .tls_dev_add = chcr_ktls_dev_add, + .tls_dev_del = chcr_ktls_dev_del, +}; + +static chcr_handler_func work_handlers[NUM_CPL_CMDS] = { + [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl, + [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl, +}; + +static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp, + const struct pkt_gl *pgl) +{ + const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp; + struct chcr_ktls_uld_ctx *u_ctx = handle; + u8 opcode = rpl->ot.opcode; + struct adapter *adap; + + adap = pci_get_drvdata(u_ctx->lldi.pdev); + + if (!work_handlers[opcode]) { + pr_err("Unsupported opcode %d received\n", opcode); + return 0; + } + + work_handlers[opcode](adap, (unsigned char *)&rsp[1]); + return 0; +} + +static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state) +{ + struct chcr_ktls_uld_ctx *u_ctx = handle; + + switch (new_state) { + case CXGB4_STATE_UP: + pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev)); + mutex_lock(&dev_mutex); + list_add_tail(&u_ctx->entry, &uld_ctx_list); + mutex_unlock(&dev_mutex); + break; + case CXGB4_STATE_START_RECOVERY: + case CXGB4_STATE_DOWN: + case CXGB4_STATE_DETACH: + pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev)); + mutex_lock(&dev_mutex); + list_del(&u_ctx->entry); + mutex_unlock(&dev_mutex); + break; + default: + break; + } + + return 0; +} + +static struct cxgb4_uld_info chcr_ktls_uld_info = { + .name = CHCR_KTLS_DRV_MODULE_NAME, + .nrxq = 1, + .rxq_size = 1024, + .add = chcr_ktls_uld_add, + .tx_handler = chcr_ktls_xmit, + .rx_handler = chcr_ktls_uld_rx_handler, + .state_change = chcr_ktls_uld_state_change, + .tlsdev_ops = &chcr_ktls_ops, +}; + +static int __init chcr_ktls_init(void) +{ + cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info); + return 0; +} + +static void __exit chcr_ktls_exit(void) +{ + struct chcr_ktls_uld_ctx *u_ctx, *tmp; + struct adapter *adap; + + pr_info("driver unloaded\n"); + + mutex_lock(&dev_mutex); + list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) { + adap = pci_get_drvdata(u_ctx->lldi.pdev); + memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats)); + list_del(&u_ctx->entry); + kfree(u_ctx); + } + mutex_unlock(&dev_mutex); + cxgb4_unregister_uld(CXGB4_ULD_KTLS); +} + +module_init(chcr_ktls_init); +module_exit(chcr_ktls_exit); + +MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver"); +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Chelsio Communications"); +MODULE_VERSION(CHCR_KTLS_DRV_VERSION); diff --git a/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h new file mode 100644 index 000000000000..4ae6ae38c406 --- /dev/null +++ b/drivers/net/ethernet/chelsio/inline_crypto/ch_ktls/chcr_ktls.h @@ -0,0 +1,101 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright (C) 2020 Chelsio Communications. All rights reserved. */ + +#ifndef __CHCR_KTLS_H__ +#define __CHCR_KTLS_H__ + +#include "cxgb4.h" +#include "t4_msg.h" +#include "t4_tcb.h" +#include "l2t.h" +#include "chcr_common.h" +#include "cxgb4_uld.h" +#include "clip_tbl.h" + +#define CHCR_KTLS_DRV_MODULE_NAME "ch_ktls" +#define CHCR_KTLS_DRV_VERSION "1.0.0.0-ko" +#define CHCR_KTLS_DRV_DESC "Chelsio NIC TLS ULD Driver" + +#define CHCR_TCB_STATE_CLOSED 0 +#define CHCR_KTLS_KEY_CTX_LEN 16 +#define CHCR_SET_TCB_FIELD_LEN sizeof(struct cpl_set_tcb_field) +#define CHCR_PLAIN_TX_DATA_LEN (sizeof(struct fw_ulptx_wr) +\ + sizeof(struct ulp_txpkt) +\ + sizeof(struct ulptx_idata) +\ + sizeof(struct cpl_tx_data)) + +#define CHCR_KTLS_WR_SIZE (CHCR_PLAIN_TX_DATA_LEN +\ + sizeof(struct cpl_tx_sec_pdu)) + +enum chcr_ktls_conn_state { + KTLS_CONN_CLOSED, + KTLS_CONN_ACT_OPEN_REQ, + KTLS_CONN_ACT_OPEN_RPL, + KTLS_CONN_SET_TCB_REQ, + KTLS_CONN_SET_TCB_RPL, + KTLS_CONN_TX_READY, +}; + +struct chcr_ktls_info { + struct sock *sk; + spinlock_t lock; /* state machine lock */ + struct ktls_key_ctx key_ctx; + struct adapter *adap; + struct l2t_entry *l2te; + struct net_device *netdev; + u64 iv; + u64 record_no; + int tid; + int atid; + int rx_qid; + u32 iv_size; + u32 prev_seq; + u32 prev_ack; + u32 salt_size; + u32 key_ctx_len; + u32 scmd0_seqno_numivs; + u32 scmd0_ivgen_hdrlen; + u32 tcp_start_seq_number; + u32 scmd0_short_seqno_numivs; + u32 scmd0_short_ivgen_hdrlen; + enum chcr_ktls_conn_state connection_state; + u16 prev_win; + u8 tx_chan; + u8 smt_idx; + u8 port_id; + u8 ip_family; + u8 first_qset; +}; + +struct chcr_ktls_ofld_ctx_tx { + struct tls_offload_context_tx base; + struct chcr_ktls_info *chcr_info; +}; + +struct chcr_ktls_uld_ctx { + struct list_head entry; + struct cxgb4_lld_info lldi; +}; + +static inline struct chcr_ktls_ofld_ctx_tx * +chcr_get_ktls_tx_context(struct tls_context *tls_ctx) +{ + BUILD_BUG_ON(sizeof(struct chcr_ktls_ofld_ctx_tx) > + TLS_OFFLOAD_CONTEXT_SIZE_TX); + return container_of(tls_offload_ctx_tx(tls_ctx), + struct chcr_ktls_ofld_ctx_tx, + base); +} + +static inline int chcr_get_first_rx_qid(struct adapter *adap) +{ + /* u_ctx is saved in adap, fetch it */ + struct chcr_ktls_uld_ctx *u_ctx = adap->uld[CXGB4_ULD_KTLS].handle; + + if (!u_ctx) + return -1; + return u_ctx->lldi.rxq_ids[0]; +} + +typedef int (*chcr_handler_func)(struct adapter *adap, unsigned char *input); +#endif /* __CHCR_KTLS_H__ */