1 // SPDX-License-Identifier: GPL-2.0-only
2 /****************************************************************************
3 * Driver for Solarflare network controllers and boards
4 * Copyright 2018 Solarflare Communications Inc.
5 * Copyright 2019-2020 Xilinx Inc.
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation, incorporated herein by reference.
12 #include <net/ip6_checksum.h>
14 #include "net_driver.h"
15 #include "tx_common.h"
16 #include "nic_common.h"
17 #include "mcdi_functions.h"
18 #include "ef100_regs.h"
21 #include "ef100_nic.h"
23 int ef100_tx_probe(struct efx_tx_queue *tx_queue)
25 /* Allocate an extra descriptor for the QMDA status completion entry */
26 return efx_nic_alloc_buffer(tx_queue->efx, &tx_queue->txd.buf,
27 (tx_queue->ptr_mask + 2) *
32 void ef100_tx_init(struct efx_tx_queue *tx_queue)
34 /* must be the inverse of lookup in efx_get_tx_channel */
36 netdev_get_tx_queue(tx_queue->efx->net_dev,
37 tx_queue->channel->channel -
38 tx_queue->efx->tx_channel_offset);
40 /* This value is purely documentational; as EF100 never passes through
41 * the switch statement in tx.c:__efx_enqueue_skb(), that switch does
42 * not handle case 3. EF100's TSOv3 descriptors are generated by
43 * ef100_make_tso_desc().
44 * Meanwhile, all efx_mcdi_tx_init() cares about is that it's not 2.
46 tx_queue->tso_version = 3;
47 if (efx_mcdi_tx_init(tx_queue))
48 netdev_WARN(tx_queue->efx->net_dev,
49 "failed to initialise TXQ %d\n", tx_queue->queue);
52 static bool ef100_tx_can_tso(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
54 struct efx_nic *efx = tx_queue->efx;
55 struct ef100_nic_data *nic_data;
56 struct efx_tx_buffer *buffer;
57 struct tcphdr *tcphdr;
62 nic_data = efx->nic_data;
64 if (!skb_is_gso_tcp(skb))
66 if (!(efx->net_dev->features & NETIF_F_TSO))
69 mss = skb_shinfo(skb)->gso_size;
70 if (unlikely(mss < 4)) {
71 WARN_ONCE(1, "MSS of %u is too small for TSO\n", mss);
75 header_len = efx_tx_tso_header_length(skb);
76 if (header_len > nic_data->tso_max_hdr_len)
79 if (skb_shinfo(skb)->gso_segs > nic_data->tso_max_payload_num_segs) {
80 /* net_dev->gso_max_segs should've caught this */
85 if (skb->data_len / mss > nic_data->tso_max_frames)
88 /* net_dev->gso_max_size should've caught this */
89 if (WARN_ON_ONCE(skb->data_len > nic_data->tso_max_payload_len))
92 /* Reserve an empty buffer for the TSO V3 descriptor.
93 * Convey the length of the header since we already know it.
95 buffer = efx_tx_queue_get_insert_buffer(tx_queue);
96 buffer->flags = EFX_TX_BUF_TSO_V3 | EFX_TX_BUF_CONT;
97 buffer->len = header_len;
98 buffer->unmap_len = 0;
100 ++tx_queue->insert_count;
102 /* Adjust the TCP checksum to exclude the total length, since we set
103 * ED_INNER_IP_LEN in the descriptor.
105 tcphdr = tcp_hdr(skb);
106 if (skb_is_gso_v6(skb)) {
107 tcphdr->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
108 &ipv6_hdr(skb)->daddr,
112 tcphdr->check = ~csum_tcpudp_magic(iphdr->saddr, iphdr->daddr,
118 static efx_oword_t *ef100_tx_desc(struct efx_tx_queue *tx_queue, unsigned int index)
120 if (likely(tx_queue->txd.buf.addr))
121 return ((efx_oword_t *)tx_queue->txd.buf.addr) + index;
126 static void ef100_notify_tx_desc(struct efx_tx_queue *tx_queue)
128 unsigned int write_ptr;
131 tx_queue->xmit_pending = false;
133 if (unlikely(tx_queue->notify_count == tx_queue->write_count))
136 write_ptr = tx_queue->write_count & tx_queue->ptr_mask;
137 /* The write pointer goes into the high word */
138 EFX_POPULATE_DWORD_1(reg, ERF_GZ_TX_RING_PIDX, write_ptr);
139 efx_writed_page(tx_queue->efx, ®,
140 ER_GZ_TX_RING_DOORBELL, tx_queue->queue);
141 tx_queue->notify_count = tx_queue->write_count;
144 static void ef100_tx_push_buffers(struct efx_tx_queue *tx_queue)
146 ef100_notify_tx_desc(tx_queue);
150 static void ef100_set_tx_csum_partial(const struct sk_buff *skb,
151 struct efx_tx_buffer *buffer, efx_oword_t *txd)
156 if (!skb || skb->ip_summed != CHECKSUM_PARTIAL)
159 /* skb->csum_start has the offset from head, but we need the offset
162 csum_start = skb_checksum_start_offset(skb);
163 EFX_POPULATE_OWORD_3(csum,
164 ESF_GZ_TX_SEND_CSO_PARTIAL_EN, 1,
165 ESF_GZ_TX_SEND_CSO_PARTIAL_START_W,
167 ESF_GZ_TX_SEND_CSO_PARTIAL_CSUM_W,
168 skb->csum_offset >> 1);
169 EFX_OR_OWORD(*txd, *txd, csum);
172 static void ef100_set_tx_hw_vlan(const struct sk_buff *skb, efx_oword_t *txd)
174 u16 vlan_tci = skb_vlan_tag_get(skb);
177 EFX_POPULATE_OWORD_2(vlan,
178 ESF_GZ_TX_SEND_VLAN_INSERT_EN, 1,
179 ESF_GZ_TX_SEND_VLAN_INSERT_TCI, vlan_tci);
180 EFX_OR_OWORD(*txd, *txd, vlan);
183 static void ef100_make_send_desc(struct efx_nic *efx,
184 const struct sk_buff *skb,
185 struct efx_tx_buffer *buffer, efx_oword_t *txd,
186 unsigned int segment_count)
188 /* TX send descriptor */
189 EFX_POPULATE_OWORD_3(*txd,
190 ESF_GZ_TX_SEND_NUM_SEGS, segment_count,
191 ESF_GZ_TX_SEND_LEN, buffer->len,
192 ESF_GZ_TX_SEND_ADDR, buffer->dma_addr);
194 if (likely(efx->net_dev->features & NETIF_F_HW_CSUM))
195 ef100_set_tx_csum_partial(skb, buffer, txd);
196 if (efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX &&
197 skb && skb_vlan_tag_present(skb))
198 ef100_set_tx_hw_vlan(skb, txd);
201 static void ef100_make_tso_desc(struct efx_nic *efx,
202 const struct sk_buff *skb,
203 struct efx_tx_buffer *buffer, efx_oword_t *txd,
204 unsigned int segment_count)
206 u32 mangleid = (efx->net_dev->features & NETIF_F_TSO_MANGLEID) ||
207 skb_shinfo(skb)->gso_type & SKB_GSO_TCP_FIXEDID ?
208 ESE_GZ_TX_DESC_IP4_ID_NO_OP :
209 ESE_GZ_TX_DESC_IP4_ID_INC_MOD16;
210 u16 vlan_enable = efx->net_dev->features & NETIF_F_HW_VLAN_CTAG_TX ?
211 skb_vlan_tag_present(skb) : 0;
212 unsigned int len, ip_offset, tcp_offset, payload_segs;
213 u16 vlan_tci = skb_vlan_tag_get(skb);
214 u32 mss = skb_shinfo(skb)->gso_size;
216 len = skb->len - buffer->len;
217 /* We use 1 for the TSO descriptor and 1 for the header */
218 payload_segs = segment_count - 2;
219 ip_offset = skb_network_offset(skb);
220 tcp_offset = skb_transport_offset(skb);
222 EFX_POPULATE_OWORD_13(*txd,
223 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_TSO,
224 ESF_GZ_TX_TSO_MSS, mss,
225 ESF_GZ_TX_TSO_HDR_NUM_SEGS, 1,
226 ESF_GZ_TX_TSO_PAYLOAD_NUM_SEGS, payload_segs,
227 ESF_GZ_TX_TSO_HDR_LEN_W, buffer->len >> 1,
228 ESF_GZ_TX_TSO_PAYLOAD_LEN, len,
229 ESF_GZ_TX_TSO_CSO_INNER_L4, 1,
230 ESF_GZ_TX_TSO_INNER_L3_OFF_W, ip_offset >> 1,
231 ESF_GZ_TX_TSO_INNER_L4_OFF_W, tcp_offset >> 1,
232 ESF_GZ_TX_TSO_ED_INNER_IP4_ID, mangleid,
233 ESF_GZ_TX_TSO_ED_INNER_IP_LEN, 1,
234 ESF_GZ_TX_TSO_VLAN_INSERT_EN, vlan_enable,
235 ESF_GZ_TX_TSO_VLAN_INSERT_TCI, vlan_tci
239 static void ef100_tx_make_descriptors(struct efx_tx_queue *tx_queue,
240 const struct sk_buff *skb,
241 unsigned int segment_count)
243 unsigned int old_write_count = tx_queue->write_count;
244 unsigned int new_write_count = old_write_count;
245 struct efx_tx_buffer *buffer;
246 unsigned int next_desc_type;
247 unsigned int write_ptr;
249 unsigned int nr_descs = tx_queue->insert_count - old_write_count;
251 if (unlikely(nr_descs == 0))
255 next_desc_type = ESE_GZ_TX_DESC_TYPE_TSO;
257 next_desc_type = ESE_GZ_TX_DESC_TYPE_SEND;
259 /* if it's a raw write (such as XDP) then always SEND single frames */
264 write_ptr = new_write_count & tx_queue->ptr_mask;
265 buffer = &tx_queue->buffer[write_ptr];
266 txd = ef100_tx_desc(tx_queue, write_ptr);
269 /* Create TX descriptor ring entry */
270 tx_queue->packet_write_count = new_write_count;
272 switch (next_desc_type) {
273 case ESE_GZ_TX_DESC_TYPE_SEND:
274 ef100_make_send_desc(tx_queue->efx, skb,
275 buffer, txd, nr_descs);
277 case ESE_GZ_TX_DESC_TYPE_TSO:
278 /* TX TSO descriptor */
279 WARN_ON_ONCE(!(buffer->flags & EFX_TX_BUF_TSO_V3));
280 ef100_make_tso_desc(tx_queue->efx, skb,
281 buffer, txd, nr_descs);
284 /* TX segment descriptor */
285 EFX_POPULATE_OWORD_3(*txd,
286 ESF_GZ_TX_DESC_TYPE, ESE_GZ_TX_DESC_TYPE_SEG,
287 ESF_GZ_TX_SEG_LEN, buffer->len,
288 ESF_GZ_TX_SEG_ADDR, buffer->dma_addr);
290 /* if it's a raw write (such as XDP) then always SEND */
291 next_desc_type = skb ? ESE_GZ_TX_DESC_TYPE_SEG :
292 ESE_GZ_TX_DESC_TYPE_SEND;
294 } while (new_write_count != tx_queue->insert_count);
296 wmb(); /* Ensure descriptors are written before they are fetched */
298 tx_queue->write_count = new_write_count;
300 /* The write_count above must be updated before reading
301 * channel->holdoff_doorbell to avoid a race with the
302 * completion path, so ensure these operations are not
303 * re-ordered. This also flushes the update of write_count
304 * back into the cache.
309 void ef100_tx_write(struct efx_tx_queue *tx_queue)
311 ef100_tx_make_descriptors(tx_queue, NULL, 0);
312 ef100_tx_push_buffers(tx_queue);
315 void ef100_ev_tx(struct efx_channel *channel, const efx_qword_t *p_event)
317 unsigned int tx_done =
318 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_NUM_DESC);
319 unsigned int qlabel =
320 EFX_QWORD_FIELD(*p_event, ESF_GZ_EV_TXCMPL_Q_LABEL);
321 struct efx_tx_queue *tx_queue =
322 efx_channel_get_tx_queue(channel, qlabel);
323 unsigned int tx_index = (tx_queue->read_count + tx_done - 1) &
326 efx_xmit_done(tx_queue, tx_index);
329 /* Add a socket buffer to a TX queue
331 * You must hold netif_tx_lock() to call this function.
333 * Returns 0 on success, error code otherwise. In case of an error this
334 * function will free the SKB.
336 int ef100_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
338 unsigned int old_insert_count = tx_queue->insert_count;
339 struct efx_nic *efx = tx_queue->efx;
340 bool xmit_more = netdev_xmit_more();
341 unsigned int fill_level;
342 unsigned int segments;
345 if (!tx_queue->buffer || !tx_queue->ptr_mask) {
346 netif_stop_queue(efx->net_dev);
347 dev_kfree_skb_any(skb);
351 segments = skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 0;
353 segments = 0; /* Don't use TSO/GSO for a single segment. */
354 if (segments && !ef100_tx_can_tso(tx_queue, skb)) {
355 rc = efx_tx_tso_fallback(tx_queue, skb);
356 tx_queue->tso_fallbacks++;
363 /* Map for DMA and create descriptors */
364 rc = efx_tx_map_data(tx_queue, skb, segments);
367 ef100_tx_make_descriptors(tx_queue, skb, segments);
369 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
370 if (fill_level > efx->txq_stop_thresh) {
371 struct efx_tx_queue *txq2;
373 netif_tx_stop_queue(tx_queue->core_txq);
374 /* Re-read after a memory barrier in case we've raced with
375 * the completion path. Otherwise there's a danger we'll never
376 * restart the queue if all completions have just happened.
379 efx_for_each_channel_tx_queue(txq2, tx_queue->channel)
380 txq2->old_read_count = READ_ONCE(txq2->read_count);
381 fill_level = efx_channel_tx_old_fill_level(tx_queue->channel);
382 if (fill_level < efx->txq_stop_thresh)
383 netif_tx_start_queue(tx_queue->core_txq);
386 tx_queue->xmit_pending = true;
388 /* If xmit_more then we don't need to push the doorbell, unless there
389 * are 256 descriptors already queued in which case we have to push to
390 * ensure we never push more than 256 at once.
392 if (__netdev_tx_sent_queue(tx_queue->core_txq, skb->len, xmit_more) ||
393 tx_queue->write_count - tx_queue->notify_count > 255)
394 ef100_tx_push_buffers(tx_queue);
397 tx_queue->tso_bursts++;
398 tx_queue->tso_packets += segments;
399 tx_queue->tx_packets += segments;
401 tx_queue->tx_packets++;
406 efx_enqueue_unwind(tx_queue, old_insert_count);
407 if (!IS_ERR_OR_NULL(skb))
408 dev_kfree_skb_any(skb);
410 /* If we're not expecting another transmit and we had something to push
411 * on this queue then we need to push here to get the previous packets
412 * out. We only enter this branch from before the xmit_more handling
413 * above, so xmit_pending still refers to the old state.
415 if (tx_queue->xmit_pending && !xmit_more)
416 ef100_tx_push_buffers(tx_queue);