2 * Copyright (C) 2015-2017 Netronome Systems, Inc.
4 * This software is dual licensed under the GNU General License Version 2,
5 * June 1991 as shown in the file COPYING in the top-level directory of this
6 * source tree or the BSD 2-Clause License provided below. You have the
7 * option to license this software under the complete terms of either license.
9 * The BSD 2-Clause License:
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
15 * 1. Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
19 * 2. Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
36 * Authors: Jakub Kicinski <jakub.kicinski@netronome.com>
37 * Jason McMullan <jason.mcmullan@netronome.com>
38 * Rolf Neugebauer <rolf.neugebauer@netronome.com>
40 * Multiplexes the NFP BARs between NFP internal resources and
41 * implements the PCIe specific interface for generic CPP bus access.
43 * The BARs are managed with refcounts and are allocated/acquired
44 * using target, token and offset/size matching. The generic CPP bus
45 * abstraction builds upon this BAR interface.
48 #include <asm/unaligned.h>
49 #include <linux/kernel.h>
50 #include <linux/module.h>
51 #include <linux/kref.h>
53 #include <linux/delay.h>
54 #include <linux/interrupt.h>
55 #include <linux/sort.h>
56 #include <linux/sched.h>
57 #include <linux/types.h>
58 #include <linux/pci.h>
62 #include "nfp6000/nfp6000.h"
64 #include "nfp6000_pcie.h"
66 #define NFP_PCIE_BAR(_pf) (0x30000 + ((_pf) & 7) * 0xc0)
67 #define NFP_PCIE_BAR_EXPLICIT_BAR0(_x, _y) \
68 (0x00000080 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
69 #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(_x) (((_x) & 0x3) << 30)
70 #define NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType_of(_x) (((_x) >> 30) & 0x3)
71 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token(_x) (((_x) & 0x3) << 28)
72 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Token_of(_x) (((_x) >> 28) & 0x3)
73 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address(_x) (((_x) & 0xffffff) << 0)
74 #define NFP_PCIE_BAR_EXPLICIT_BAR0_Address_of(_x) (((_x) >> 0) & 0xffffff)
75 #define NFP_PCIE_BAR_EXPLICIT_BAR1(_x, _y) \
76 (0x00000084 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
77 #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(_x) (((_x) & 0x7f) << 24)
78 #define NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef_of(_x) (((_x) >> 24) & 0x7f)
79 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(_x) (((_x) & 0x3ff) << 14)
80 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster_of(_x) (((_x) >> 14) & 0x3ff)
81 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(_x) (((_x) & 0x3fff) << 0)
82 #define NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef_of(_x) (((_x) >> 0) & 0x3fff)
83 #define NFP_PCIE_BAR_EXPLICIT_BAR2(_x, _y) \
84 (0x00000088 + (0x40 * ((_x) & 0x3)) + (0x10 * ((_y) & 0x3)))
85 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target(_x) (((_x) & 0xf) << 28)
86 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Target_of(_x) (((_x) >> 28) & 0xf)
87 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action(_x) (((_x) & 0x1f) << 23)
88 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Action_of(_x) (((_x) >> 23) & 0x1f)
89 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length(_x) (((_x) & 0x1f) << 18)
90 #define NFP_PCIE_BAR_EXPLICIT_BAR2_Length_of(_x) (((_x) >> 18) & 0x1f)
91 #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(_x) (((_x) & 0xff) << 10)
92 #define NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask_of(_x) (((_x) >> 10) & 0xff)
93 #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(_x) (((_x) & 0x3ff) << 0)
94 #define NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster_of(_x) (((_x) >> 0) & 0x3ff)
96 #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(_x) (((_x) & 0x1f) << 16)
97 #define NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(_x) (((_x) >> 16) & 0x1f)
98 #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress(_x) (((_x) & 0xffff) << 0)
99 #define NFP_PCIE_BAR_PCIE2CPP_BaseAddress_of(_x) (((_x) >> 0) & 0xffff)
100 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect(_x) (((_x) & 0x3) << 27)
101 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(_x) (((_x) >> 27) & 0x3)
102 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT 0
103 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT 1
104 #define NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE 3
105 #define NFP_PCIE_BAR_PCIE2CPP_MapType(_x) (((_x) & 0x7) << 29)
106 #define NFP_PCIE_BAR_PCIE2CPP_MapType_of(_x) (((_x) >> 29) & 0x7)
107 #define NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED 0
108 #define NFP_PCIE_BAR_PCIE2CPP_MapType_BULK 1
109 #define NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET 2
110 #define NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL 3
111 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0 4
112 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1 5
113 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2 6
114 #define NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3 7
115 #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(_x) (((_x) & 0xf) << 23)
116 #define NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(_x) (((_x) >> 23) & 0xf)
117 #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(_x) (((_x) & 0x3) << 21)
118 #define NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(_x) (((_x) >> 21) & 0x3)
119 #define NFP_PCIE_EM 0x020000
120 #define NFP_PCIE_SRAM 0x000000
122 /* Minimal size of the PCIe cfg memory we depend on being mapped,
123 * queue controller and DMA controller don't have to be covered.
125 #define NFP_PCI_MIN_MAP_SIZE 0x080000
127 #define NFP_PCIE_P2C_FIXED_SIZE(bar) (1 << (bar)->bitsize)
128 #define NFP_PCIE_P2C_BULK_SIZE(bar) (1 << (bar)->bitsize)
129 #define NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(bar, x) ((x) << ((bar)->bitsize - 2))
130 #define NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(bar, x) ((x) << ((bar)->bitsize - 4))
131 #define NFP_PCIE_P2C_GENERAL_SIZE(bar) (1 << ((bar)->bitsize - 4))
133 #define NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
134 (0x400 + ((bar) * 8 + (slot)) * 4)
136 #define NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(bar, slot) \
137 (((bar) * 8 + (slot)) * 4)
139 /* The number of explicit BARs to reserve.
140 * Minimum is 0, maximum is 4 on the NFP6000.
142 #define NFP_PCIE_EXPLICIT_BARS 2
145 struct nfp6000_area_priv;
148 * struct nfp_bar - describes BAR configuration and usage
149 * @nfp: backlink to owner
150 * @barcfg: cached contents of BAR config CSR
151 * @base: the BAR's base CPP offset
152 * @mask: mask for the BAR aperture (read only)
153 * @bitsize: bitsize of BAR aperture (read only)
154 * @index: index of the BAR
155 * @refcnt: number of current users
156 * @iomem: mapped IO memory
157 * @resource: iomem resource window
160 struct nfp6000_pcie *nfp;
162 u64 base; /* CPP address base */
163 u64 mask; /* Bit mask of the bar */
164 u32 bitsize; /* Bit size of the bar */
169 struct resource *resource;
172 #define NFP_PCI_BAR_MAX (PCI_64BIT_BAR_COUNT * 8)
174 struct nfp6000_pcie {
175 struct pci_dev *pdev;
178 /* PCI BAR management */
179 spinlock_t bar_lock; /* Protect the PCI2CPP BAR cache */
181 struct nfp_bar bar[NFP_PCI_BAR_MAX];
182 wait_queue_head_t bar_waiters;
184 /* Reserved BAR access */
188 void __iomem *expl[4];
191 /* Explicit IO access */
193 struct mutex mutex; /* Lock access to this explicit group */
205 static u32 nfp_bar_maptype(struct nfp_bar *bar)
207 return NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
210 static resource_size_t nfp_bar_resource_len(struct nfp_bar *bar)
212 return pci_resource_len(bar->nfp->pdev, (bar->index / 8) * 2) / 8;
215 static resource_size_t nfp_bar_resource_start(struct nfp_bar *bar)
217 return pci_resource_start(bar->nfp->pdev, (bar->index / 8) * 2)
218 + nfp_bar_resource_len(bar) * (bar->index & 7);
221 #define TARGET_WIDTH_32 4
222 #define TARGET_WIDTH_64 8
225 compute_bar(const struct nfp6000_pcie *nfp, const struct nfp_bar *bar,
226 u32 *bar_config, u64 *bar_base,
227 int tgt, int act, int tok, u64 offset, size_t size, int width)
232 if (tgt >= NFP_CPP_NUM_TARGETS)
237 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
238 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT);
241 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
242 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT);
245 newcfg = NFP_PCIE_BAR_PCIE2CPP_LengthSelect(
246 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE);
252 if (act != NFP_CPP_ACTION_RW && act != 0) {
253 /* Fixed CPP mapping with specific action */
254 u64 mask = ~(NFP_PCIE_P2C_FIXED_SIZE(bar) - 1);
256 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
257 NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED);
258 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
259 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress(act);
260 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
262 if ((offset & mask) != ((offset + size - 1) & mask))
268 u64 mask = ~(NFP_PCIE_P2C_BULK_SIZE(bar) - 1);
271 newcfg |= NFP_PCIE_BAR_PCIE2CPP_MapType(
272 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK);
273 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(tgt);
274 newcfg |= NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress(tok);
276 if ((offset & mask) != ((offset + size - 1) & mask))
284 if (bar->bitsize < bitsize)
287 newcfg |= offset >> bitsize;
293 *bar_config = newcfg;
299 nfp6000_bar_write(struct nfp6000_pcie *nfp, struct nfp_bar *bar, u32 newcfg)
304 base = bar->index >> 3;
305 slot = bar->index & 7;
307 if (nfp->iomem.csr) {
308 xbar = NFP_PCIE_CPP_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
309 writel(newcfg, nfp->iomem.csr + xbar);
310 /* Readback to ensure BAR is flushed */
311 readl(nfp->iomem.csr + xbar);
313 xbar = NFP_PCIE_CFG_BAR_PCIETOCPPEXPANSIONBAR(base, slot);
314 pci_write_config_dword(nfp->pdev, xbar, newcfg);
317 bar->barcfg = newcfg;
323 reconfigure_bar(struct nfp6000_pcie *nfp, struct nfp_bar *bar,
324 int tgt, int act, int tok, u64 offset, size_t size, int width)
330 err = compute_bar(nfp, bar, &newcfg, &newbase,
331 tgt, act, tok, offset, size, width);
337 return nfp6000_bar_write(nfp, bar, newcfg);
340 /* Check if BAR can be used with the given parameters. */
341 static int matching_bar(struct nfp_bar *bar, u32 tgt, u32 act, u32 tok,
342 u64 offset, size_t size, int width)
344 int bartgt, baract, bartok;
348 maptype = NFP_PCIE_BAR_PCIE2CPP_MapType_of(bar->barcfg);
349 bartgt = NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress_of(bar->barcfg);
350 bartok = NFP_PCIE_BAR_PCIE2CPP_Token_BaseAddress_of(bar->barcfg);
351 baract = NFP_PCIE_BAR_PCIE2CPP_Action_BaseAddress_of(bar->barcfg);
353 barwidth = NFP_PCIE_BAR_PCIE2CPP_LengthSelect_of(bar->barcfg);
355 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT:
358 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_64BIT:
361 case NFP_PCIE_BAR_PCIE2CPP_LengthSelect_0BYTE:
370 case NFP_PCIE_BAR_PCIE2CPP_MapType_TARGET:
373 case NFP_PCIE_BAR_PCIE2CPP_MapType_BULK:
374 baract = NFP_CPP_ACTION_RW;
376 act = NFP_CPP_ACTION_RW;
378 case NFP_PCIE_BAR_PCIE2CPP_MapType_FIXED:
381 /* We don't match explicit bars through the area interface */
385 /* Make sure to match up the width */
386 if (barwidth != width)
389 if ((bartgt < 0 || bartgt == tgt) &&
390 (bartok < 0 || bartok == tok) &&
392 bar->base <= offset &&
393 (bar->base + (1 << bar->bitsize)) >= (offset + size))
401 find_matching_bar(struct nfp6000_pcie *nfp,
402 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
406 for (n = 0; n < nfp->bars; n++) {
407 struct nfp_bar *bar = &nfp->bar[n];
409 if (matching_bar(bar, tgt, act, tok, offset, size, width))
416 /* Return EAGAIN if no resource is available */
418 find_unused_bar_noblock(const struct nfp6000_pcie *nfp,
419 int tgt, int act, int tok,
420 u64 offset, size_t size, int width)
424 for (n = 0; n < nfp->bars; n++) {
425 const struct nfp_bar *bar = &nfp->bar[n];
431 /* Just check to see if we can make it fit... */
432 err = compute_bar(nfp, bar, NULL, NULL,
433 tgt, act, tok, offset, size, width);
437 if (!atomic_read(&bar->refcnt))
443 if (WARN(!busy, "No suitable BAR found for request tgt:0x%x act:0x%x tok:0x%x off:0x%llx size:%zd width:%d\n",
444 tgt, act, tok, offset, size, width))
451 find_unused_bar_and_lock(struct nfp6000_pcie *nfp,
452 int tgt, int act, int tok,
453 u64 offset, size_t size, int width)
458 spin_lock_irqsave(&nfp->bar_lock, flags);
460 n = find_unused_bar_noblock(nfp, tgt, act, tok, offset, size, width);
462 spin_unlock_irqrestore(&nfp->bar_lock, flags);
464 __release(&nfp->bar_lock);
469 static void nfp_bar_get(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
471 atomic_inc(&bar->refcnt);
474 static void nfp_bar_put(struct nfp6000_pcie *nfp, struct nfp_bar *bar)
476 if (atomic_dec_and_test(&bar->refcnt))
477 wake_up_interruptible(&nfp->bar_waiters);
481 nfp_wait_for_bar(struct nfp6000_pcie *nfp, int *barnum,
482 u32 tgt, u32 act, u32 tok, u64 offset, size_t size, int width)
484 return wait_event_interruptible(nfp->bar_waiters,
485 (*barnum = find_unused_bar_and_lock(nfp, tgt, act, tok,
486 offset, size, width))
491 nfp_alloc_bar(struct nfp6000_pcie *nfp,
492 u32 tgt, u32 act, u32 tok,
493 u64 offset, size_t size, int width, int nonblocking)
495 unsigned long irqflags;
498 if (size > (1 << 24))
501 spin_lock_irqsave(&nfp->bar_lock, irqflags);
502 barnum = find_matching_bar(nfp, tgt, act, tok, offset, size, width);
504 /* Found a perfect match. */
505 nfp_bar_get(nfp, &nfp->bar[barnum]);
506 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
510 barnum = find_unused_bar_noblock(nfp, tgt, act, tok,
511 offset, size, width);
516 /* Wait until a BAR becomes available. The
517 * find_unused_bar function will reclaim the bar_lock
518 * if a free BAR is found.
520 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
521 retval = nfp_wait_for_bar(nfp, &barnum, tgt, act, tok,
522 offset, size, width);
525 __acquire(&nfp->bar_lock);
528 nfp_bar_get(nfp, &nfp->bar[barnum]);
529 retval = reconfigure_bar(nfp, &nfp->bar[barnum],
530 tgt, act, tok, offset, size, width);
532 nfp_bar_put(nfp, &nfp->bar[barnum]);
537 spin_unlock_irqrestore(&nfp->bar_lock, irqflags);
541 static void disable_bars(struct nfp6000_pcie *nfp);
543 static int bar_cmp(const void *aptr, const void *bptr)
545 const struct nfp_bar *a = aptr, *b = bptr;
547 if (a->bitsize == b->bitsize)
548 return a->index - b->index;
550 return a->bitsize - b->bitsize;
553 /* Map all PCI bars and fetch the actual BAR configurations from the
554 * board. We assume that the BAR with the PCIe config block is
557 * BAR0.0: Reserved for General Mapping (for MSI-X access to PCIe SRAM)
558 * BAR0.1: Reserved for XPB access (for MSI-X access to PCIe PBA)
561 * BAR0.4: Reserved for Explicit 0.0-0.3 access
562 * BAR0.5: Reserved for Explicit 1.0-1.3 access
563 * BAR0.6: Reserved for Explicit 2.0-2.3 access
564 * BAR0.7: Reserved for Explicit 3.0-3.3 access
569 static int enable_bars(struct nfp6000_pcie *nfp, u16 interface)
571 const u32 barcfg_msix_general =
572 NFP_PCIE_BAR_PCIE2CPP_MapType(
573 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) |
574 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT;
575 const u32 barcfg_msix_xpb =
576 NFP_PCIE_BAR_PCIE2CPP_MapType(
577 NFP_PCIE_BAR_PCIE2CPP_MapType_BULK) |
578 NFP_PCIE_BAR_PCIE2CPP_LengthSelect_32BIT |
579 NFP_PCIE_BAR_PCIE2CPP_Target_BaseAddress(
580 NFP_CPP_TARGET_ISLAND_XPB);
581 const u32 barcfg_explicit[4] = {
582 NFP_PCIE_BAR_PCIE2CPP_MapType(
583 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT0),
584 NFP_PCIE_BAR_PCIE2CPP_MapType(
585 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT1),
586 NFP_PCIE_BAR_PCIE2CPP_MapType(
587 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT2),
588 NFP_PCIE_BAR_PCIE2CPP_MapType(
589 NFP_PCIE_BAR_PCIE2CPP_MapType_EXPLICIT3),
591 char status_msg[196] = {};
598 snprintf(status_msg, sizeof(status_msg) - 1, "RESERVED BARs: ");
599 end = status_msg + sizeof(status_msg) - 1;
602 for (i = 0; i < ARRAY_SIZE(nfp->bar); i++, bar++) {
603 struct resource *res;
605 res = &nfp->pdev->resource[(i >> 3) * 2];
607 /* Skip over BARs that are not IORESOURCE_MEM */
608 if (!(resource_type(res) & IORESOURCE_MEM)) {
618 bar->mask = nfp_bar_resource_len(bar) - 1;
619 bar->bitsize = fls(bar->mask);
624 nfp->bars = bar - &nfp->bar[0];
626 dev_err(nfp->dev, "No usable BARs found!\n");
630 bars_free = nfp->bars;
632 /* Convert unit ID (0..3) to signal master/data master ID (0x40..0x70)
634 mutex_init(&nfp->expl.mutex);
636 nfp->expl.master_id = ((NFP_CPP_INTERFACE_UNIT_of(interface) & 3) + 4)
638 nfp->expl.signal_ref = 0x10;
640 /* Configure, and lock, BAR0.0 for General Target use (MSI-X SRAM) */
642 if (nfp_bar_resource_len(bar) >= NFP_PCI_MIN_MAP_SIZE)
643 bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
644 nfp_bar_resource_len(bar));
646 msg += snprintf(msg, end - msg, "0.0: General/MSI-X SRAM, ");
647 atomic_inc(&bar->refcnt);
650 nfp6000_bar_write(nfp, bar, barcfg_msix_general);
652 nfp->expl.data = bar->iomem + NFP_PCIE_SRAM + 0x1000;
654 if (nfp->pdev->device == PCI_DEVICE_ID_NETRONOME_NFP4000 ||
655 nfp->pdev->device == PCI_DEVICE_ID_NETRONOME_NFP6000) {
656 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(0);
658 int pf = nfp->pdev->devfn & 7;
660 nfp->iomem.csr = bar->iomem + NFP_PCIE_BAR(pf);
662 nfp->iomem.em = bar->iomem + NFP_PCIE_EM;
665 if (nfp->pdev->device == PCI_DEVICE_ID_NETRONOME_NFP4000 ||
666 nfp->pdev->device == PCI_DEVICE_ID_NETRONOME_NFP6000)
671 /* Configure, and lock, BAR0.1 for PCIe XPB (MSI-X PBA) */
673 msg += snprintf(msg, end - msg, "0.1: PCIe XPB/MSI-X PBA, ");
674 atomic_inc(&bar->refcnt);
677 nfp6000_bar_write(nfp, bar, barcfg_msix_xpb);
679 /* Use BAR0.4..BAR0.7 for EXPL IO */
680 for (i = 0; i < 4; i++) {
683 if (i >= NFP_PCIE_EXPLICIT_BARS || i >= expl_groups) {
684 nfp->expl.group[i].bitsize = 0;
688 bar = &nfp->bar[4 + i];
689 bar->iomem = ioremap_nocache(nfp_bar_resource_start(bar),
690 nfp_bar_resource_len(bar));
692 msg += snprintf(msg, end - msg,
693 "0.%d: Explicit%d, ", 4 + i, i);
694 atomic_inc(&bar->refcnt);
697 nfp->expl.group[i].bitsize = bar->bitsize;
698 nfp->expl.group[i].addr = bar->iomem;
699 nfp6000_bar_write(nfp, bar, barcfg_explicit[i]);
701 for (j = 0; j < 4; j++)
702 nfp->expl.group[i].free[j] = true;
704 nfp->iomem.expl[i] = bar->iomem;
707 /* Sort bars by bit size - use the smallest possible first. */
708 sort(&nfp->bar[0], nfp->bars, sizeof(nfp->bar[0]),
711 dev_info(nfp->dev, "%sfree: %d/%d\n", status_msg, bars_free, nfp->bars);
716 static void disable_bars(struct nfp6000_pcie *nfp)
718 struct nfp_bar *bar = &nfp->bar[0];
721 for (n = 0; n < nfp->bars; n++, bar++) {
730 * Generic CPP bus access interface.
733 struct nfp6000_area_priv {
752 struct resource resource;
755 static int nfp6000_area_init(struct nfp_cpp_area *area, u32 dest,
756 unsigned long long address, unsigned long size)
758 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
759 u32 target = NFP_CPP_ID_TARGET_of(dest);
760 u32 action = NFP_CPP_ID_ACTION_of(dest);
761 u32 token = NFP_CPP_ID_TOKEN_of(dest);
764 pp = nfp_target_pushpull(NFP_CPP_ID(target, action, token), address);
768 priv->width.read = PUSH_WIDTH(pp);
769 priv->width.write = PULL_WIDTH(pp);
770 if (priv->width.read > 0 &&
771 priv->width.write > 0 &&
772 priv->width.read != priv->width.write) {
776 if (priv->width.read > 0)
777 priv->width.bar = priv->width.read;
779 priv->width.bar = priv->width.write;
781 atomic_set(&priv->refcnt, 0);
784 priv->target = target;
785 priv->action = action;
787 priv->offset = address;
789 memset(&priv->resource, 0, sizeof(priv->resource));
794 static void nfp6000_area_cleanup(struct nfp_cpp_area *area)
798 static void priv_area_get(struct nfp_cpp_area *area)
800 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
802 atomic_inc(&priv->refcnt);
805 static int priv_area_put(struct nfp_cpp_area *area)
807 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
809 if (WARN_ON(!atomic_read(&priv->refcnt)))
812 return atomic_dec_and_test(&priv->refcnt);
815 static int nfp6000_area_acquire(struct nfp_cpp_area *area)
817 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
818 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
822 /* Already allocated. */
827 barnum = nfp_alloc_bar(nfp, priv->target, priv->action, priv->token,
828 priv->offset, priv->size, priv->width.bar, 1);
834 priv->bar = &nfp->bar[barnum];
836 /* Calculate offset into BAR. */
837 if (nfp_bar_maptype(priv->bar) ==
838 NFP_PCIE_BAR_PCIE2CPP_MapType_GENERAL) {
839 priv->bar_offset = priv->offset &
840 (NFP_PCIE_P2C_GENERAL_SIZE(priv->bar) - 1);
841 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TARGET_OFFSET(
842 priv->bar, priv->target);
843 priv->bar_offset += NFP_PCIE_P2C_GENERAL_TOKEN_OFFSET(
844 priv->bar, priv->token);
846 priv->bar_offset = priv->offset & priv->bar->mask;
849 /* We don't actually try to acquire the resource area using
850 * request_resource. This would prevent sharing the mapped
851 * BAR between multiple CPP areas and prevent us from
852 * effectively utilizing the limited amount of BAR resources.
854 priv->phys = nfp_bar_resource_start(priv->bar) + priv->bar_offset;
855 priv->resource.name = nfp_cpp_area_name(area);
856 priv->resource.start = priv->phys;
857 priv->resource.end = priv->resource.start + priv->size - 1;
858 priv->resource.flags = IORESOURCE_MEM;
860 /* If the bar is already mapped in, use its mapping */
861 if (priv->bar->iomem)
862 priv->iomem = priv->bar->iomem + priv->bar_offset;
864 /* Must have been too big. Sub-allocate. */
865 priv->iomem = ioremap_nocache(priv->phys, priv->size);
867 if (IS_ERR_OR_NULL(priv->iomem)) {
868 dev_err(nfp->dev, "Can't ioremap() a %d byte region of BAR %d\n",
869 (int)priv->size, priv->bar->index);
870 err = !priv->iomem ? -ENOMEM : PTR_ERR(priv->iomem);
872 goto err_iomem_remap;
879 nfp_bar_put(nfp, priv->bar);
885 static void nfp6000_area_release(struct nfp_cpp_area *area)
887 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_area_cpp(area));
888 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
890 if (!priv_area_put(area))
893 if (!priv->bar->iomem)
894 iounmap(priv->iomem);
896 nfp_bar_put(nfp, priv->bar);
902 static phys_addr_t nfp6000_area_phys(struct nfp_cpp_area *area)
904 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
909 static void __iomem *nfp6000_area_iomem(struct nfp_cpp_area *area)
911 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
916 static struct resource *nfp6000_area_resource(struct nfp_cpp_area *area)
918 /* Use the BAR resource as the resource for the CPP area.
919 * This enables us to share the BAR among multiple CPP areas
920 * without resource conflicts.
922 struct nfp6000_area_priv *priv = nfp_cpp_area_priv(area);
924 return priv->bar->resource;
927 static int nfp6000_area_read(struct nfp_cpp_area *area, void *kernel_vaddr,
928 unsigned long offset, unsigned int length)
930 u64 __maybe_unused *wrptr64 = kernel_vaddr;
931 const u64 __iomem __maybe_unused *rdptr64;
932 struct nfp6000_area_priv *priv;
933 u32 *wrptr32 = kernel_vaddr;
934 const u32 __iomem *rdptr32;
937 priv = nfp_cpp_area_priv(area);
938 rdptr64 = priv->iomem + offset;
939 rdptr32 = priv->iomem + offset;
941 if (offset + length > priv->size)
944 width = priv->width.read;
948 /* MU reads via a PCIe2CPP BAR support 32bit (and other) lengths */
949 if (priv->target == (NFP_CPP_TARGET_MU & NFP_CPP_TARGET_ID_MASK) &&
950 priv->action == NFP_CPP_ACTION_RW &&
951 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
952 width = TARGET_WIDTH_32;
954 /* Unaligned? Translate to an explicit access */
955 if ((priv->offset + offset) & (width - 1))
956 return nfp_cpp_explicit_read(nfp_cpp_area_cpp(area),
957 NFP_CPP_ID(priv->target,
960 priv->offset + offset,
961 kernel_vaddr, length, width);
963 if (WARN_ON(!priv->bar))
967 case TARGET_WIDTH_32:
968 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
971 for (n = 0; n < length; n += sizeof(u32))
972 *wrptr32++ = __raw_readl(rdptr32++);
975 case TARGET_WIDTH_64:
976 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
979 for (n = 0; n < length; n += sizeof(u64))
980 *wrptr64++ = __raw_readq(rdptr64++);
989 nfp6000_area_write(struct nfp_cpp_area *area,
990 const void *kernel_vaddr,
991 unsigned long offset, unsigned int length)
993 const u64 __maybe_unused *rdptr64 = kernel_vaddr;
994 u64 __iomem __maybe_unused *wrptr64;
995 const u32 *rdptr32 = kernel_vaddr;
996 struct nfp6000_area_priv *priv;
997 u32 __iomem *wrptr32;
1000 priv = nfp_cpp_area_priv(area);
1001 wrptr64 = priv->iomem + offset;
1002 wrptr32 = priv->iomem + offset;
1004 if (offset + length > priv->size)
1007 width = priv->width.write;
1011 /* MU writes via a PCIe2CPP BAR support 32bit (and other) lengths */
1012 if (priv->target == (NFP_CPP_TARGET_ID_MASK & NFP_CPP_TARGET_MU) &&
1013 priv->action == NFP_CPP_ACTION_RW &&
1014 (offset % sizeof(u64) == 4 || length % sizeof(u64) == 4))
1015 width = TARGET_WIDTH_32;
1017 /* Unaligned? Translate to an explicit access */
1018 if ((priv->offset + offset) & (width - 1))
1019 return nfp_cpp_explicit_write(nfp_cpp_area_cpp(area),
1020 NFP_CPP_ID(priv->target,
1023 priv->offset + offset,
1024 kernel_vaddr, length, width);
1026 if (WARN_ON(!priv->bar))
1030 case TARGET_WIDTH_32:
1031 if (offset % sizeof(u32) != 0 || length % sizeof(u32) != 0)
1034 for (n = 0; n < length; n += sizeof(u32)) {
1035 __raw_writel(*rdptr32++, wrptr32++);
1040 case TARGET_WIDTH_64:
1041 if (offset % sizeof(u64) != 0 || length % sizeof(u64) != 0)
1044 for (n = 0; n < length; n += sizeof(u64)) {
1045 __raw_writeq(*rdptr64++, wrptr64++);
1055 struct nfp6000_explicit_priv {
1056 struct nfp6000_pcie *nfp;
1066 static int nfp6000_explicit_acquire(struct nfp_cpp_explicit *expl)
1068 struct nfp6000_pcie *nfp = nfp_cpp_priv(nfp_cpp_explicit_cpp(expl));
1069 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1072 mutex_lock(&nfp->expl.mutex);
1073 for (i = 0; i < ARRAY_SIZE(nfp->expl.group); i++) {
1074 if (!nfp->expl.group[i].bitsize)
1077 for (j = 0; j < ARRAY_SIZE(nfp->expl.group[i].free); j++) {
1080 if (!nfp->expl.group[i].free[j])
1084 priv->bar.group = i;
1086 priv->bitsize = nfp->expl.group[i].bitsize - 2;
1088 data_offset = (priv->bar.group << 9) +
1089 (priv->bar.area << 7);
1090 priv->data = nfp->expl.data + data_offset;
1091 priv->addr = nfp->expl.group[i].addr +
1092 (priv->bar.area << priv->bitsize);
1093 nfp->expl.group[i].free[j] = false;
1095 mutex_unlock(&nfp->expl.mutex);
1099 mutex_unlock(&nfp->expl.mutex);
1104 static void nfp6000_explicit_release(struct nfp_cpp_explicit *expl)
1106 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1107 struct nfp6000_pcie *nfp = priv->nfp;
1109 mutex_lock(&nfp->expl.mutex);
1110 nfp->expl.group[priv->bar.group].free[priv->bar.area] = true;
1111 mutex_unlock(&nfp->expl.mutex);
1114 static int nfp6000_explicit_put(struct nfp_cpp_explicit *expl,
1115 const void *buff, size_t len)
1117 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1118 const u32 *src = buff;
1121 for (i = 0; i < len; i += sizeof(u32))
1122 writel(*(src++), priv->data + i);
1128 nfp6000_explicit_do(struct nfp_cpp_explicit *expl,
1129 const struct nfp_cpp_explicit_command *cmd, u64 address)
1131 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1132 u8 signal_master, signal_ref, data_master;
1133 struct nfp6000_pcie *nfp = priv->nfp;
1139 sigmask |= 1 << cmd->siga;
1141 sigmask |= 1 << cmd->sigb;
1143 signal_master = cmd->signal_master;
1145 signal_master = nfp->expl.master_id;
1147 signal_ref = cmd->signal_ref;
1148 if (signal_master == nfp->expl.master_id)
1149 signal_ref = nfp->expl.signal_ref +
1150 ((priv->bar.group * 4 + priv->bar.area) << 1);
1152 data_master = cmd->data_master;
1154 data_master = nfp->expl.master_id;
1156 data_ref = cmd->data_ref;
1157 if (data_master == nfp->expl.master_id)
1159 (priv->bar.group << 9) + (priv->bar.area << 7);
1161 csr[0] = NFP_PCIE_BAR_EXPLICIT_BAR0_SignalType(sigmask) |
1162 NFP_PCIE_BAR_EXPLICIT_BAR0_Token(
1163 NFP_CPP_ID_TOKEN_of(cmd->cpp_id)) |
1164 NFP_PCIE_BAR_EXPLICIT_BAR0_Address(address >> 16);
1166 csr[1] = NFP_PCIE_BAR_EXPLICIT_BAR1_SignalRef(signal_ref) |
1167 NFP_PCIE_BAR_EXPLICIT_BAR1_DataMaster(data_master) |
1168 NFP_PCIE_BAR_EXPLICIT_BAR1_DataRef(data_ref);
1170 csr[2] = NFP_PCIE_BAR_EXPLICIT_BAR2_Target(
1171 NFP_CPP_ID_TARGET_of(cmd->cpp_id)) |
1172 NFP_PCIE_BAR_EXPLICIT_BAR2_Action(
1173 NFP_CPP_ID_ACTION_of(cmd->cpp_id)) |
1174 NFP_PCIE_BAR_EXPLICIT_BAR2_Length(cmd->len) |
1175 NFP_PCIE_BAR_EXPLICIT_BAR2_ByteMask(cmd->byte_mask) |
1176 NFP_PCIE_BAR_EXPLICIT_BAR2_SignalMaster(signal_master);
1178 if (nfp->iomem.csr) {
1179 writel(csr[0], nfp->iomem.csr +
1180 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1182 writel(csr[1], nfp->iomem.csr +
1183 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1185 writel(csr[2], nfp->iomem.csr +
1186 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1188 /* Readback to ensure BAR is flushed */
1189 readl(nfp->iomem.csr +
1190 NFP_PCIE_BAR_EXPLICIT_BAR0(priv->bar.group,
1192 readl(nfp->iomem.csr +
1193 NFP_PCIE_BAR_EXPLICIT_BAR1(priv->bar.group,
1195 readl(nfp->iomem.csr +
1196 NFP_PCIE_BAR_EXPLICIT_BAR2(priv->bar.group,
1199 pci_write_config_dword(nfp->pdev, 0x400 +
1200 NFP_PCIE_BAR_EXPLICIT_BAR0(
1201 priv->bar.group, priv->bar.area),
1204 pci_write_config_dword(nfp->pdev, 0x400 +
1205 NFP_PCIE_BAR_EXPLICIT_BAR1(
1206 priv->bar.group, priv->bar.area),
1209 pci_write_config_dword(nfp->pdev, 0x400 +
1210 NFP_PCIE_BAR_EXPLICIT_BAR2(
1211 priv->bar.group, priv->bar.area),
1215 /* Issue the 'kickoff' transaction */
1216 readb(priv->addr + (address & ((1 << priv->bitsize) - 1)));
1221 static int nfp6000_explicit_get(struct nfp_cpp_explicit *expl,
1222 void *buff, size_t len)
1224 struct nfp6000_explicit_priv *priv = nfp_cpp_explicit_priv(expl);
1228 for (i = 0; i < len; i += sizeof(u32))
1229 *(dst++) = readl(priv->data + i);
1234 static int nfp6000_init(struct nfp_cpp *cpp)
1236 nfp_cpp_area_cache_add(cpp, SZ_64K);
1237 nfp_cpp_area_cache_add(cpp, SZ_64K);
1238 nfp_cpp_area_cache_add(cpp, SZ_256K);
1243 static void nfp6000_free(struct nfp_cpp *cpp)
1245 struct nfp6000_pcie *nfp = nfp_cpp_priv(cpp);
1251 static void nfp6000_read_serial(struct device *dev, u8 *serial)
1253 struct pci_dev *pdev = to_pci_dev(dev);
1257 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
1259 memset(serial, 0, NFP_SERIAL_LEN);
1263 pci_read_config_dword(pdev, pos + 4, ®);
1264 put_unaligned_be16(reg >> 16, serial + 4);
1265 pci_read_config_dword(pdev, pos + 8, ®);
1266 put_unaligned_be32(reg, serial);
1269 static u16 nfp6000_get_interface(struct device *dev)
1271 struct pci_dev *pdev = to_pci_dev(dev);
1275 pos = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
1277 return NFP_CPP_INTERFACE(NFP_CPP_INTERFACE_TYPE_PCI, 0, 0xff);
1279 pci_read_config_dword(pdev, pos + 4, ®);
1281 return reg & 0xffff;
1284 static const struct nfp_cpp_operations nfp6000_pcie_ops = {
1285 .owner = THIS_MODULE,
1287 .init = nfp6000_init,
1288 .free = nfp6000_free,
1290 .read_serial = nfp6000_read_serial,
1291 .get_interface = nfp6000_get_interface,
1293 .area_priv_size = sizeof(struct nfp6000_area_priv),
1294 .area_init = nfp6000_area_init,
1295 .area_cleanup = nfp6000_area_cleanup,
1296 .area_acquire = nfp6000_area_acquire,
1297 .area_release = nfp6000_area_release,
1298 .area_phys = nfp6000_area_phys,
1299 .area_iomem = nfp6000_area_iomem,
1300 .area_resource = nfp6000_area_resource,
1301 .area_read = nfp6000_area_read,
1302 .area_write = nfp6000_area_write,
1304 .explicit_priv_size = sizeof(struct nfp6000_explicit_priv),
1305 .explicit_acquire = nfp6000_explicit_acquire,
1306 .explicit_release = nfp6000_explicit_release,
1307 .explicit_put = nfp6000_explicit_put,
1308 .explicit_do = nfp6000_explicit_do,
1309 .explicit_get = nfp6000_explicit_get,
1313 * nfp_cpp_from_nfp6000_pcie() - Build a NFP CPP bus from a NFP6000 PCI device
1314 * @pdev: NFP6000 PCI device
1316 * Return: NFP CPP handle
1318 struct nfp_cpp *nfp_cpp_from_nfp6000_pcie(struct pci_dev *pdev)
1320 struct nfp6000_pcie *nfp;
1324 /* Finished with card initialization. */
1325 dev_info(&pdev->dev,
1326 "Netronome Flow Processor NFP4000/NFP6000 PCIe Card Probe\n");
1327 pcie_print_link_status(pdev);
1329 nfp = kzalloc(sizeof(*nfp), GFP_KERNEL);
1335 nfp->dev = &pdev->dev;
1337 init_waitqueue_head(&nfp->bar_waiters);
1338 spin_lock_init(&nfp->bar_lock);
1340 interface = nfp6000_get_interface(&pdev->dev);
1342 if (NFP_CPP_INTERFACE_TYPE_of(interface) !=
1343 NFP_CPP_INTERFACE_TYPE_PCI) {
1345 "Interface type %d is not the expected %d\n",
1346 NFP_CPP_INTERFACE_TYPE_of(interface),
1347 NFP_CPP_INTERFACE_TYPE_PCI);
1352 if (NFP_CPP_INTERFACE_CHANNEL_of(interface) !=
1353 NFP_CPP_INTERFACE_CHANNEL_PEROPENER) {
1354 dev_err(&pdev->dev, "Interface channel %d is not the expected %d\n",
1355 NFP_CPP_INTERFACE_CHANNEL_of(interface),
1356 NFP_CPP_INTERFACE_CHANNEL_PEROPENER);
1361 err = enable_bars(nfp, interface);
1365 /* Probe for all the common NFP devices */
1366 return nfp_cpp_from_operations(&nfp6000_pcie_ops, &pdev->dev, nfp);
1371 dev_err(&pdev->dev, "NFP6000 PCI setup failed\n");
1372 return ERR_PTR(err);