1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for Tegra SoCs
5 * Copyright (c) 2010, CompuLab, Ltd.
6 * Author: Mike Rapoport <mike@compulab.co.il>
8 * Based on NVIDIA PCIe driver
9 * Copyright (c) 2008-2009, NVIDIA Corporation.
11 * Bits taken from arch/arm/mach-dove/pcie.c
13 * Author: Thierry Reding <treding@nvidia.com>
16 #include <linux/clk.h>
17 #include <linux/debugfs.h>
18 #include <linux/delay.h>
19 #include <linux/export.h>
20 #include <linux/gpio/consumer.h>
21 #include <linux/interrupt.h>
22 #include <linux/iopoll.h>
23 #include <linux/irq.h>
24 #include <linux/irqchip/chained_irq.h>
25 #include <linux/irqdomain.h>
26 #include <linux/kernel.h>
27 #include <linux/init.h>
28 #include <linux/module.h>
29 #include <linux/msi.h>
30 #include <linux/of_address.h>
31 #include <linux/of_pci.h>
32 #include <linux/of_platform.h>
33 #include <linux/pci.h>
34 #include <linux/phy/phy.h>
35 #include <linux/pinctrl/consumer.h>
36 #include <linux/platform_device.h>
37 #include <linux/reset.h>
38 #include <linux/sizes.h>
39 #include <linux/slab.h>
40 #include <linux/vmalloc.h>
41 #include <linux/regulator/consumer.h>
43 #include <soc/tegra/cpuidle.h>
44 #include <soc/tegra/pmc.h>
48 #define INT_PCI_MSI_NR (8 * 32)
50 /* register definitions */
52 #define AFI_AXI_BAR0_SZ 0x00
53 #define AFI_AXI_BAR1_SZ 0x04
54 #define AFI_AXI_BAR2_SZ 0x08
55 #define AFI_AXI_BAR3_SZ 0x0c
56 #define AFI_AXI_BAR4_SZ 0x10
57 #define AFI_AXI_BAR5_SZ 0x14
59 #define AFI_AXI_BAR0_START 0x18
60 #define AFI_AXI_BAR1_START 0x1c
61 #define AFI_AXI_BAR2_START 0x20
62 #define AFI_AXI_BAR3_START 0x24
63 #define AFI_AXI_BAR4_START 0x28
64 #define AFI_AXI_BAR5_START 0x2c
66 #define AFI_FPCI_BAR0 0x30
67 #define AFI_FPCI_BAR1 0x34
68 #define AFI_FPCI_BAR2 0x38
69 #define AFI_FPCI_BAR3 0x3c
70 #define AFI_FPCI_BAR4 0x40
71 #define AFI_FPCI_BAR5 0x44
73 #define AFI_CACHE_BAR0_SZ 0x48
74 #define AFI_CACHE_BAR0_ST 0x4c
75 #define AFI_CACHE_BAR1_SZ 0x50
76 #define AFI_CACHE_BAR1_ST 0x54
78 #define AFI_MSI_BAR_SZ 0x60
79 #define AFI_MSI_FPCI_BAR_ST 0x64
80 #define AFI_MSI_AXI_BAR_ST 0x68
82 #define AFI_MSI_VEC(x) (0x6c + ((x) * 4))
83 #define AFI_MSI_EN_VEC(x) (0x8c + ((x) * 4))
85 #define AFI_CONFIGURATION 0xac
86 #define AFI_CONFIGURATION_EN_FPCI (1 << 0)
87 #define AFI_CONFIGURATION_CLKEN_OVERRIDE (1 << 31)
89 #define AFI_FPCI_ERROR_MASKS 0xb0
91 #define AFI_INTR_MASK 0xb4
92 #define AFI_INTR_MASK_INT_MASK (1 << 0)
93 #define AFI_INTR_MASK_MSI_MASK (1 << 8)
95 #define AFI_INTR_CODE 0xb8
96 #define AFI_INTR_CODE_MASK 0xf
97 #define AFI_INTR_INI_SLAVE_ERROR 1
98 #define AFI_INTR_INI_DECODE_ERROR 2
99 #define AFI_INTR_TARGET_ABORT 3
100 #define AFI_INTR_MASTER_ABORT 4
101 #define AFI_INTR_INVALID_WRITE 5
102 #define AFI_INTR_LEGACY 6
103 #define AFI_INTR_FPCI_DECODE_ERROR 7
104 #define AFI_INTR_AXI_DECODE_ERROR 8
105 #define AFI_INTR_FPCI_TIMEOUT 9
106 #define AFI_INTR_PE_PRSNT_SENSE 10
107 #define AFI_INTR_PE_CLKREQ_SENSE 11
108 #define AFI_INTR_CLKCLAMP_SENSE 12
109 #define AFI_INTR_RDY4PD_SENSE 13
110 #define AFI_INTR_P2P_ERROR 14
112 #define AFI_INTR_SIGNATURE 0xbc
113 #define AFI_UPPER_FPCI_ADDRESS 0xc0
114 #define AFI_SM_INTR_ENABLE 0xc4
115 #define AFI_SM_INTR_INTA_ASSERT (1 << 0)
116 #define AFI_SM_INTR_INTB_ASSERT (1 << 1)
117 #define AFI_SM_INTR_INTC_ASSERT (1 << 2)
118 #define AFI_SM_INTR_INTD_ASSERT (1 << 3)
119 #define AFI_SM_INTR_INTA_DEASSERT (1 << 4)
120 #define AFI_SM_INTR_INTB_DEASSERT (1 << 5)
121 #define AFI_SM_INTR_INTC_DEASSERT (1 << 6)
122 #define AFI_SM_INTR_INTD_DEASSERT (1 << 7)
124 #define AFI_AFI_INTR_ENABLE 0xc8
125 #define AFI_INTR_EN_INI_SLVERR (1 << 0)
126 #define AFI_INTR_EN_INI_DECERR (1 << 1)
127 #define AFI_INTR_EN_TGT_SLVERR (1 << 2)
128 #define AFI_INTR_EN_TGT_DECERR (1 << 3)
129 #define AFI_INTR_EN_TGT_WRERR (1 << 4)
130 #define AFI_INTR_EN_DFPCI_DECERR (1 << 5)
131 #define AFI_INTR_EN_AXI_DECERR (1 << 6)
132 #define AFI_INTR_EN_FPCI_TIMEOUT (1 << 7)
133 #define AFI_INTR_EN_PRSNT_SENSE (1 << 8)
135 #define AFI_PCIE_PME 0xf0
137 #define AFI_PCIE_CONFIG 0x0f8
138 #define AFI_PCIE_CONFIG_PCIE_DISABLE(x) (1 << ((x) + 1))
139 #define AFI_PCIE_CONFIG_PCIE_DISABLE_ALL 0xe
140 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK (0xf << 20)
141 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE (0x0 << 20)
142 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420 (0x0 << 20)
143 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1 (0x0 << 20)
144 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401 (0x0 << 20)
145 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL (0x1 << 20)
146 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222 (0x1 << 20)
147 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1 (0x1 << 20)
148 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211 (0x1 << 20)
149 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411 (0x2 << 20)
150 #define AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111 (0x2 << 20)
151 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(x) (1 << ((x) + 29))
152 #define AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL (0x7 << 29)
154 #define AFI_FUSE 0x104
155 #define AFI_FUSE_PCIE_T0_GEN2_DIS (1 << 2)
157 #define AFI_PEX0_CTRL 0x110
158 #define AFI_PEX1_CTRL 0x118
159 #define AFI_PEX_CTRL_RST (1 << 0)
160 #define AFI_PEX_CTRL_CLKREQ_EN (1 << 1)
161 #define AFI_PEX_CTRL_REFCLK_EN (1 << 3)
162 #define AFI_PEX_CTRL_OVERRIDE_EN (1 << 4)
164 #define AFI_PLLE_CONTROL 0x160
165 #define AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL (1 << 9)
166 #define AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN (1 << 1)
168 #define AFI_PEXBIAS_CTRL_0 0x168
170 #define RP_ECTL_2_R1 0x00000e84
171 #define RP_ECTL_2_R1_RX_CTLE_1C_MASK 0xffff
173 #define RP_ECTL_4_R1 0x00000e8c
174 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK (0xffff << 16)
175 #define RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT 16
177 #define RP_ECTL_5_R1 0x00000e90
178 #define RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK 0xffffffff
180 #define RP_ECTL_6_R1 0x00000e94
181 #define RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK 0xffffffff
183 #define RP_ECTL_2_R2 0x00000ea4
184 #define RP_ECTL_2_R2_RX_CTLE_1C_MASK 0xffff
186 #define RP_ECTL_4_R2 0x00000eac
187 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK (0xffff << 16)
188 #define RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT 16
190 #define RP_ECTL_5_R2 0x00000eb0
191 #define RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK 0xffffffff
193 #define RP_ECTL_6_R2 0x00000eb4
194 #define RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK 0xffffffff
196 #define RP_VEND_XP 0x00000f00
197 #define RP_VEND_XP_DL_UP (1 << 30)
198 #define RP_VEND_XP_OPPORTUNISTIC_ACK (1 << 27)
199 #define RP_VEND_XP_OPPORTUNISTIC_UPDATEFC (1 << 28)
200 #define RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK (0xff << 18)
202 #define RP_VEND_CTL0 0x00000f44
203 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK (0xf << 12)
204 #define RP_VEND_CTL0_DSK_RST_PULSE_WIDTH (0x9 << 12)
206 #define RP_VEND_CTL1 0x00000f48
207 #define RP_VEND_CTL1_ERPT (1 << 13)
209 #define RP_VEND_XP_BIST 0x00000f4c
210 #define RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE (1 << 28)
212 #define RP_VEND_CTL2 0x00000fa8
213 #define RP_VEND_CTL2_PCA_ENABLE (1 << 7)
215 #define RP_PRIV_MISC 0x00000fe0
216 #define RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT (0xe << 0)
217 #define RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT (0xf << 0)
218 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK (0x7f << 16)
219 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD (0xf << 16)
220 #define RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE (1 << 23)
221 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK (0x7f << 24)
222 #define RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD (0xf << 24)
223 #define RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE (1 << 31)
225 #define RP_LINK_CONTROL_STATUS 0x00000090
226 #define RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE 0x20000000
227 #define RP_LINK_CONTROL_STATUS_LINKSTAT_MASK 0x3fff0000
229 #define RP_LINK_CONTROL_STATUS_2 0x000000b0
231 #define PADS_CTL_SEL 0x0000009c
233 #define PADS_CTL 0x000000a0
234 #define PADS_CTL_IDDQ_1L (1 << 0)
235 #define PADS_CTL_TX_DATA_EN_1L (1 << 6)
236 #define PADS_CTL_RX_DATA_EN_1L (1 << 10)
238 #define PADS_PLL_CTL_TEGRA20 0x000000b8
239 #define PADS_PLL_CTL_TEGRA30 0x000000b4
240 #define PADS_PLL_CTL_RST_B4SM (1 << 1)
241 #define PADS_PLL_CTL_LOCKDET (1 << 8)
242 #define PADS_PLL_CTL_REFCLK_MASK (0x3 << 16)
243 #define PADS_PLL_CTL_REFCLK_INTERNAL_CML (0 << 16)
244 #define PADS_PLL_CTL_REFCLK_INTERNAL_CMOS (1 << 16)
245 #define PADS_PLL_CTL_REFCLK_EXTERNAL (2 << 16)
246 #define PADS_PLL_CTL_TXCLKREF_MASK (0x1 << 20)
247 #define PADS_PLL_CTL_TXCLKREF_DIV10 (0 << 20)
248 #define PADS_PLL_CTL_TXCLKREF_DIV5 (1 << 20)
249 #define PADS_PLL_CTL_TXCLKREF_BUF_EN (1 << 22)
251 #define PADS_REFCLK_CFG0 0x000000c8
252 #define PADS_REFCLK_CFG1 0x000000cc
253 #define PADS_REFCLK_BIAS 0x000000d0
256 * Fields in PADS_REFCLK_CFG*. Those registers form an array of 16-bit
257 * entries, one entry per PCIe port. These field definitions and desired
258 * values aren't in the TRM, but do come from NVIDIA.
260 #define PADS_REFCLK_CFG_TERM_SHIFT 2 /* 6:2 */
261 #define PADS_REFCLK_CFG_E_TERM_SHIFT 7
262 #define PADS_REFCLK_CFG_PREDI_SHIFT 8 /* 11:8 */
263 #define PADS_REFCLK_CFG_DRVI_SHIFT 12 /* 15:12 */
265 #define PME_ACK_TIMEOUT 10000
266 #define LINK_RETRAIN_TIMEOUT 100000 /* in usec */
269 DECLARE_BITMAP(used, INT_PCI_MSI_NR);
270 struct irq_domain *domain;
271 struct mutex map_lock;
272 spinlock_t mask_lock;
278 /* used to differentiate between Tegra SoC generations */
279 struct tegra_pcie_port_soc {
286 struct tegra_pcie_soc {
287 unsigned int num_ports;
288 const struct tegra_pcie_port_soc *ports;
289 unsigned int msi_base_shift;
290 unsigned long afi_pex2_ctrl;
293 u32 pads_refclk_cfg0;
294 u32 pads_refclk_cfg1;
295 u32 update_fc_threshold;
296 bool has_pex_clkreq_en;
297 bool has_pex_bias_ctrl;
298 bool has_intr_prsnt_sense;
301 bool force_pca_enable;
303 bool update_clamp_threshold;
304 bool program_deskew_time;
305 bool update_fc_timer;
337 struct reset_control *pex_rst;
338 struct reset_control *afi_rst;
339 struct reset_control *pcie_xrst;
344 struct tegra_msi msi;
346 struct list_head ports;
349 struct regulator_bulk_data *supplies;
350 unsigned int num_supplies;
352 const struct tegra_pcie_soc *soc;
353 struct dentry *debugfs;
356 static inline struct tegra_pcie *msi_to_pcie(struct tegra_msi *msi)
358 return container_of(msi, struct tegra_pcie, msi);
361 struct tegra_pcie_port {
362 struct tegra_pcie *pcie;
363 struct device_node *np;
364 struct list_head list;
365 struct resource regs;
372 struct gpio_desc *reset_gpio;
375 struct tegra_pcie_bus {
376 struct list_head list;
380 static inline void afi_writel(struct tegra_pcie *pcie, u32 value,
381 unsigned long offset)
383 writel(value, pcie->afi + offset);
386 static inline u32 afi_readl(struct tegra_pcie *pcie, unsigned long offset)
388 return readl(pcie->afi + offset);
391 static inline void pads_writel(struct tegra_pcie *pcie, u32 value,
392 unsigned long offset)
394 writel(value, pcie->pads + offset);
397 static inline u32 pads_readl(struct tegra_pcie *pcie, unsigned long offset)
399 return readl(pcie->pads + offset);
403 * The configuration space mapping on Tegra is somewhat similar to the ECAM
404 * defined by PCIe. However it deviates a bit in how the 4 bits for extended
405 * register accesses are mapped:
407 * [27:24] extended register number
409 * [15:11] device number
410 * [10: 8] function number
411 * [ 7: 0] register number
413 * Mapping the whole extended configuration space would require 256 MiB of
414 * virtual address space, only a small part of which will actually be used.
416 * To work around this, a 4 KiB region is used to generate the required
417 * configuration transaction with relevant B:D:F and register offset values.
418 * This is achieved by dynamically programming base address and size of
419 * AFI_AXI_BAR used for end point config space mapping to make sure that the
420 * address (access to which generates correct config transaction) falls in
423 static unsigned int tegra_pcie_conf_offset(u8 bus, unsigned int devfn,
426 return ((where & 0xf00) << 16) | (bus << 16) | (PCI_SLOT(devfn) << 11) |
427 (PCI_FUNC(devfn) << 8) | (where & 0xff);
430 static void __iomem *tegra_pcie_map_bus(struct pci_bus *bus,
434 struct tegra_pcie *pcie = bus->sysdata;
435 void __iomem *addr = NULL;
437 if (bus->number == 0) {
438 unsigned int slot = PCI_SLOT(devfn);
439 struct tegra_pcie_port *port;
441 list_for_each_entry(port, &pcie->ports, list) {
442 if (port->index + 1 == slot) {
443 addr = port->base + (where & ~3);
451 offset = tegra_pcie_conf_offset(bus->number, devfn, where);
453 /* move 4 KiB window to offset within the FPCI region */
454 base = 0xfe100000 + ((offset & ~(SZ_4K - 1)) >> 8);
455 afi_writel(pcie, base, AFI_FPCI_BAR0);
457 /* move to correct offset within the 4 KiB page */
458 addr = pcie->cfg + (offset & (SZ_4K - 1));
464 static int tegra_pcie_config_read(struct pci_bus *bus, unsigned int devfn,
465 int where, int size, u32 *value)
467 if (bus->number == 0)
468 return pci_generic_config_read32(bus, devfn, where, size,
471 return pci_generic_config_read(bus, devfn, where, size, value);
474 static int tegra_pcie_config_write(struct pci_bus *bus, unsigned int devfn,
475 int where, int size, u32 value)
477 if (bus->number == 0)
478 return pci_generic_config_write32(bus, devfn, where, size,
481 return pci_generic_config_write(bus, devfn, where, size, value);
484 static struct pci_ops tegra_pcie_ops = {
485 .map_bus = tegra_pcie_map_bus,
486 .read = tegra_pcie_config_read,
487 .write = tegra_pcie_config_write,
490 static unsigned long tegra_pcie_port_get_pex_ctrl(struct tegra_pcie_port *port)
492 const struct tegra_pcie_soc *soc = port->pcie->soc;
493 unsigned long ret = 0;
495 switch (port->index) {
505 ret = soc->afi_pex2_ctrl;
512 static void tegra_pcie_port_reset(struct tegra_pcie_port *port)
514 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
517 /* pulse reset signal */
518 if (port->reset_gpio) {
519 gpiod_set_value(port->reset_gpio, 1);
521 value = afi_readl(port->pcie, ctrl);
522 value &= ~AFI_PEX_CTRL_RST;
523 afi_writel(port->pcie, value, ctrl);
526 usleep_range(1000, 2000);
528 if (port->reset_gpio) {
529 gpiod_set_value(port->reset_gpio, 0);
531 value = afi_readl(port->pcie, ctrl);
532 value |= AFI_PEX_CTRL_RST;
533 afi_writel(port->pcie, value, ctrl);
537 static void tegra_pcie_enable_rp_features(struct tegra_pcie_port *port)
539 const struct tegra_pcie_soc *soc = port->pcie->soc;
542 /* Enable AER capability */
543 value = readl(port->base + RP_VEND_CTL1);
544 value |= RP_VEND_CTL1_ERPT;
545 writel(value, port->base + RP_VEND_CTL1);
547 /* Optimal settings to enhance bandwidth */
548 value = readl(port->base + RP_VEND_XP);
549 value |= RP_VEND_XP_OPPORTUNISTIC_ACK;
550 value |= RP_VEND_XP_OPPORTUNISTIC_UPDATEFC;
551 writel(value, port->base + RP_VEND_XP);
554 * LTSSM will wait for DLLP to finish before entering L1 or L2,
555 * to avoid truncation of PM messages which results in receiver errors
557 value = readl(port->base + RP_VEND_XP_BIST);
558 value |= RP_VEND_XP_BIST_GOTO_L1_L2_AFTER_DLLP_DONE;
559 writel(value, port->base + RP_VEND_XP_BIST);
561 value = readl(port->base + RP_PRIV_MISC);
562 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_ENABLE;
563 value |= RP_PRIV_MISC_TMS_CLK_CLAMP_ENABLE;
565 if (soc->update_clamp_threshold) {
566 value &= ~(RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD_MASK |
567 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD_MASK);
568 value |= RP_PRIV_MISC_CTLR_CLK_CLAMP_THRESHOLD |
569 RP_PRIV_MISC_TMS_CLK_CLAMP_THRESHOLD;
572 writel(value, port->base + RP_PRIV_MISC);
575 static void tegra_pcie_program_ectl_settings(struct tegra_pcie_port *port)
577 const struct tegra_pcie_soc *soc = port->pcie->soc;
580 value = readl(port->base + RP_ECTL_2_R1);
581 value &= ~RP_ECTL_2_R1_RX_CTLE_1C_MASK;
582 value |= soc->ectl.regs.rp_ectl_2_r1;
583 writel(value, port->base + RP_ECTL_2_R1);
585 value = readl(port->base + RP_ECTL_4_R1);
586 value &= ~RP_ECTL_4_R1_RX_CDR_CTRL_1C_MASK;
587 value |= soc->ectl.regs.rp_ectl_4_r1 <<
588 RP_ECTL_4_R1_RX_CDR_CTRL_1C_SHIFT;
589 writel(value, port->base + RP_ECTL_4_R1);
591 value = readl(port->base + RP_ECTL_5_R1);
592 value &= ~RP_ECTL_5_R1_RX_EQ_CTRL_L_1C_MASK;
593 value |= soc->ectl.regs.rp_ectl_5_r1;
594 writel(value, port->base + RP_ECTL_5_R1);
596 value = readl(port->base + RP_ECTL_6_R1);
597 value &= ~RP_ECTL_6_R1_RX_EQ_CTRL_H_1C_MASK;
598 value |= soc->ectl.regs.rp_ectl_6_r1;
599 writel(value, port->base + RP_ECTL_6_R1);
601 value = readl(port->base + RP_ECTL_2_R2);
602 value &= ~RP_ECTL_2_R2_RX_CTLE_1C_MASK;
603 value |= soc->ectl.regs.rp_ectl_2_r2;
604 writel(value, port->base + RP_ECTL_2_R2);
606 value = readl(port->base + RP_ECTL_4_R2);
607 value &= ~RP_ECTL_4_R2_RX_CDR_CTRL_1C_MASK;
608 value |= soc->ectl.regs.rp_ectl_4_r2 <<
609 RP_ECTL_4_R2_RX_CDR_CTRL_1C_SHIFT;
610 writel(value, port->base + RP_ECTL_4_R2);
612 value = readl(port->base + RP_ECTL_5_R2);
613 value &= ~RP_ECTL_5_R2_RX_EQ_CTRL_L_1C_MASK;
614 value |= soc->ectl.regs.rp_ectl_5_r2;
615 writel(value, port->base + RP_ECTL_5_R2);
617 value = readl(port->base + RP_ECTL_6_R2);
618 value &= ~RP_ECTL_6_R2_RX_EQ_CTRL_H_1C_MASK;
619 value |= soc->ectl.regs.rp_ectl_6_r2;
620 writel(value, port->base + RP_ECTL_6_R2);
623 static void tegra_pcie_apply_sw_fixup(struct tegra_pcie_port *port)
625 const struct tegra_pcie_soc *soc = port->pcie->soc;
629 * Sometimes link speed change from Gen2 to Gen1 fails due to
630 * instability in deskew logic on lane-0. Increase the deskew
631 * retry time to resolve this issue.
633 if (soc->program_deskew_time) {
634 value = readl(port->base + RP_VEND_CTL0);
635 value &= ~RP_VEND_CTL0_DSK_RST_PULSE_WIDTH_MASK;
636 value |= RP_VEND_CTL0_DSK_RST_PULSE_WIDTH;
637 writel(value, port->base + RP_VEND_CTL0);
640 if (soc->update_fc_timer) {
641 value = readl(port->base + RP_VEND_XP);
642 value &= ~RP_VEND_XP_UPDATE_FC_THRESHOLD_MASK;
643 value |= soc->update_fc_threshold;
644 writel(value, port->base + RP_VEND_XP);
648 * PCIe link doesn't come up with few legacy PCIe endpoints if
649 * root port advertises both Gen-1 and Gen-2 speeds in Tegra.
650 * Hence, the strategy followed here is to initially advertise
651 * only Gen-1 and after link is up, retrain link to Gen-2 speed
653 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
654 value &= ~PCI_EXP_LNKSTA_CLS;
655 value |= PCI_EXP_LNKSTA_CLS_2_5GB;
656 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
659 static void tegra_pcie_port_enable(struct tegra_pcie_port *port)
661 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
662 const struct tegra_pcie_soc *soc = port->pcie->soc;
665 /* enable reference clock */
666 value = afi_readl(port->pcie, ctrl);
667 value |= AFI_PEX_CTRL_REFCLK_EN;
669 if (soc->has_pex_clkreq_en)
670 value |= AFI_PEX_CTRL_CLKREQ_EN;
672 value |= AFI_PEX_CTRL_OVERRIDE_EN;
674 afi_writel(port->pcie, value, ctrl);
676 tegra_pcie_port_reset(port);
678 if (soc->force_pca_enable) {
679 value = readl(port->base + RP_VEND_CTL2);
680 value |= RP_VEND_CTL2_PCA_ENABLE;
681 writel(value, port->base + RP_VEND_CTL2);
684 tegra_pcie_enable_rp_features(port);
686 if (soc->ectl.enable)
687 tegra_pcie_program_ectl_settings(port);
689 tegra_pcie_apply_sw_fixup(port);
692 static void tegra_pcie_port_disable(struct tegra_pcie_port *port)
694 unsigned long ctrl = tegra_pcie_port_get_pex_ctrl(port);
695 const struct tegra_pcie_soc *soc = port->pcie->soc;
698 /* assert port reset */
699 value = afi_readl(port->pcie, ctrl);
700 value &= ~AFI_PEX_CTRL_RST;
701 afi_writel(port->pcie, value, ctrl);
703 /* disable reference clock */
704 value = afi_readl(port->pcie, ctrl);
706 if (soc->has_pex_clkreq_en)
707 value &= ~AFI_PEX_CTRL_CLKREQ_EN;
709 value &= ~AFI_PEX_CTRL_REFCLK_EN;
710 afi_writel(port->pcie, value, ctrl);
712 /* disable PCIe port and set CLKREQ# as GPIO to allow PLLE power down */
713 value = afi_readl(port->pcie, AFI_PCIE_CONFIG);
714 value |= AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
715 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
716 afi_writel(port->pcie, value, AFI_PCIE_CONFIG);
719 static void tegra_pcie_port_free(struct tegra_pcie_port *port)
721 struct tegra_pcie *pcie = port->pcie;
722 struct device *dev = pcie->dev;
724 devm_iounmap(dev, port->base);
725 devm_release_mem_region(dev, port->regs.start,
726 resource_size(&port->regs));
727 list_del(&port->list);
728 devm_kfree(dev, port);
731 /* Tegra PCIE root complex wrongly reports device class */
732 static void tegra_pcie_fixup_class(struct pci_dev *dev)
734 dev->class = PCI_CLASS_BRIDGE_PCI << 8;
736 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_fixup_class);
737 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_fixup_class);
738 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_fixup_class);
739 DECLARE_PCI_FIXUP_EARLY(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_fixup_class);
741 /* Tegra20 and Tegra30 PCIE requires relaxed ordering */
742 static void tegra_pcie_relax_enable(struct pci_dev *dev)
744 pcie_capability_set_word(dev, PCI_EXP_DEVCTL, PCI_EXP_DEVCTL_RELAX_EN);
746 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf0, tegra_pcie_relax_enable);
747 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0bf1, tegra_pcie_relax_enable);
748 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1c, tegra_pcie_relax_enable);
749 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_NVIDIA, 0x0e1d, tegra_pcie_relax_enable);
751 static int tegra_pcie_map_irq(const struct pci_dev *pdev, u8 slot, u8 pin)
753 struct tegra_pcie *pcie = pdev->bus->sysdata;
756 tegra_cpuidle_pcie_irqs_in_use();
758 irq = of_irq_parse_and_map_pci(pdev, slot, pin);
765 static irqreturn_t tegra_pcie_isr(int irq, void *arg)
767 const char *err_msg[] = {
775 "Response decoding error",
776 "AXI response decoding error",
777 "Transaction timeout",
778 "Slot present pin change",
779 "Slot clock request change",
780 "TMS clock ramp change",
781 "TMS ready for power down",
784 struct tegra_pcie *pcie = arg;
785 struct device *dev = pcie->dev;
788 code = afi_readl(pcie, AFI_INTR_CODE) & AFI_INTR_CODE_MASK;
789 signature = afi_readl(pcie, AFI_INTR_SIGNATURE);
790 afi_writel(pcie, 0, AFI_INTR_CODE);
792 if (code == AFI_INTR_LEGACY)
795 if (code >= ARRAY_SIZE(err_msg))
799 * do not pollute kernel log with master abort reports since they
800 * happen a lot during enumeration
802 if (code == AFI_INTR_MASTER_ABORT || code == AFI_INTR_PE_PRSNT_SENSE)
803 dev_dbg(dev, "%s, signature: %08x\n", err_msg[code], signature);
805 dev_err(dev, "%s, signature: %08x\n", err_msg[code], signature);
807 if (code == AFI_INTR_TARGET_ABORT || code == AFI_INTR_MASTER_ABORT ||
808 code == AFI_INTR_FPCI_DECODE_ERROR) {
809 u32 fpci = afi_readl(pcie, AFI_UPPER_FPCI_ADDRESS) & 0xff;
810 u64 address = (u64)fpci << 32 | (signature & 0xfffffffc);
812 if (code == AFI_INTR_MASTER_ABORT)
813 dev_dbg(dev, " FPCI address: %10llx\n", address);
815 dev_err(dev, " FPCI address: %10llx\n", address);
822 * FPCI map is as follows:
823 * - 0xfdfc000000: I/O space
824 * - 0xfdfe000000: type 0 configuration space
825 * - 0xfdff000000: type 1 configuration space
826 * - 0xfe00000000: type 0 extended configuration space
827 * - 0xfe10000000: type 1 extended configuration space
829 static void tegra_pcie_setup_translations(struct tegra_pcie *pcie)
832 struct resource_entry *entry;
833 struct pci_host_bridge *bridge = pci_host_bridge_from_priv(pcie);
835 /* Bar 0: type 1 extended configuration space */
836 size = resource_size(&pcie->cs);
837 afi_writel(pcie, pcie->cs.start, AFI_AXI_BAR0_START);
838 afi_writel(pcie, size >> 12, AFI_AXI_BAR0_SZ);
840 resource_list_for_each_entry(entry, &bridge->windows) {
841 u32 fpci_bar, axi_address;
842 struct resource *res = entry->res;
844 size = resource_size(res);
846 switch (resource_type(res)) {
848 /* Bar 1: downstream IO bar */
849 fpci_bar = 0xfdfc0000;
850 axi_address = pci_pio_to_address(res->start);
851 afi_writel(pcie, axi_address, AFI_AXI_BAR1_START);
852 afi_writel(pcie, size >> 12, AFI_AXI_BAR1_SZ);
853 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR1);
856 fpci_bar = (((res->start >> 12) & 0x0fffffff) << 4) | 0x1;
857 axi_address = res->start;
859 if (res->flags & IORESOURCE_PREFETCH) {
860 /* Bar 2: prefetchable memory BAR */
861 afi_writel(pcie, axi_address, AFI_AXI_BAR2_START);
862 afi_writel(pcie, size >> 12, AFI_AXI_BAR2_SZ);
863 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR2);
866 /* Bar 3: non prefetchable memory BAR */
867 afi_writel(pcie, axi_address, AFI_AXI_BAR3_START);
868 afi_writel(pcie, size >> 12, AFI_AXI_BAR3_SZ);
869 afi_writel(pcie, fpci_bar, AFI_FPCI_BAR3);
875 /* NULL out the remaining BARs as they are not used */
876 afi_writel(pcie, 0, AFI_AXI_BAR4_START);
877 afi_writel(pcie, 0, AFI_AXI_BAR4_SZ);
878 afi_writel(pcie, 0, AFI_FPCI_BAR4);
880 afi_writel(pcie, 0, AFI_AXI_BAR5_START);
881 afi_writel(pcie, 0, AFI_AXI_BAR5_SZ);
882 afi_writel(pcie, 0, AFI_FPCI_BAR5);
884 if (pcie->soc->has_cache_bars) {
885 /* map all upstream transactions as uncached */
886 afi_writel(pcie, 0, AFI_CACHE_BAR0_ST);
887 afi_writel(pcie, 0, AFI_CACHE_BAR0_SZ);
888 afi_writel(pcie, 0, AFI_CACHE_BAR1_ST);
889 afi_writel(pcie, 0, AFI_CACHE_BAR1_SZ);
892 /* MSI translations are setup only when needed */
893 afi_writel(pcie, 0, AFI_MSI_FPCI_BAR_ST);
894 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
895 afi_writel(pcie, 0, AFI_MSI_AXI_BAR_ST);
896 afi_writel(pcie, 0, AFI_MSI_BAR_SZ);
899 static int tegra_pcie_pll_wait(struct tegra_pcie *pcie, unsigned long timeout)
901 const struct tegra_pcie_soc *soc = pcie->soc;
904 timeout = jiffies + msecs_to_jiffies(timeout);
906 while (time_before(jiffies, timeout)) {
907 value = pads_readl(pcie, soc->pads_pll_ctl);
908 if (value & PADS_PLL_CTL_LOCKDET)
915 static int tegra_pcie_phy_enable(struct tegra_pcie *pcie)
917 struct device *dev = pcie->dev;
918 const struct tegra_pcie_soc *soc = pcie->soc;
922 /* initialize internal PHY, enable up to 16 PCIE lanes */
923 pads_writel(pcie, 0x0, PADS_CTL_SEL);
925 /* override IDDQ to 1 on all 4 lanes */
926 value = pads_readl(pcie, PADS_CTL);
927 value |= PADS_CTL_IDDQ_1L;
928 pads_writel(pcie, value, PADS_CTL);
931 * Set up PHY PLL inputs select PLLE output as refclock,
932 * set TX ref sel to div10 (not div5).
934 value = pads_readl(pcie, soc->pads_pll_ctl);
935 value &= ~(PADS_PLL_CTL_REFCLK_MASK | PADS_PLL_CTL_TXCLKREF_MASK);
936 value |= PADS_PLL_CTL_REFCLK_INTERNAL_CML | soc->tx_ref_sel;
937 pads_writel(pcie, value, soc->pads_pll_ctl);
940 value = pads_readl(pcie, soc->pads_pll_ctl);
941 value &= ~PADS_PLL_CTL_RST_B4SM;
942 pads_writel(pcie, value, soc->pads_pll_ctl);
944 usleep_range(20, 100);
946 /* take PLL out of reset */
947 value = pads_readl(pcie, soc->pads_pll_ctl);
948 value |= PADS_PLL_CTL_RST_B4SM;
949 pads_writel(pcie, value, soc->pads_pll_ctl);
951 /* wait for the PLL to lock */
952 err = tegra_pcie_pll_wait(pcie, 500);
954 dev_err(dev, "PLL failed to lock: %d\n", err);
958 /* turn off IDDQ override */
959 value = pads_readl(pcie, PADS_CTL);
960 value &= ~PADS_CTL_IDDQ_1L;
961 pads_writel(pcie, value, PADS_CTL);
963 /* enable TX/RX data */
964 value = pads_readl(pcie, PADS_CTL);
965 value |= PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L;
966 pads_writel(pcie, value, PADS_CTL);
971 static int tegra_pcie_phy_disable(struct tegra_pcie *pcie)
973 const struct tegra_pcie_soc *soc = pcie->soc;
976 /* disable TX/RX data */
977 value = pads_readl(pcie, PADS_CTL);
978 value &= ~(PADS_CTL_TX_DATA_EN_1L | PADS_CTL_RX_DATA_EN_1L);
979 pads_writel(pcie, value, PADS_CTL);
982 value = pads_readl(pcie, PADS_CTL);
983 value |= PADS_CTL_IDDQ_1L;
984 pads_writel(pcie, value, PADS_CTL);
987 value = pads_readl(pcie, soc->pads_pll_ctl);
988 value &= ~PADS_PLL_CTL_RST_B4SM;
989 pads_writel(pcie, value, soc->pads_pll_ctl);
991 usleep_range(20, 100);
996 static int tegra_pcie_port_phy_power_on(struct tegra_pcie_port *port)
998 struct device *dev = port->pcie->dev;
1002 for (i = 0; i < port->lanes; i++) {
1003 err = phy_power_on(port->phys[i]);
1005 dev_err(dev, "failed to power on PHY#%u: %d\n", i, err);
1013 static int tegra_pcie_port_phy_power_off(struct tegra_pcie_port *port)
1015 struct device *dev = port->pcie->dev;
1019 for (i = 0; i < port->lanes; i++) {
1020 err = phy_power_off(port->phys[i]);
1022 dev_err(dev, "failed to power off PHY#%u: %d\n", i,
1031 static int tegra_pcie_phy_power_on(struct tegra_pcie *pcie)
1033 struct device *dev = pcie->dev;
1034 struct tegra_pcie_port *port;
1037 if (pcie->legacy_phy) {
1039 err = phy_power_on(pcie->phy);
1041 err = tegra_pcie_phy_enable(pcie);
1044 dev_err(dev, "failed to power on PHY: %d\n", err);
1049 list_for_each_entry(port, &pcie->ports, list) {
1050 err = tegra_pcie_port_phy_power_on(port);
1053 "failed to power on PCIe port %u PHY: %d\n",
1062 static int tegra_pcie_phy_power_off(struct tegra_pcie *pcie)
1064 struct device *dev = pcie->dev;
1065 struct tegra_pcie_port *port;
1068 if (pcie->legacy_phy) {
1070 err = phy_power_off(pcie->phy);
1072 err = tegra_pcie_phy_disable(pcie);
1075 dev_err(dev, "failed to power off PHY: %d\n", err);
1080 list_for_each_entry(port, &pcie->ports, list) {
1081 err = tegra_pcie_port_phy_power_off(port);
1084 "failed to power off PCIe port %u PHY: %d\n",
1093 static void tegra_pcie_enable_controller(struct tegra_pcie *pcie)
1095 const struct tegra_pcie_soc *soc = pcie->soc;
1096 struct tegra_pcie_port *port;
1097 unsigned long value;
1099 /* enable PLL power down */
1101 value = afi_readl(pcie, AFI_PLLE_CONTROL);
1102 value &= ~AFI_PLLE_CONTROL_BYPASS_PADS2PLLE_CONTROL;
1103 value |= AFI_PLLE_CONTROL_PADS2PLLE_CONTROL_EN;
1104 afi_writel(pcie, value, AFI_PLLE_CONTROL);
1107 /* power down PCIe slot clock bias pad */
1108 if (soc->has_pex_bias_ctrl)
1109 afi_writel(pcie, 0, AFI_PEXBIAS_CTRL_0);
1111 /* configure mode and disable all ports */
1112 value = afi_readl(pcie, AFI_PCIE_CONFIG);
1113 value &= ~AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_MASK;
1114 value |= AFI_PCIE_CONFIG_PCIE_DISABLE_ALL | pcie->xbar_config;
1115 value |= AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO_ALL;
1117 list_for_each_entry(port, &pcie->ports, list) {
1118 value &= ~AFI_PCIE_CONFIG_PCIE_DISABLE(port->index);
1119 value &= ~AFI_PCIE_CONFIG_PCIE_CLKREQ_GPIO(port->index);
1122 afi_writel(pcie, value, AFI_PCIE_CONFIG);
1124 if (soc->has_gen2) {
1125 value = afi_readl(pcie, AFI_FUSE);
1126 value &= ~AFI_FUSE_PCIE_T0_GEN2_DIS;
1127 afi_writel(pcie, value, AFI_FUSE);
1129 value = afi_readl(pcie, AFI_FUSE);
1130 value |= AFI_FUSE_PCIE_T0_GEN2_DIS;
1131 afi_writel(pcie, value, AFI_FUSE);
1134 /* Disable AFI dynamic clock gating and enable PCIe */
1135 value = afi_readl(pcie, AFI_CONFIGURATION);
1136 value |= AFI_CONFIGURATION_EN_FPCI;
1137 value |= AFI_CONFIGURATION_CLKEN_OVERRIDE;
1138 afi_writel(pcie, value, AFI_CONFIGURATION);
1140 value = AFI_INTR_EN_INI_SLVERR | AFI_INTR_EN_INI_DECERR |
1141 AFI_INTR_EN_TGT_SLVERR | AFI_INTR_EN_TGT_DECERR |
1142 AFI_INTR_EN_TGT_WRERR | AFI_INTR_EN_DFPCI_DECERR;
1144 if (soc->has_intr_prsnt_sense)
1145 value |= AFI_INTR_EN_PRSNT_SENSE;
1147 afi_writel(pcie, value, AFI_AFI_INTR_ENABLE);
1148 afi_writel(pcie, 0xffffffff, AFI_SM_INTR_ENABLE);
1150 /* don't enable MSI for now, only when needed */
1151 afi_writel(pcie, AFI_INTR_MASK_INT_MASK, AFI_INTR_MASK);
1153 /* disable all exceptions */
1154 afi_writel(pcie, 0, AFI_FPCI_ERROR_MASKS);
1157 static void tegra_pcie_power_off(struct tegra_pcie *pcie)
1159 struct device *dev = pcie->dev;
1160 const struct tegra_pcie_soc *soc = pcie->soc;
1163 reset_control_assert(pcie->afi_rst);
1165 clk_disable_unprepare(pcie->pll_e);
1166 if (soc->has_cml_clk)
1167 clk_disable_unprepare(pcie->cml_clk);
1168 clk_disable_unprepare(pcie->afi_clk);
1170 if (!dev->pm_domain)
1171 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1173 err = regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1175 dev_warn(dev, "failed to disable regulators: %d\n", err);
1178 static int tegra_pcie_power_on(struct tegra_pcie *pcie)
1180 struct device *dev = pcie->dev;
1181 const struct tegra_pcie_soc *soc = pcie->soc;
1184 reset_control_assert(pcie->pcie_xrst);
1185 reset_control_assert(pcie->afi_rst);
1186 reset_control_assert(pcie->pex_rst);
1188 if (!dev->pm_domain)
1189 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1191 /* enable regulators */
1192 err = regulator_bulk_enable(pcie->num_supplies, pcie->supplies);
1194 dev_err(dev, "failed to enable regulators: %d\n", err);
1196 if (!dev->pm_domain) {
1197 err = tegra_powergate_power_on(TEGRA_POWERGATE_PCIE);
1199 dev_err(dev, "failed to power ungate: %d\n", err);
1200 goto regulator_disable;
1202 err = tegra_powergate_remove_clamping(TEGRA_POWERGATE_PCIE);
1204 dev_err(dev, "failed to remove clamp: %d\n", err);
1209 err = clk_prepare_enable(pcie->afi_clk);
1211 dev_err(dev, "failed to enable AFI clock: %d\n", err);
1215 if (soc->has_cml_clk) {
1216 err = clk_prepare_enable(pcie->cml_clk);
1218 dev_err(dev, "failed to enable CML clock: %d\n", err);
1219 goto disable_afi_clk;
1223 err = clk_prepare_enable(pcie->pll_e);
1225 dev_err(dev, "failed to enable PLLE clock: %d\n", err);
1226 goto disable_cml_clk;
1229 reset_control_deassert(pcie->afi_rst);
1234 if (soc->has_cml_clk)
1235 clk_disable_unprepare(pcie->cml_clk);
1237 clk_disable_unprepare(pcie->afi_clk);
1239 if (!dev->pm_domain)
1240 tegra_powergate_power_off(TEGRA_POWERGATE_PCIE);
1242 regulator_bulk_disable(pcie->num_supplies, pcie->supplies);
1247 static void tegra_pcie_apply_pad_settings(struct tegra_pcie *pcie)
1249 const struct tegra_pcie_soc *soc = pcie->soc;
1251 /* Configure the reference clock driver */
1252 pads_writel(pcie, soc->pads_refclk_cfg0, PADS_REFCLK_CFG0);
1254 if (soc->num_ports > 2)
1255 pads_writel(pcie, soc->pads_refclk_cfg1, PADS_REFCLK_CFG1);
1258 static int tegra_pcie_clocks_get(struct tegra_pcie *pcie)
1260 struct device *dev = pcie->dev;
1261 const struct tegra_pcie_soc *soc = pcie->soc;
1263 pcie->pex_clk = devm_clk_get(dev, "pex");
1264 if (IS_ERR(pcie->pex_clk))
1265 return PTR_ERR(pcie->pex_clk);
1267 pcie->afi_clk = devm_clk_get(dev, "afi");
1268 if (IS_ERR(pcie->afi_clk))
1269 return PTR_ERR(pcie->afi_clk);
1271 pcie->pll_e = devm_clk_get(dev, "pll_e");
1272 if (IS_ERR(pcie->pll_e))
1273 return PTR_ERR(pcie->pll_e);
1275 if (soc->has_cml_clk) {
1276 pcie->cml_clk = devm_clk_get(dev, "cml");
1277 if (IS_ERR(pcie->cml_clk))
1278 return PTR_ERR(pcie->cml_clk);
1284 static int tegra_pcie_resets_get(struct tegra_pcie *pcie)
1286 struct device *dev = pcie->dev;
1288 pcie->pex_rst = devm_reset_control_get_exclusive(dev, "pex");
1289 if (IS_ERR(pcie->pex_rst))
1290 return PTR_ERR(pcie->pex_rst);
1292 pcie->afi_rst = devm_reset_control_get_exclusive(dev, "afi");
1293 if (IS_ERR(pcie->afi_rst))
1294 return PTR_ERR(pcie->afi_rst);
1296 pcie->pcie_xrst = devm_reset_control_get_exclusive(dev, "pcie_x");
1297 if (IS_ERR(pcie->pcie_xrst))
1298 return PTR_ERR(pcie->pcie_xrst);
1303 static int tegra_pcie_phys_get_legacy(struct tegra_pcie *pcie)
1305 struct device *dev = pcie->dev;
1308 pcie->phy = devm_phy_optional_get(dev, "pcie");
1309 if (IS_ERR(pcie->phy)) {
1310 err = PTR_ERR(pcie->phy);
1311 dev_err(dev, "failed to get PHY: %d\n", err);
1315 err = phy_init(pcie->phy);
1317 dev_err(dev, "failed to initialize PHY: %d\n", err);
1321 pcie->legacy_phy = true;
1326 static struct phy *devm_of_phy_optional_get_index(struct device *dev,
1327 struct device_node *np,
1328 const char *consumer,
1334 name = kasprintf(GFP_KERNEL, "%s-%u", consumer, index);
1336 return ERR_PTR(-ENOMEM);
1338 phy = devm_of_phy_get(dev, np, name);
1341 if (PTR_ERR(phy) == -ENODEV)
1347 static int tegra_pcie_port_get_phys(struct tegra_pcie_port *port)
1349 struct device *dev = port->pcie->dev;
1354 port->phys = devm_kcalloc(dev, sizeof(phy), port->lanes, GFP_KERNEL);
1358 for (i = 0; i < port->lanes; i++) {
1359 phy = devm_of_phy_optional_get_index(dev, port->np, "pcie", i);
1361 dev_err(dev, "failed to get PHY#%u: %ld\n", i,
1363 return PTR_ERR(phy);
1366 err = phy_init(phy);
1368 dev_err(dev, "failed to initialize PHY#%u: %d\n", i,
1373 port->phys[i] = phy;
1379 static int tegra_pcie_phys_get(struct tegra_pcie *pcie)
1381 const struct tegra_pcie_soc *soc = pcie->soc;
1382 struct device_node *np = pcie->dev->of_node;
1383 struct tegra_pcie_port *port;
1386 if (!soc->has_gen2 || of_find_property(np, "phys", NULL) != NULL)
1387 return tegra_pcie_phys_get_legacy(pcie);
1389 list_for_each_entry(port, &pcie->ports, list) {
1390 err = tegra_pcie_port_get_phys(port);
1398 static void tegra_pcie_phys_put(struct tegra_pcie *pcie)
1400 struct tegra_pcie_port *port;
1401 struct device *dev = pcie->dev;
1404 if (pcie->legacy_phy) {
1405 err = phy_exit(pcie->phy);
1407 dev_err(dev, "failed to teardown PHY: %d\n", err);
1411 list_for_each_entry(port, &pcie->ports, list) {
1412 for (i = 0; i < port->lanes; i++) {
1413 err = phy_exit(port->phys[i]);
1415 dev_err(dev, "failed to teardown PHY#%u: %d\n",
1421 static int tegra_pcie_get_resources(struct tegra_pcie *pcie)
1423 struct device *dev = pcie->dev;
1424 struct platform_device *pdev = to_platform_device(dev);
1425 struct resource *res;
1426 const struct tegra_pcie_soc *soc = pcie->soc;
1429 err = tegra_pcie_clocks_get(pcie);
1431 dev_err(dev, "failed to get clocks: %d\n", err);
1435 err = tegra_pcie_resets_get(pcie);
1437 dev_err(dev, "failed to get resets: %d\n", err);
1441 if (soc->program_uphy) {
1442 err = tegra_pcie_phys_get(pcie);
1444 dev_err(dev, "failed to get PHYs: %d\n", err);
1449 pcie->pads = devm_platform_ioremap_resource_byname(pdev, "pads");
1450 if (IS_ERR(pcie->pads)) {
1451 err = PTR_ERR(pcie->pads);
1455 pcie->afi = devm_platform_ioremap_resource_byname(pdev, "afi");
1456 if (IS_ERR(pcie->afi)) {
1457 err = PTR_ERR(pcie->afi);
1461 /* request configuration space, but remap later, on demand */
1462 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "cs");
1464 err = -EADDRNOTAVAIL;
1470 /* constrain configuration space to 4 KiB */
1471 pcie->cs.end = pcie->cs.start + SZ_4K - 1;
1473 pcie->cfg = devm_ioremap_resource(dev, &pcie->cs);
1474 if (IS_ERR(pcie->cfg)) {
1475 err = PTR_ERR(pcie->cfg);
1479 /* request interrupt */
1480 err = platform_get_irq_byname(pdev, "intr");
1486 err = request_irq(pcie->irq, tegra_pcie_isr, IRQF_SHARED, "PCIE", pcie);
1488 dev_err(dev, "failed to register IRQ: %d\n", err);
1495 if (soc->program_uphy)
1496 tegra_pcie_phys_put(pcie);
1501 static int tegra_pcie_put_resources(struct tegra_pcie *pcie)
1503 const struct tegra_pcie_soc *soc = pcie->soc;
1506 free_irq(pcie->irq, pcie);
1508 if (soc->program_uphy)
1509 tegra_pcie_phys_put(pcie);
1514 static void tegra_pcie_pme_turnoff(struct tegra_pcie_port *port)
1516 struct tegra_pcie *pcie = port->pcie;
1517 const struct tegra_pcie_soc *soc = pcie->soc;
1522 val = afi_readl(pcie, AFI_PCIE_PME);
1523 val |= (0x1 << soc->ports[port->index].pme.turnoff_bit);
1524 afi_writel(pcie, val, AFI_PCIE_PME);
1526 ack_bit = soc->ports[port->index].pme.ack_bit;
1527 err = readl_poll_timeout(pcie->afi + AFI_PCIE_PME, val,
1528 val & (0x1 << ack_bit), 1, PME_ACK_TIMEOUT);
1530 dev_err(pcie->dev, "PME Ack is not received on port: %d\n",
1533 usleep_range(10000, 11000);
1535 val = afi_readl(pcie, AFI_PCIE_PME);
1536 val &= ~(0x1 << soc->ports[port->index].pme.turnoff_bit);
1537 afi_writel(pcie, val, AFI_PCIE_PME);
1540 static void tegra_pcie_msi_irq(struct irq_desc *desc)
1542 struct tegra_pcie *pcie = irq_desc_get_handler_data(desc);
1543 struct irq_chip *chip = irq_desc_get_chip(desc);
1544 struct tegra_msi *msi = &pcie->msi;
1545 struct device *dev = pcie->dev;
1548 chained_irq_enter(chip, desc);
1550 for (i = 0; i < 8; i++) {
1551 unsigned long reg = afi_readl(pcie, AFI_MSI_VEC(i));
1554 unsigned int offset = find_first_bit(®, 32);
1555 unsigned int index = i * 32 + offset;
1558 ret = generic_handle_domain_irq(msi->domain->parent, index);
1561 * that's weird who triggered this?
1564 dev_info(dev, "unexpected MSI\n");
1565 afi_writel(pcie, BIT(index % 32), AFI_MSI_VEC(index));
1568 /* see if there's any more pending in this vector */
1569 reg = afi_readl(pcie, AFI_MSI_VEC(i));
1573 chained_irq_exit(chip, desc);
1576 static void tegra_msi_top_irq_ack(struct irq_data *d)
1578 irq_chip_ack_parent(d);
1581 static void tegra_msi_top_irq_mask(struct irq_data *d)
1583 pci_msi_mask_irq(d);
1584 irq_chip_mask_parent(d);
1587 static void tegra_msi_top_irq_unmask(struct irq_data *d)
1589 pci_msi_unmask_irq(d);
1590 irq_chip_unmask_parent(d);
1593 static struct irq_chip tegra_msi_top_chip = {
1594 .name = "Tegra PCIe MSI",
1595 .irq_ack = tegra_msi_top_irq_ack,
1596 .irq_mask = tegra_msi_top_irq_mask,
1597 .irq_unmask = tegra_msi_top_irq_unmask,
1600 static void tegra_msi_irq_ack(struct irq_data *d)
1602 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1603 struct tegra_pcie *pcie = msi_to_pcie(msi);
1604 unsigned int index = d->hwirq / 32;
1606 /* clear the interrupt */
1607 afi_writel(pcie, BIT(d->hwirq % 32), AFI_MSI_VEC(index));
1610 static void tegra_msi_irq_mask(struct irq_data *d)
1612 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1613 struct tegra_pcie *pcie = msi_to_pcie(msi);
1614 unsigned int index = d->hwirq / 32;
1615 unsigned long flags;
1618 spin_lock_irqsave(&msi->mask_lock, flags);
1619 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1620 value &= ~BIT(d->hwirq % 32);
1621 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1622 spin_unlock_irqrestore(&msi->mask_lock, flags);
1625 static void tegra_msi_irq_unmask(struct irq_data *d)
1627 struct tegra_msi *msi = irq_data_get_irq_chip_data(d);
1628 struct tegra_pcie *pcie = msi_to_pcie(msi);
1629 unsigned int index = d->hwirq / 32;
1630 unsigned long flags;
1633 spin_lock_irqsave(&msi->mask_lock, flags);
1634 value = afi_readl(pcie, AFI_MSI_EN_VEC(index));
1635 value |= BIT(d->hwirq % 32);
1636 afi_writel(pcie, value, AFI_MSI_EN_VEC(index));
1637 spin_unlock_irqrestore(&msi->mask_lock, flags);
1640 static int tegra_msi_set_affinity(struct irq_data *d, const struct cpumask *mask, bool force)
1645 static void tegra_compose_msi_msg(struct irq_data *data, struct msi_msg *msg)
1647 struct tegra_msi *msi = irq_data_get_irq_chip_data(data);
1649 msg->address_lo = lower_32_bits(msi->phys);
1650 msg->address_hi = upper_32_bits(msi->phys);
1651 msg->data = data->hwirq;
1654 static struct irq_chip tegra_msi_bottom_chip = {
1655 .name = "Tegra MSI",
1656 .irq_ack = tegra_msi_irq_ack,
1657 .irq_mask = tegra_msi_irq_mask,
1658 .irq_unmask = tegra_msi_irq_unmask,
1659 .irq_set_affinity = tegra_msi_set_affinity,
1660 .irq_compose_msi_msg = tegra_compose_msi_msg,
1663 static int tegra_msi_domain_alloc(struct irq_domain *domain, unsigned int virq,
1664 unsigned int nr_irqs, void *args)
1666 struct tegra_msi *msi = domain->host_data;
1670 mutex_lock(&msi->map_lock);
1672 hwirq = bitmap_find_free_region(msi->used, INT_PCI_MSI_NR, order_base_2(nr_irqs));
1674 mutex_unlock(&msi->map_lock);
1679 for (i = 0; i < nr_irqs; i++)
1680 irq_domain_set_info(domain, virq + i, hwirq + i,
1681 &tegra_msi_bottom_chip, domain->host_data,
1682 handle_edge_irq, NULL, NULL);
1684 tegra_cpuidle_pcie_irqs_in_use();
1689 static void tegra_msi_domain_free(struct irq_domain *domain, unsigned int virq,
1690 unsigned int nr_irqs)
1692 struct irq_data *d = irq_domain_get_irq_data(domain, virq);
1693 struct tegra_msi *msi = domain->host_data;
1695 mutex_lock(&msi->map_lock);
1697 bitmap_release_region(msi->used, d->hwirq, order_base_2(nr_irqs));
1699 mutex_unlock(&msi->map_lock);
1702 static const struct irq_domain_ops tegra_msi_domain_ops = {
1703 .alloc = tegra_msi_domain_alloc,
1704 .free = tegra_msi_domain_free,
1707 static struct msi_domain_info tegra_msi_info = {
1708 .flags = (MSI_FLAG_USE_DEF_DOM_OPS | MSI_FLAG_USE_DEF_CHIP_OPS |
1710 .chip = &tegra_msi_top_chip,
1713 static int tegra_allocate_domains(struct tegra_msi *msi)
1715 struct tegra_pcie *pcie = msi_to_pcie(msi);
1716 struct fwnode_handle *fwnode = dev_fwnode(pcie->dev);
1717 struct irq_domain *parent;
1719 parent = irq_domain_create_linear(fwnode, INT_PCI_MSI_NR,
1720 &tegra_msi_domain_ops, msi);
1722 dev_err(pcie->dev, "failed to create IRQ domain\n");
1725 irq_domain_update_bus_token(parent, DOMAIN_BUS_NEXUS);
1727 msi->domain = pci_msi_create_irq_domain(fwnode, &tegra_msi_info, parent);
1729 dev_err(pcie->dev, "failed to create MSI domain\n");
1730 irq_domain_remove(parent);
1737 static void tegra_free_domains(struct tegra_msi *msi)
1739 struct irq_domain *parent = msi->domain->parent;
1741 irq_domain_remove(msi->domain);
1742 irq_domain_remove(parent);
1745 static int tegra_pcie_msi_setup(struct tegra_pcie *pcie)
1747 struct platform_device *pdev = to_platform_device(pcie->dev);
1748 struct tegra_msi *msi = &pcie->msi;
1749 struct device *dev = pcie->dev;
1752 mutex_init(&msi->map_lock);
1753 spin_lock_init(&msi->mask_lock);
1755 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1756 err = tegra_allocate_domains(msi);
1761 err = platform_get_irq_byname(pdev, "msi");
1763 goto free_irq_domain;
1767 irq_set_chained_handler_and_data(msi->irq, tegra_pcie_msi_irq, pcie);
1769 /* Though the PCIe controller can address >32-bit address space, to
1770 * facilitate endpoints that support only 32-bit MSI target address,
1771 * the mask is set to 32-bit to make sure that MSI target address is
1772 * always a 32-bit address
1774 err = dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
1776 dev_err(dev, "failed to set DMA coherent mask: %d\n", err);
1780 msi->virt = dma_alloc_attrs(dev, PAGE_SIZE, &msi->phys, GFP_KERNEL,
1781 DMA_ATTR_NO_KERNEL_MAPPING);
1783 dev_err(dev, "failed to allocate DMA memory for MSI\n");
1791 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1793 if (IS_ENABLED(CONFIG_PCI_MSI))
1794 tegra_free_domains(msi);
1799 static void tegra_pcie_enable_msi(struct tegra_pcie *pcie)
1801 const struct tegra_pcie_soc *soc = pcie->soc;
1802 struct tegra_msi *msi = &pcie->msi;
1803 u32 reg, msi_state[INT_PCI_MSI_NR / 32];
1806 afi_writel(pcie, msi->phys >> soc->msi_base_shift, AFI_MSI_FPCI_BAR_ST);
1807 afi_writel(pcie, msi->phys, AFI_MSI_AXI_BAR_ST);
1808 /* this register is in 4K increments */
1809 afi_writel(pcie, 1, AFI_MSI_BAR_SZ);
1811 /* Restore the MSI allocation state */
1812 bitmap_to_arr32(msi_state, msi->used, INT_PCI_MSI_NR);
1813 for (i = 0; i < ARRAY_SIZE(msi_state); i++)
1814 afi_writel(pcie, msi_state[i], AFI_MSI_EN_VEC(i));
1816 /* and unmask the MSI interrupt */
1817 reg = afi_readl(pcie, AFI_INTR_MASK);
1818 reg |= AFI_INTR_MASK_MSI_MASK;
1819 afi_writel(pcie, reg, AFI_INTR_MASK);
1822 static void tegra_pcie_msi_teardown(struct tegra_pcie *pcie)
1824 struct tegra_msi *msi = &pcie->msi;
1825 unsigned int i, irq;
1827 dma_free_attrs(pcie->dev, PAGE_SIZE, msi->virt, msi->phys,
1828 DMA_ATTR_NO_KERNEL_MAPPING);
1830 for (i = 0; i < INT_PCI_MSI_NR; i++) {
1831 irq = irq_find_mapping(msi->domain, i);
1833 irq_domain_free_irqs(irq, 1);
1836 irq_set_chained_handler_and_data(msi->irq, NULL, NULL);
1838 if (IS_ENABLED(CONFIG_PCI_MSI))
1839 tegra_free_domains(msi);
1842 static int tegra_pcie_disable_msi(struct tegra_pcie *pcie)
1846 /* mask the MSI interrupt */
1847 value = afi_readl(pcie, AFI_INTR_MASK);
1848 value &= ~AFI_INTR_MASK_MSI_MASK;
1849 afi_writel(pcie, value, AFI_INTR_MASK);
1854 static void tegra_pcie_disable_interrupts(struct tegra_pcie *pcie)
1858 value = afi_readl(pcie, AFI_INTR_MASK);
1859 value &= ~AFI_INTR_MASK_INT_MASK;
1860 afi_writel(pcie, value, AFI_INTR_MASK);
1863 static int tegra_pcie_get_xbar_config(struct tegra_pcie *pcie, u32 lanes,
1866 struct device *dev = pcie->dev;
1867 struct device_node *np = dev->of_node;
1869 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
1872 dev_info(dev, "4x1, 1x1 configuration\n");
1873 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_401;
1877 dev_info(dev, "2x1, 1X1, 1x1 configuration\n");
1878 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1882 dev_info(dev, "1x1, 1x1, 1x1 configuration\n");
1883 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_111;
1887 dev_info(dev, "wrong configuration updated in DT, "
1888 "switching to default 2x1, 1x1, 1x1 "
1890 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_211;
1893 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie") ||
1894 of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
1897 dev_info(dev, "4x1, 1x1 configuration\n");
1898 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X4_X1;
1902 dev_info(dev, "2x1, 1x1 configuration\n");
1903 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_X2_X1;
1906 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
1909 dev_info(dev, "4x1, 2x1 configuration\n");
1910 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_420;
1914 dev_info(dev, "2x3 configuration\n");
1915 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_222;
1919 dev_info(dev, "4x1, 1x2 configuration\n");
1920 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_411;
1923 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
1926 dev_info(dev, "single-mode configuration\n");
1927 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_SINGLE;
1931 dev_info(dev, "dual-mode configuration\n");
1932 *xbar = AFI_PCIE_CONFIG_SM2TMS0_XBAR_CONFIG_DUAL;
1941 * Check whether a given set of supplies is available in a device tree node.
1942 * This is used to check whether the new or the legacy device tree bindings
1945 static bool of_regulator_bulk_available(struct device_node *np,
1946 struct regulator_bulk_data *supplies,
1947 unsigned int num_supplies)
1952 for (i = 0; i < num_supplies; i++) {
1953 snprintf(property, 32, "%s-supply", supplies[i].supply);
1955 if (of_find_property(np, property, NULL) == NULL)
1963 * Old versions of the device tree binding for this device used a set of power
1964 * supplies that didn't match the hardware inputs. This happened to work for a
1965 * number of cases but is not future proof. However to preserve backwards-
1966 * compatibility with old device trees, this function will try to use the old
1969 static int tegra_pcie_get_legacy_regulators(struct tegra_pcie *pcie)
1971 struct device *dev = pcie->dev;
1972 struct device_node *np = dev->of_node;
1974 if (of_device_is_compatible(np, "nvidia,tegra30-pcie"))
1975 pcie->num_supplies = 3;
1976 else if (of_device_is_compatible(np, "nvidia,tegra20-pcie"))
1977 pcie->num_supplies = 2;
1979 if (pcie->num_supplies == 0) {
1980 dev_err(dev, "device %pOF not supported in legacy mode\n", np);
1984 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
1985 sizeof(*pcie->supplies),
1987 if (!pcie->supplies)
1990 pcie->supplies[0].supply = "pex-clk";
1991 pcie->supplies[1].supply = "vdd";
1993 if (pcie->num_supplies > 2)
1994 pcie->supplies[2].supply = "avdd";
1996 return devm_regulator_bulk_get(dev, pcie->num_supplies, pcie->supplies);
2000 * Obtains the list of regulators required for a particular generation of the
2003 * This would've been nice to do simply by providing static tables for use
2004 * with the regulator_bulk_*() API, but unfortunately Tegra30 is a bit quirky
2005 * in that it has two pairs or AVDD_PEX and VDD_PEX supplies (PEXA and PEXB)
2006 * and either seems to be optional depending on which ports are being used.
2008 static int tegra_pcie_get_regulators(struct tegra_pcie *pcie, u32 lane_mask)
2010 struct device *dev = pcie->dev;
2011 struct device_node *np = dev->of_node;
2014 if (of_device_is_compatible(np, "nvidia,tegra186-pcie")) {
2015 pcie->num_supplies = 4;
2017 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2018 sizeof(*pcie->supplies),
2020 if (!pcie->supplies)
2023 pcie->supplies[i++].supply = "dvdd-pex";
2024 pcie->supplies[i++].supply = "hvdd-pex-pll";
2025 pcie->supplies[i++].supply = "hvdd-pex";
2026 pcie->supplies[i++].supply = "vddio-pexctl-aud";
2027 } else if (of_device_is_compatible(np, "nvidia,tegra210-pcie")) {
2028 pcie->num_supplies = 3;
2030 pcie->supplies = devm_kcalloc(pcie->dev, pcie->num_supplies,
2031 sizeof(*pcie->supplies),
2033 if (!pcie->supplies)
2036 pcie->supplies[i++].supply = "hvddio-pex";
2037 pcie->supplies[i++].supply = "dvddio-pex";
2038 pcie->supplies[i++].supply = "vddio-pex-ctl";
2039 } else if (of_device_is_compatible(np, "nvidia,tegra124-pcie")) {
2040 pcie->num_supplies = 4;
2042 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2043 sizeof(*pcie->supplies),
2045 if (!pcie->supplies)
2048 pcie->supplies[i++].supply = "avddio-pex";
2049 pcie->supplies[i++].supply = "dvddio-pex";
2050 pcie->supplies[i++].supply = "hvdd-pex";
2051 pcie->supplies[i++].supply = "vddio-pex-ctl";
2052 } else if (of_device_is_compatible(np, "nvidia,tegra30-pcie")) {
2053 bool need_pexa = false, need_pexb = false;
2055 /* VDD_PEXA and AVDD_PEXA supply lanes 0 to 3 */
2056 if (lane_mask & 0x0f)
2059 /* VDD_PEXB and AVDD_PEXB supply lanes 4 to 5 */
2060 if (lane_mask & 0x30)
2063 pcie->num_supplies = 4 + (need_pexa ? 2 : 0) +
2064 (need_pexb ? 2 : 0);
2066 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2067 sizeof(*pcie->supplies),
2069 if (!pcie->supplies)
2072 pcie->supplies[i++].supply = "avdd-pex-pll";
2073 pcie->supplies[i++].supply = "hvdd-pex";
2074 pcie->supplies[i++].supply = "vddio-pex-ctl";
2075 pcie->supplies[i++].supply = "avdd-plle";
2078 pcie->supplies[i++].supply = "avdd-pexa";
2079 pcie->supplies[i++].supply = "vdd-pexa";
2083 pcie->supplies[i++].supply = "avdd-pexb";
2084 pcie->supplies[i++].supply = "vdd-pexb";
2086 } else if (of_device_is_compatible(np, "nvidia,tegra20-pcie")) {
2087 pcie->num_supplies = 5;
2089 pcie->supplies = devm_kcalloc(dev, pcie->num_supplies,
2090 sizeof(*pcie->supplies),
2092 if (!pcie->supplies)
2095 pcie->supplies[0].supply = "avdd-pex";
2096 pcie->supplies[1].supply = "vdd-pex";
2097 pcie->supplies[2].supply = "avdd-pex-pll";
2098 pcie->supplies[3].supply = "avdd-plle";
2099 pcie->supplies[4].supply = "vddio-pex-clk";
2102 if (of_regulator_bulk_available(dev->of_node, pcie->supplies,
2103 pcie->num_supplies))
2104 return devm_regulator_bulk_get(dev, pcie->num_supplies,
2108 * If not all regulators are available for this new scheme, assume
2109 * that the device tree complies with an older version of the device
2112 dev_info(dev, "using legacy DT binding for power supplies\n");
2114 devm_kfree(dev, pcie->supplies);
2115 pcie->num_supplies = 0;
2117 return tegra_pcie_get_legacy_regulators(pcie);
2120 static int tegra_pcie_parse_dt(struct tegra_pcie *pcie)
2122 struct device *dev = pcie->dev;
2123 struct device_node *np = dev->of_node, *port;
2124 const struct tegra_pcie_soc *soc = pcie->soc;
2125 u32 lanes = 0, mask = 0;
2126 unsigned int lane = 0;
2129 /* parse root ports */
2130 for_each_child_of_node(np, port) {
2131 struct tegra_pcie_port *rp;
2136 err = of_pci_get_devfn(port);
2138 dev_err(dev, "failed to parse address: %d\n", err);
2142 index = PCI_SLOT(err);
2144 if (index < 1 || index > soc->num_ports) {
2145 dev_err(dev, "invalid port number: %d\n", index);
2152 err = of_property_read_u32(port, "nvidia,num-lanes", &value);
2154 dev_err(dev, "failed to parse # of lanes: %d\n",
2160 dev_err(dev, "invalid # of lanes: %u\n", value);
2165 lanes |= value << (index << 3);
2167 if (!of_device_is_available(port)) {
2172 mask |= ((1 << value) - 1) << lane;
2175 rp = devm_kzalloc(dev, sizeof(*rp), GFP_KERNEL);
2181 err = of_address_to_resource(port, 0, &rp->regs);
2183 dev_err(dev, "failed to parse address: %d\n", err);
2187 INIT_LIST_HEAD(&rp->list);
2193 rp->base = devm_pci_remap_cfg_resource(dev, &rp->regs);
2194 if (IS_ERR(rp->base))
2195 return PTR_ERR(rp->base);
2197 label = devm_kasprintf(dev, GFP_KERNEL, "pex-reset-%u", index);
2199 dev_err(dev, "failed to create reset GPIO label\n");
2204 * Returns -ENOENT if reset-gpios property is not populated
2205 * and in this case fall back to using AFI per port register
2206 * to toggle PERST# SFIO line.
2208 rp->reset_gpio = devm_gpiod_get_from_of_node(dev, port,
2212 if (IS_ERR(rp->reset_gpio)) {
2213 if (PTR_ERR(rp->reset_gpio) == -ENOENT) {
2214 rp->reset_gpio = NULL;
2216 dev_err(dev, "failed to get reset GPIO: %ld\n",
2217 PTR_ERR(rp->reset_gpio));
2218 return PTR_ERR(rp->reset_gpio);
2222 list_add_tail(&rp->list, &pcie->ports);
2225 err = tegra_pcie_get_xbar_config(pcie, lanes, &pcie->xbar_config);
2227 dev_err(dev, "invalid lane configuration\n");
2231 err = tegra_pcie_get_regulators(pcie, mask);
2243 * FIXME: If there are no PCIe cards attached, then calling this function
2244 * can result in the increase of the bootup time as there are big timeout
2247 #define TEGRA_PCIE_LINKUP_TIMEOUT 200 /* up to 1.2 seconds */
2248 static bool tegra_pcie_port_check_link(struct tegra_pcie_port *port)
2250 struct device *dev = port->pcie->dev;
2251 unsigned int retries = 3;
2252 unsigned long value;
2254 /* override presence detection */
2255 value = readl(port->base + RP_PRIV_MISC);
2256 value &= ~RP_PRIV_MISC_PRSNT_MAP_EP_ABSNT;
2257 value |= RP_PRIV_MISC_PRSNT_MAP_EP_PRSNT;
2258 writel(value, port->base + RP_PRIV_MISC);
2261 unsigned int timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2264 value = readl(port->base + RP_VEND_XP);
2266 if (value & RP_VEND_XP_DL_UP)
2269 usleep_range(1000, 2000);
2270 } while (--timeout);
2273 dev_dbg(dev, "link %u down, retrying\n", port->index);
2277 timeout = TEGRA_PCIE_LINKUP_TIMEOUT;
2280 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2282 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2285 usleep_range(1000, 2000);
2286 } while (--timeout);
2289 tegra_pcie_port_reset(port);
2290 } while (--retries);
2295 static void tegra_pcie_change_link_speed(struct tegra_pcie *pcie)
2297 struct device *dev = pcie->dev;
2298 struct tegra_pcie_port *port;
2302 list_for_each_entry(port, &pcie->ports, list) {
2304 * "Supported Link Speeds Vector" in "Link Capabilities 2"
2305 * is not supported by Tegra. tegra_pcie_change_link_speed()
2306 * is called only for Tegra chips which support Gen2.
2307 * So there no harm if supported link speed is not verified.
2309 value = readl(port->base + RP_LINK_CONTROL_STATUS_2);
2310 value &= ~PCI_EXP_LNKSTA_CLS;
2311 value |= PCI_EXP_LNKSTA_CLS_5_0GB;
2312 writel(value, port->base + RP_LINK_CONTROL_STATUS_2);
2315 * Poll until link comes back from recovery to avoid race
2318 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2320 while (ktime_before(ktime_get(), deadline)) {
2321 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2322 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2325 usleep_range(2000, 3000);
2328 if (value & PCI_EXP_LNKSTA_LT)
2329 dev_warn(dev, "PCIe port %u link is in recovery\n",
2332 /* Retrain the link */
2333 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2334 value |= PCI_EXP_LNKCTL_RL;
2335 writel(value, port->base + RP_LINK_CONTROL_STATUS);
2337 deadline = ktime_add_us(ktime_get(), LINK_RETRAIN_TIMEOUT);
2339 while (ktime_before(ktime_get(), deadline)) {
2340 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2341 if ((value & PCI_EXP_LNKSTA_LT) == 0)
2344 usleep_range(2000, 3000);
2347 if (value & PCI_EXP_LNKSTA_LT)
2348 dev_err(dev, "failed to retrain link of port %u\n",
2353 static void tegra_pcie_enable_ports(struct tegra_pcie *pcie)
2355 struct device *dev = pcie->dev;
2356 struct tegra_pcie_port *port, *tmp;
2358 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2359 dev_info(dev, "probing port %u, using %u lanes\n",
2360 port->index, port->lanes);
2362 tegra_pcie_port_enable(port);
2365 /* Start LTSSM from Tegra side */
2366 reset_control_deassert(pcie->pcie_xrst);
2368 list_for_each_entry_safe(port, tmp, &pcie->ports, list) {
2369 if (tegra_pcie_port_check_link(port))
2372 dev_info(dev, "link %u down, ignoring\n", port->index);
2374 tegra_pcie_port_disable(port);
2375 tegra_pcie_port_free(port);
2378 if (pcie->soc->has_gen2)
2379 tegra_pcie_change_link_speed(pcie);
2382 static void tegra_pcie_disable_ports(struct tegra_pcie *pcie)
2384 struct tegra_pcie_port *port, *tmp;
2386 reset_control_assert(pcie->pcie_xrst);
2388 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2389 tegra_pcie_port_disable(port);
2392 static const struct tegra_pcie_port_soc tegra20_pcie_ports[] = {
2393 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2394 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2397 static const struct tegra_pcie_soc tegra20_pcie = {
2399 .ports = tegra20_pcie_ports,
2400 .msi_base_shift = 0,
2401 .pads_pll_ctl = PADS_PLL_CTL_TEGRA20,
2402 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_DIV10,
2403 .pads_refclk_cfg0 = 0xfa5cfa5c,
2404 .has_pex_clkreq_en = false,
2405 .has_pex_bias_ctrl = false,
2406 .has_intr_prsnt_sense = false,
2407 .has_cml_clk = false,
2409 .force_pca_enable = false,
2410 .program_uphy = true,
2411 .update_clamp_threshold = false,
2412 .program_deskew_time = false,
2413 .update_fc_timer = false,
2414 .has_cache_bars = true,
2415 .ectl.enable = false,
2418 static const struct tegra_pcie_port_soc tegra30_pcie_ports[] = {
2419 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2420 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2421 { .pme.turnoff_bit = 16, .pme.ack_bit = 18 },
2424 static const struct tegra_pcie_soc tegra30_pcie = {
2426 .ports = tegra30_pcie_ports,
2427 .msi_base_shift = 8,
2428 .afi_pex2_ctrl = 0x128,
2429 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2430 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2431 .pads_refclk_cfg0 = 0xfa5cfa5c,
2432 .pads_refclk_cfg1 = 0xfa5cfa5c,
2433 .has_pex_clkreq_en = true,
2434 .has_pex_bias_ctrl = true,
2435 .has_intr_prsnt_sense = true,
2436 .has_cml_clk = true,
2438 .force_pca_enable = false,
2439 .program_uphy = true,
2440 .update_clamp_threshold = false,
2441 .program_deskew_time = false,
2442 .update_fc_timer = false,
2443 .has_cache_bars = false,
2444 .ectl.enable = false,
2447 static const struct tegra_pcie_soc tegra124_pcie = {
2449 .ports = tegra20_pcie_ports,
2450 .msi_base_shift = 8,
2451 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2452 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2453 .pads_refclk_cfg0 = 0x44ac44ac,
2454 .has_pex_clkreq_en = true,
2455 .has_pex_bias_ctrl = true,
2456 .has_intr_prsnt_sense = true,
2457 .has_cml_clk = true,
2459 .force_pca_enable = false,
2460 .program_uphy = true,
2461 .update_clamp_threshold = true,
2462 .program_deskew_time = false,
2463 .update_fc_timer = false,
2464 .has_cache_bars = false,
2465 .ectl.enable = false,
2468 static const struct tegra_pcie_soc tegra210_pcie = {
2470 .ports = tegra20_pcie_ports,
2471 .msi_base_shift = 8,
2472 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2473 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2474 .pads_refclk_cfg0 = 0x90b890b8,
2475 /* FC threshold is bit[25:18] */
2476 .update_fc_threshold = 0x01800000,
2477 .has_pex_clkreq_en = true,
2478 .has_pex_bias_ctrl = true,
2479 .has_intr_prsnt_sense = true,
2480 .has_cml_clk = true,
2482 .force_pca_enable = true,
2483 .program_uphy = true,
2484 .update_clamp_threshold = true,
2485 .program_deskew_time = true,
2486 .update_fc_timer = true,
2487 .has_cache_bars = false,
2490 .rp_ectl_2_r1 = 0x0000000f,
2491 .rp_ectl_4_r1 = 0x00000067,
2492 .rp_ectl_5_r1 = 0x55010000,
2493 .rp_ectl_6_r1 = 0x00000001,
2494 .rp_ectl_2_r2 = 0x0000008f,
2495 .rp_ectl_4_r2 = 0x000000c7,
2496 .rp_ectl_5_r2 = 0x55010000,
2497 .rp_ectl_6_r2 = 0x00000001,
2503 static const struct tegra_pcie_port_soc tegra186_pcie_ports[] = {
2504 { .pme.turnoff_bit = 0, .pme.ack_bit = 5 },
2505 { .pme.turnoff_bit = 8, .pme.ack_bit = 10 },
2506 { .pme.turnoff_bit = 12, .pme.ack_bit = 14 },
2509 static const struct tegra_pcie_soc tegra186_pcie = {
2511 .ports = tegra186_pcie_ports,
2512 .msi_base_shift = 8,
2513 .afi_pex2_ctrl = 0x19c,
2514 .pads_pll_ctl = PADS_PLL_CTL_TEGRA30,
2515 .tx_ref_sel = PADS_PLL_CTL_TXCLKREF_BUF_EN,
2516 .pads_refclk_cfg0 = 0x80b880b8,
2517 .pads_refclk_cfg1 = 0x000480b8,
2518 .has_pex_clkreq_en = true,
2519 .has_pex_bias_ctrl = true,
2520 .has_intr_prsnt_sense = true,
2521 .has_cml_clk = false,
2523 .force_pca_enable = false,
2524 .program_uphy = false,
2525 .update_clamp_threshold = false,
2526 .program_deskew_time = false,
2527 .update_fc_timer = false,
2528 .has_cache_bars = false,
2529 .ectl.enable = false,
2532 static const struct of_device_id tegra_pcie_of_match[] = {
2533 { .compatible = "nvidia,tegra186-pcie", .data = &tegra186_pcie },
2534 { .compatible = "nvidia,tegra210-pcie", .data = &tegra210_pcie },
2535 { .compatible = "nvidia,tegra124-pcie", .data = &tegra124_pcie },
2536 { .compatible = "nvidia,tegra30-pcie", .data = &tegra30_pcie },
2537 { .compatible = "nvidia,tegra20-pcie", .data = &tegra20_pcie },
2540 MODULE_DEVICE_TABLE(of, tegra_pcie_of_match);
2542 static void *tegra_pcie_ports_seq_start(struct seq_file *s, loff_t *pos)
2544 struct tegra_pcie *pcie = s->private;
2546 if (list_empty(&pcie->ports))
2549 seq_printf(s, "Index Status\n");
2551 return seq_list_start(&pcie->ports, *pos);
2554 static void *tegra_pcie_ports_seq_next(struct seq_file *s, void *v, loff_t *pos)
2556 struct tegra_pcie *pcie = s->private;
2558 return seq_list_next(v, &pcie->ports, pos);
2561 static void tegra_pcie_ports_seq_stop(struct seq_file *s, void *v)
2565 static int tegra_pcie_ports_seq_show(struct seq_file *s, void *v)
2567 bool up = false, active = false;
2568 struct tegra_pcie_port *port;
2571 port = list_entry(v, struct tegra_pcie_port, list);
2573 value = readl(port->base + RP_VEND_XP);
2575 if (value & RP_VEND_XP_DL_UP)
2578 value = readl(port->base + RP_LINK_CONTROL_STATUS);
2580 if (value & RP_LINK_CONTROL_STATUS_DL_LINK_ACTIVE)
2583 seq_printf(s, "%2u ", port->index);
2586 seq_printf(s, "up");
2590 seq_printf(s, ", ");
2592 seq_printf(s, "active");
2595 seq_printf(s, "\n");
2599 static const struct seq_operations tegra_pcie_ports_sops = {
2600 .start = tegra_pcie_ports_seq_start,
2601 .next = tegra_pcie_ports_seq_next,
2602 .stop = tegra_pcie_ports_seq_stop,
2603 .show = tegra_pcie_ports_seq_show,
2606 DEFINE_SEQ_ATTRIBUTE(tegra_pcie_ports);
2608 static void tegra_pcie_debugfs_exit(struct tegra_pcie *pcie)
2610 debugfs_remove_recursive(pcie->debugfs);
2611 pcie->debugfs = NULL;
2614 static void tegra_pcie_debugfs_init(struct tegra_pcie *pcie)
2616 pcie->debugfs = debugfs_create_dir("pcie", NULL);
2618 debugfs_create_file("ports", S_IFREG | S_IRUGO, pcie->debugfs, pcie,
2619 &tegra_pcie_ports_fops);
2622 static int tegra_pcie_probe(struct platform_device *pdev)
2624 struct device *dev = &pdev->dev;
2625 struct pci_host_bridge *host;
2626 struct tegra_pcie *pcie;
2629 host = devm_pci_alloc_host_bridge(dev, sizeof(*pcie));
2633 pcie = pci_host_bridge_priv(host);
2634 host->sysdata = pcie;
2635 platform_set_drvdata(pdev, pcie);
2637 pcie->soc = of_device_get_match_data(dev);
2638 INIT_LIST_HEAD(&pcie->ports);
2641 err = tegra_pcie_parse_dt(pcie);
2645 err = tegra_pcie_get_resources(pcie);
2647 dev_err(dev, "failed to request resources: %d\n", err);
2651 err = tegra_pcie_msi_setup(pcie);
2653 dev_err(dev, "failed to enable MSI support: %d\n", err);
2657 pm_runtime_enable(pcie->dev);
2658 err = pm_runtime_get_sync(pcie->dev);
2660 dev_err(dev, "fail to enable pcie controller: %d\n", err);
2661 goto pm_runtime_put;
2664 host->ops = &tegra_pcie_ops;
2665 host->map_irq = tegra_pcie_map_irq;
2667 err = pci_host_probe(host);
2669 dev_err(dev, "failed to register host: %d\n", err);
2670 goto pm_runtime_put;
2673 if (IS_ENABLED(CONFIG_DEBUG_FS))
2674 tegra_pcie_debugfs_init(pcie);
2679 pm_runtime_put_sync(pcie->dev);
2680 pm_runtime_disable(pcie->dev);
2681 tegra_pcie_msi_teardown(pcie);
2683 tegra_pcie_put_resources(pcie);
2687 static int tegra_pcie_remove(struct platform_device *pdev)
2689 struct tegra_pcie *pcie = platform_get_drvdata(pdev);
2690 struct pci_host_bridge *host = pci_host_bridge_from_priv(pcie);
2691 struct tegra_pcie_port *port, *tmp;
2693 if (IS_ENABLED(CONFIG_DEBUG_FS))
2694 tegra_pcie_debugfs_exit(pcie);
2696 pci_stop_root_bus(host->bus);
2697 pci_remove_root_bus(host->bus);
2698 pm_runtime_put_sync(pcie->dev);
2699 pm_runtime_disable(pcie->dev);
2701 if (IS_ENABLED(CONFIG_PCI_MSI))
2702 tegra_pcie_msi_teardown(pcie);
2704 tegra_pcie_put_resources(pcie);
2706 list_for_each_entry_safe(port, tmp, &pcie->ports, list)
2707 tegra_pcie_port_free(port);
2712 static int __maybe_unused tegra_pcie_pm_suspend(struct device *dev)
2714 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2715 struct tegra_pcie_port *port;
2718 list_for_each_entry(port, &pcie->ports, list)
2719 tegra_pcie_pme_turnoff(port);
2721 tegra_pcie_disable_ports(pcie);
2724 * AFI_INTR is unmasked in tegra_pcie_enable_controller(), mask it to
2725 * avoid unwanted interrupts raised by AFI after pex_rst is asserted.
2727 tegra_pcie_disable_interrupts(pcie);
2729 if (pcie->soc->program_uphy) {
2730 err = tegra_pcie_phy_power_off(pcie);
2732 dev_err(dev, "failed to power off PHY(s): %d\n", err);
2735 reset_control_assert(pcie->pex_rst);
2736 clk_disable_unprepare(pcie->pex_clk);
2738 if (IS_ENABLED(CONFIG_PCI_MSI))
2739 tegra_pcie_disable_msi(pcie);
2741 pinctrl_pm_select_idle_state(dev);
2742 tegra_pcie_power_off(pcie);
2747 static int __maybe_unused tegra_pcie_pm_resume(struct device *dev)
2749 struct tegra_pcie *pcie = dev_get_drvdata(dev);
2752 err = tegra_pcie_power_on(pcie);
2754 dev_err(dev, "tegra pcie power on fail: %d\n", err);
2758 err = pinctrl_pm_select_default_state(dev);
2760 dev_err(dev, "failed to disable PCIe IO DPD: %d\n", err);
2764 tegra_pcie_enable_controller(pcie);
2765 tegra_pcie_setup_translations(pcie);
2767 if (IS_ENABLED(CONFIG_PCI_MSI))
2768 tegra_pcie_enable_msi(pcie);
2770 err = clk_prepare_enable(pcie->pex_clk);
2772 dev_err(dev, "failed to enable PEX clock: %d\n", err);
2773 goto pex_dpd_enable;
2776 reset_control_deassert(pcie->pex_rst);
2778 if (pcie->soc->program_uphy) {
2779 err = tegra_pcie_phy_power_on(pcie);
2781 dev_err(dev, "failed to power on PHY(s): %d\n", err);
2782 goto disable_pex_clk;
2786 tegra_pcie_apply_pad_settings(pcie);
2787 tegra_pcie_enable_ports(pcie);
2792 reset_control_assert(pcie->pex_rst);
2793 clk_disable_unprepare(pcie->pex_clk);
2795 pinctrl_pm_select_idle_state(dev);
2797 tegra_pcie_power_off(pcie);
2802 static const struct dev_pm_ops tegra_pcie_pm_ops = {
2803 SET_RUNTIME_PM_OPS(tegra_pcie_pm_suspend, tegra_pcie_pm_resume, NULL)
2804 SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(tegra_pcie_pm_suspend,
2805 tegra_pcie_pm_resume)
2808 static struct platform_driver tegra_pcie_driver = {
2810 .name = "tegra-pcie",
2811 .of_match_table = tegra_pcie_of_match,
2812 .suppress_bind_attrs = true,
2813 .pm = &tegra_pcie_pm_ops,
2815 .probe = tegra_pcie_probe,
2816 .remove = tegra_pcie_remove,
2818 module_platform_driver(tegra_pcie_driver);
2819 MODULE_LICENSE("GPL");