1 // SPDX-License-Identifier: GPL-2.0+
3 * PCIe host controller driver for Tegra194 SoC
5 * Copyright (C) 2019 NVIDIA Corporation.
7 * Author: Vidya Sagar <vidyas@nvidia.com>
10 #include <linux/clk.h>
11 #include <linux/debugfs.h>
12 #include <linux/delay.h>
13 #include <linux/gpio.h>
14 #include <linux/gpio/consumer.h>
15 #include <linux/interrupt.h>
16 #include <linux/iopoll.h>
17 #include <linux/kernel.h>
18 #include <linux/module.h>
20 #include <linux/of_device.h>
21 #include <linux/of_gpio.h>
22 #include <linux/of_irq.h>
23 #include <linux/of_pci.h>
24 #include <linux/pci.h>
25 #include <linux/phy/phy.h>
26 #include <linux/pinctrl/consumer.h>
27 #include <linux/platform_device.h>
28 #include <linux/pm_runtime.h>
29 #include <linux/random.h>
30 #include <linux/reset.h>
31 #include <linux/resource.h>
32 #include <linux/types.h>
33 #include "pcie-designware.h"
34 #include <soc/tegra/bpmp.h>
35 #include <soc/tegra/bpmp-abi.h>
36 #include "../../pci.h"
38 #define APPL_PINMUX 0x0
39 #define APPL_PINMUX_PEX_RST BIT(0)
40 #define APPL_PINMUX_CLKREQ_OVERRIDE_EN BIT(2)
41 #define APPL_PINMUX_CLKREQ_OVERRIDE BIT(3)
42 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN BIT(4)
43 #define APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE BIT(5)
46 #define APPL_CTRL_SYS_PRE_DET_STATE BIT(6)
47 #define APPL_CTRL_LTSSM_EN BIT(7)
48 #define APPL_CTRL_HW_HOT_RST_EN BIT(20)
49 #define APPL_CTRL_HW_HOT_RST_MODE_MASK GENMASK(1, 0)
50 #define APPL_CTRL_HW_HOT_RST_MODE_SHIFT 22
51 #define APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST 0x1
53 #define APPL_INTR_EN_L0_0 0x8
54 #define APPL_INTR_EN_L0_0_LINK_STATE_INT_EN BIT(0)
55 #define APPL_INTR_EN_L0_0_MSI_RCV_INT_EN BIT(4)
56 #define APPL_INTR_EN_L0_0_INT_INT_EN BIT(8)
57 #define APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN BIT(15)
58 #define APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN BIT(19)
59 #define APPL_INTR_EN_L0_0_SYS_INTR_EN BIT(30)
60 #define APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN BIT(31)
62 #define APPL_INTR_STATUS_L0 0xC
63 #define APPL_INTR_STATUS_L0_LINK_STATE_INT BIT(0)
64 #define APPL_INTR_STATUS_L0_INT_INT BIT(8)
65 #define APPL_INTR_STATUS_L0_PCI_CMD_EN_INT BIT(15)
66 #define APPL_INTR_STATUS_L0_PEX_RST_INT BIT(16)
67 #define APPL_INTR_STATUS_L0_CDM_REG_CHK_INT BIT(18)
69 #define APPL_INTR_EN_L1_0_0 0x1C
70 #define APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN BIT(1)
71 #define APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN BIT(3)
72 #define APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN BIT(30)
74 #define APPL_INTR_STATUS_L1_0_0 0x20
75 #define APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED BIT(1)
76 #define APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED BIT(3)
77 #define APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE BIT(30)
79 #define APPL_INTR_STATUS_L1_1 0x2C
80 #define APPL_INTR_STATUS_L1_2 0x30
81 #define APPL_INTR_STATUS_L1_3 0x34
82 #define APPL_INTR_STATUS_L1_6 0x3C
83 #define APPL_INTR_STATUS_L1_7 0x40
84 #define APPL_INTR_STATUS_L1_15_CFG_BME_CHGED BIT(1)
86 #define APPL_INTR_EN_L1_8_0 0x44
87 #define APPL_INTR_EN_L1_8_BW_MGT_INT_EN BIT(2)
88 #define APPL_INTR_EN_L1_8_AUTO_BW_INT_EN BIT(3)
89 #define APPL_INTR_EN_L1_8_INTX_EN BIT(11)
90 #define APPL_INTR_EN_L1_8_AER_INT_EN BIT(15)
92 #define APPL_INTR_STATUS_L1_8_0 0x4C
93 #define APPL_INTR_STATUS_L1_8_0_EDMA_INT_MASK GENMASK(11, 6)
94 #define APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS BIT(2)
95 #define APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS BIT(3)
97 #define APPL_INTR_STATUS_L1_9 0x54
98 #define APPL_INTR_STATUS_L1_10 0x58
99 #define APPL_INTR_STATUS_L1_11 0x64
100 #define APPL_INTR_STATUS_L1_13 0x74
101 #define APPL_INTR_STATUS_L1_14 0x78
102 #define APPL_INTR_STATUS_L1_15 0x7C
103 #define APPL_INTR_STATUS_L1_17 0x88
105 #define APPL_INTR_EN_L1_18 0x90
106 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMPLT BIT(2)
107 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
108 #define APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
110 #define APPL_INTR_STATUS_L1_18 0x94
111 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT BIT(2)
112 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR BIT(1)
113 #define APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR BIT(0)
115 #define APPL_MSI_CTRL_1 0xAC
117 #define APPL_MSI_CTRL_2 0xB0
119 #define APPL_LEGACY_INTX 0xB8
121 #define APPL_LTR_MSG_1 0xC4
122 #define LTR_MSG_REQ BIT(15)
123 #define LTR_MST_NO_SNOOP_SHIFT 16
125 #define APPL_LTR_MSG_2 0xC8
126 #define APPL_LTR_MSG_2_LTR_MSG_REQ_STATE BIT(3)
128 #define APPL_LINK_STATUS 0xCC
129 #define APPL_LINK_STATUS_RDLH_LINK_UP BIT(0)
131 #define APPL_DEBUG 0xD0
132 #define APPL_DEBUG_PM_LINKST_IN_L2_LAT BIT(21)
133 #define APPL_DEBUG_PM_LINKST_IN_L0 0x11
134 #define APPL_DEBUG_LTSSM_STATE_MASK GENMASK(8, 3)
135 #define APPL_DEBUG_LTSSM_STATE_SHIFT 3
136 #define LTSSM_STATE_PRE_DETECT 5
138 #define APPL_RADM_STATUS 0xE4
139 #define APPL_PM_XMT_TURNOFF_STATE BIT(0)
141 #define APPL_DM_TYPE 0x100
142 #define APPL_DM_TYPE_MASK GENMASK(3, 0)
143 #define APPL_DM_TYPE_RP 0x4
144 #define APPL_DM_TYPE_EP 0x0
146 #define APPL_CFG_BASE_ADDR 0x104
147 #define APPL_CFG_BASE_ADDR_MASK GENMASK(31, 12)
149 #define APPL_CFG_IATU_DMA_BASE_ADDR 0x108
150 #define APPL_CFG_IATU_DMA_BASE_ADDR_MASK GENMASK(31, 18)
152 #define APPL_CFG_MISC 0x110
153 #define APPL_CFG_MISC_SLV_EP_MODE BIT(14)
154 #define APPL_CFG_MISC_ARCACHE_MASK GENMASK(13, 10)
155 #define APPL_CFG_MISC_ARCACHE_SHIFT 10
156 #define APPL_CFG_MISC_ARCACHE_VAL 3
158 #define APPL_CFG_SLCG_OVERRIDE 0x114
159 #define APPL_CFG_SLCG_OVERRIDE_SLCG_EN_MASTER BIT(0)
161 #define APPL_CAR_RESET_OVRD 0x12C
162 #define APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N BIT(0)
164 #define IO_BASE_IO_DECODE BIT(0)
165 #define IO_BASE_IO_DECODE_BIT8 BIT(8)
167 #define CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE BIT(0)
168 #define CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE BIT(16)
170 #define CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF 0x718
171 #define CFG_TIMER_CTRL_ACK_NAK_SHIFT (19)
173 #define EVENT_COUNTER_ALL_CLEAR 0x3
174 #define EVENT_COUNTER_ENABLE_ALL 0x7
175 #define EVENT_COUNTER_ENABLE_SHIFT 2
176 #define EVENT_COUNTER_EVENT_SEL_MASK GENMASK(7, 0)
177 #define EVENT_COUNTER_EVENT_SEL_SHIFT 16
178 #define EVENT_COUNTER_EVENT_Tx_L0S 0x2
179 #define EVENT_COUNTER_EVENT_Rx_L0S 0x3
180 #define EVENT_COUNTER_EVENT_L1 0x5
181 #define EVENT_COUNTER_EVENT_L1_1 0x7
182 #define EVENT_COUNTER_EVENT_L1_2 0x8
183 #define EVENT_COUNTER_GROUP_SEL_SHIFT 24
184 #define EVENT_COUNTER_GROUP_5 0x5
186 #define PORT_LOGIC_ACK_F_ASPM_CTRL 0x70C
187 #define ENTER_ASPM BIT(30)
188 #define L0S_ENTRANCE_LAT_SHIFT 24
189 #define L0S_ENTRANCE_LAT_MASK GENMASK(26, 24)
190 #define L1_ENTRANCE_LAT_SHIFT 27
191 #define L1_ENTRANCE_LAT_MASK GENMASK(29, 27)
192 #define N_FTS_SHIFT 8
193 #define N_FTS_MASK GENMASK(7, 0)
196 #define PORT_LOGIC_GEN2_CTRL 0x80C
197 #define PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE BIT(17)
198 #define FTS_MASK GENMASK(7, 0)
201 #define PORT_LOGIC_MSI_CTRL_INT_0_EN 0x828
203 #define GEN3_EQ_CONTROL_OFF 0x8a8
204 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT 8
205 #define GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK GENMASK(23, 8)
206 #define GEN3_EQ_CONTROL_OFF_FB_MODE_MASK GENMASK(3, 0)
208 #define GEN3_RELATED_OFF 0x890
209 #define GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL BIT(0)
210 #define GEN3_RELATED_OFF_GEN3_EQ_DISABLE BIT(16)
211 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT 24
212 #define GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK GENMASK(25, 24)
214 #define PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT 0x8D0
215 #define AMBA_ERROR_RESPONSE_CRS_SHIFT 3
216 #define AMBA_ERROR_RESPONSE_CRS_MASK GENMASK(1, 0)
217 #define AMBA_ERROR_RESPONSE_CRS_OKAY 0
218 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFFFFFF 1
219 #define AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 2
221 #define MSIX_ADDR_MATCH_LOW_OFF 0x940
222 #define MSIX_ADDR_MATCH_LOW_OFF_EN BIT(0)
223 #define MSIX_ADDR_MATCH_LOW_OFF_MASK GENMASK(31, 2)
225 #define MSIX_ADDR_MATCH_HIGH_OFF 0x944
226 #define MSIX_ADDR_MATCH_HIGH_OFF_MASK GENMASK(31, 0)
228 #define PORT_LOGIC_MSIX_DOORBELL 0x948
230 #define CAP_SPCIE_CAP_OFF 0x154
231 #define CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK GENMASK(3, 0)
232 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK GENMASK(11, 8)
233 #define CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT 8
235 #define PME_ACK_TIMEOUT 10000
237 #define LTSSM_TIMEOUT 50000 /* 50ms */
239 #define GEN3_GEN4_EQ_PRESET_INIT 5
241 #define GEN1_CORE_CLK_FREQ 62500000
242 #define GEN2_CORE_CLK_FREQ 125000000
243 #define GEN3_CORE_CLK_FREQ 250000000
244 #define GEN4_CORE_CLK_FREQ 500000000
246 #define LTR_MSG_TIMEOUT (100 * 1000)
248 #define PERST_DEBOUNCE_TIME (5 * 1000)
250 #define EP_STATE_DISABLED 0
251 #define EP_STATE_ENABLED 1
253 static const unsigned int pcie_gen_freq[] = {
260 static const u32 event_cntr_ctrl_offset[] = {
269 static const u32 event_cntr_data_offset[] = {
278 struct tegra_pcie_dw {
280 struct resource *appl_res;
281 struct resource *dbi_res;
282 struct resource *atu_dma_res;
283 void __iomem *appl_base;
284 struct clk *core_clk;
285 struct reset_control *core_apb_rst;
286 struct reset_control *core_rst;
288 struct tegra_bpmp *bpmp;
290 enum dw_pcie_device_mode mode;
292 bool supports_clkreq;
293 bool enable_cdm_check;
295 bool update_fc_fixup;
301 u32 cfg_link_cap_l1sub;
305 u32 aspm_l0s_enter_lat;
307 struct regulator *pex_ctl_supply;
308 struct regulator *slot_ctl_3v3;
309 struct regulator *slot_ctl_12v;
311 unsigned int phy_count;
314 struct dentry *debugfs;
316 /* Endpoint mode specific */
317 struct gpio_desc *pex_rst_gpiod;
318 struct gpio_desc *pex_refclk_sel_gpiod;
319 unsigned int pex_rst_irq;
323 struct tegra_pcie_dw_of_data {
324 enum dw_pcie_device_mode mode;
327 static inline struct tegra_pcie_dw *to_tegra_pcie(struct dw_pcie *pci)
329 return container_of(pci, struct tegra_pcie_dw, pci);
332 static inline void appl_writel(struct tegra_pcie_dw *pcie, const u32 value,
335 writel_relaxed(value, pcie->appl_base + reg);
338 static inline u32 appl_readl(struct tegra_pcie_dw *pcie, const u32 reg)
340 return readl_relaxed(pcie->appl_base + reg);
343 struct tegra_pcie_soc {
344 enum dw_pcie_device_mode mode;
347 static void apply_bad_link_workaround(struct pcie_port *pp)
349 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
350 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
351 u32 current_link_width;
355 * NOTE:- Since this scenario is uncommon and link as such is not
356 * stable anyway, not waiting to confirm if link is really
357 * transitioning to Gen-2 speed
359 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
360 if (val & PCI_EXP_LNKSTA_LBMS) {
361 current_link_width = (val & PCI_EXP_LNKSTA_NLW) >>
362 PCI_EXP_LNKSTA_NLW_SHIFT;
363 if (pcie->init_link_width > current_link_width) {
364 dev_warn(pci->dev, "PCIe link is bad, width reduced\n");
365 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
367 val &= ~PCI_EXP_LNKCTL2_TLS;
368 val |= PCI_EXP_LNKCTL2_TLS_2_5GT;
369 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
370 PCI_EXP_LNKCTL2, val);
372 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
374 val |= PCI_EXP_LNKCTL_RL;
375 dw_pcie_writew_dbi(pci, pcie->pcie_cap_base +
376 PCI_EXP_LNKCTL, val);
381 static irqreturn_t tegra_pcie_rp_irq_handler(int irq, void *arg)
383 struct tegra_pcie_dw *pcie = arg;
384 struct dw_pcie *pci = &pcie->pci;
385 struct pcie_port *pp = &pci->pp;
389 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
390 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
391 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
392 if (val & APPL_INTR_STATUS_L1_0_0_LINK_REQ_RST_NOT_CHGED) {
393 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
395 /* SBR & Surprise Link Down WAR */
396 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
397 val &= ~APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
398 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
400 val = appl_readl(pcie, APPL_CAR_RESET_OVRD);
401 val |= APPL_CAR_RESET_OVRD_CYA_OVERRIDE_CORE_RST_N;
402 appl_writel(pcie, val, APPL_CAR_RESET_OVRD);
404 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
405 val |= PORT_LOGIC_GEN2_CTRL_DIRECT_SPEED_CHANGE;
406 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
410 if (val & APPL_INTR_STATUS_L0_INT_INT) {
411 val = appl_readl(pcie, APPL_INTR_STATUS_L1_8_0);
412 if (val & APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS) {
414 APPL_INTR_STATUS_L1_8_0_AUTO_BW_INT_STS,
415 APPL_INTR_STATUS_L1_8_0);
416 apply_bad_link_workaround(pp);
418 if (val & APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS) {
420 APPL_INTR_STATUS_L1_8_0_BW_MGT_INT_STS,
421 APPL_INTR_STATUS_L1_8_0);
423 val_w = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base +
425 dev_dbg(pci->dev, "Link Speed : Gen-%u\n", val_w &
430 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
431 if (val & APPL_INTR_STATUS_L0_CDM_REG_CHK_INT) {
432 val = appl_readl(pcie, APPL_INTR_STATUS_L1_18);
433 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS);
434 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMPLT) {
435 dev_info(pci->dev, "CDM check complete\n");
436 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPLETE;
438 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_CMP_ERR) {
439 dev_err(pci->dev, "CDM comparison mismatch\n");
440 tmp |= PCIE_PL_CHK_REG_CHK_REG_COMPARISON_ERROR;
442 if (val & APPL_INTR_STATUS_L1_18_CDM_REG_CHK_LOGIC_ERR) {
443 dev_err(pci->dev, "CDM Logic error\n");
444 tmp |= PCIE_PL_CHK_REG_CHK_REG_LOGIC_ERROR;
446 dw_pcie_writel_dbi(pci, PCIE_PL_CHK_REG_CONTROL_STATUS, tmp);
447 tmp = dw_pcie_readl_dbi(pci, PCIE_PL_CHK_REG_ERR_ADDR);
448 dev_err(pci->dev, "CDM Error Address Offset = 0x%08X\n", tmp);
454 static void pex_ep_event_hot_rst_done(struct tegra_pcie_dw *pcie)
458 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
459 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
460 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
461 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
462 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
463 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
464 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
465 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
466 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
467 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
468 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
469 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
470 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
471 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
472 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
473 appl_writel(pcie, 0xFFFFFFFF, APPL_MSI_CTRL_2);
475 val = appl_readl(pcie, APPL_CTRL);
476 val |= APPL_CTRL_LTSSM_EN;
477 appl_writel(pcie, val, APPL_CTRL);
480 static irqreturn_t tegra_pcie_ep_irq_thread(int irq, void *arg)
482 struct tegra_pcie_dw *pcie = arg;
483 struct dw_pcie *pci = &pcie->pci;
486 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
488 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
490 /* If EP doesn't advertise L1SS, just return */
491 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
492 if (!(val & (PCI_L1SS_CAP_ASPM_L1_1 | PCI_L1SS_CAP_ASPM_L1_2)))
495 /* Check if BME is set to '1' */
496 val = dw_pcie_readl_dbi(pci, PCI_COMMAND);
497 if (val & PCI_COMMAND_MASTER) {
500 /* 110us for both snoop and no-snoop */
501 val = 110 | (2 << PCI_LTR_SCALE_SHIFT) | LTR_MSG_REQ;
502 val |= (val << LTR_MST_NO_SNOOP_SHIFT);
503 appl_writel(pcie, val, APPL_LTR_MSG_1);
505 /* Send LTR upstream */
506 val = appl_readl(pcie, APPL_LTR_MSG_2);
507 val |= APPL_LTR_MSG_2_LTR_MSG_REQ_STATE;
508 appl_writel(pcie, val, APPL_LTR_MSG_2);
510 timeout = ktime_add_us(ktime_get(), LTR_MSG_TIMEOUT);
512 val = appl_readl(pcie, APPL_LTR_MSG_2);
513 if (!(val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE))
515 if (ktime_after(ktime_get(), timeout))
517 usleep_range(1000, 1100);
519 if (val & APPL_LTR_MSG_2_LTR_MSG_REQ_STATE)
520 dev_err(pcie->dev, "Failed to send LTR message\n");
526 static irqreturn_t tegra_pcie_ep_hard_irq(int irq, void *arg)
528 struct tegra_pcie_dw *pcie = arg;
529 struct dw_pcie_ep *ep = &pcie->pci.ep;
533 val = appl_readl(pcie, APPL_INTR_STATUS_L0);
534 if (val & APPL_INTR_STATUS_L0_LINK_STATE_INT) {
535 val = appl_readl(pcie, APPL_INTR_STATUS_L1_0_0);
536 appl_writel(pcie, val, APPL_INTR_STATUS_L1_0_0);
538 if (val & APPL_INTR_STATUS_L1_0_0_HOT_RESET_DONE)
539 pex_ep_event_hot_rst_done(pcie);
541 if (val & APPL_INTR_STATUS_L1_0_0_RDLH_LINK_UP_CHGED) {
542 tmp = appl_readl(pcie, APPL_LINK_STATUS);
543 if (tmp & APPL_LINK_STATUS_RDLH_LINK_UP) {
544 dev_dbg(pcie->dev, "Link is up with Host\n");
545 dw_pcie_ep_linkup(ep);
552 if (val & APPL_INTR_STATUS_L0_PCI_CMD_EN_INT) {
553 val = appl_readl(pcie, APPL_INTR_STATUS_L1_15);
554 appl_writel(pcie, val, APPL_INTR_STATUS_L1_15);
556 if (val & APPL_INTR_STATUS_L1_15_CFG_BME_CHGED)
557 return IRQ_WAKE_THREAD;
563 dev_warn(pcie->dev, "Random interrupt (STATUS = 0x%08X)\n",
565 appl_writel(pcie, val, APPL_INTR_STATUS_L0);
571 static int tegra_pcie_dw_rd_own_conf(struct pci_bus *bus, u32 devfn, int where,
575 * This is an endpoint mode specific register happen to appear even
576 * when controller is operating in root port mode and system hangs
577 * when it is accessed with link being in ASPM-L1 state.
578 * So skip accessing it altogether
580 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL) {
582 return PCIBIOS_SUCCESSFUL;
585 return pci_generic_config_read(bus, devfn, where, size, val);
588 static int tegra_pcie_dw_wr_own_conf(struct pci_bus *bus, u32 devfn, int where,
592 * This is an endpoint mode specific register happen to appear even
593 * when controller is operating in root port mode and system hangs
594 * when it is accessed with link being in ASPM-L1 state.
595 * So skip accessing it altogether
597 if (!PCI_SLOT(devfn) && where == PORT_LOGIC_MSIX_DOORBELL)
598 return PCIBIOS_SUCCESSFUL;
600 return pci_generic_config_write(bus, devfn, where, size, val);
603 static struct pci_ops tegra_pci_ops = {
604 .map_bus = dw_pcie_own_conf_map_bus,
605 .read = tegra_pcie_dw_rd_own_conf,
606 .write = tegra_pcie_dw_wr_own_conf,
609 #if defined(CONFIG_PCIEASPM)
610 static void disable_aspm_l11(struct tegra_pcie_dw *pcie)
614 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
615 val &= ~PCI_L1SS_CAP_ASPM_L1_1;
616 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
619 static void disable_aspm_l12(struct tegra_pcie_dw *pcie)
623 val = dw_pcie_readl_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub);
624 val &= ~PCI_L1SS_CAP_ASPM_L1_2;
625 dw_pcie_writel_dbi(&pcie->pci, pcie->cfg_link_cap_l1sub, val);
628 static inline u32 event_counter_prog(struct tegra_pcie_dw *pcie, u32 event)
632 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid]);
633 val &= ~(EVENT_COUNTER_EVENT_SEL_MASK << EVENT_COUNTER_EVENT_SEL_SHIFT);
634 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
635 val |= event << EVENT_COUNTER_EVENT_SEL_SHIFT;
636 val |= EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
637 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
638 val = dw_pcie_readl_dbi(&pcie->pci, event_cntr_data_offset[pcie->cid]);
643 static int aspm_state_cnt(struct seq_file *s, void *data)
645 struct tegra_pcie_dw *pcie = (struct tegra_pcie_dw *)
646 dev_get_drvdata(s->private);
649 seq_printf(s, "Tx L0s entry count : %u\n",
650 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Tx_L0S));
652 seq_printf(s, "Rx L0s entry count : %u\n",
653 event_counter_prog(pcie, EVENT_COUNTER_EVENT_Rx_L0S));
655 seq_printf(s, "Link L1 entry count : %u\n",
656 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1));
658 seq_printf(s, "Link L1.1 entry count : %u\n",
659 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_1));
661 seq_printf(s, "Link L1.2 entry count : %u\n",
662 event_counter_prog(pcie, EVENT_COUNTER_EVENT_L1_2));
664 /* Clear all counters */
665 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid],
666 EVENT_COUNTER_ALL_CLEAR);
668 /* Re-enable counting */
669 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
670 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
671 dw_pcie_writel_dbi(&pcie->pci, event_cntr_ctrl_offset[pcie->cid], val);
676 static void init_host_aspm(struct tegra_pcie_dw *pcie)
678 struct dw_pcie *pci = &pcie->pci;
681 val = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_L1SS);
682 pcie->cfg_link_cap_l1sub = val + PCI_L1SS_CAP;
684 /* Enable ASPM counters */
685 val = EVENT_COUNTER_ENABLE_ALL << EVENT_COUNTER_ENABLE_SHIFT;
686 val |= EVENT_COUNTER_GROUP_5 << EVENT_COUNTER_GROUP_SEL_SHIFT;
687 dw_pcie_writel_dbi(pci, event_cntr_ctrl_offset[pcie->cid], val);
689 /* Program T_cmrt and T_pwr_on values */
690 val = dw_pcie_readl_dbi(pci, pcie->cfg_link_cap_l1sub);
691 val &= ~(PCI_L1SS_CAP_CM_RESTORE_TIME | PCI_L1SS_CAP_P_PWR_ON_VALUE);
692 val |= (pcie->aspm_cmrt << 8);
693 val |= (pcie->aspm_pwr_on_t << 19);
694 dw_pcie_writel_dbi(pci, pcie->cfg_link_cap_l1sub, val);
696 /* Program L0s and L1 entrance latencies */
697 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
698 val &= ~L0S_ENTRANCE_LAT_MASK;
699 val |= (pcie->aspm_l0s_enter_lat << L0S_ENTRANCE_LAT_SHIFT);
701 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
704 static int init_debugfs(struct tegra_pcie_dw *pcie)
708 d = debugfs_create_devm_seqfile(pcie->dev, "aspm_state_cnt",
709 pcie->debugfs, aspm_state_cnt);
710 if (IS_ERR_OR_NULL(d))
712 "Failed to create debugfs file \"aspm_state_cnt\"\n");
717 static inline void disable_aspm_l12(struct tegra_pcie_dw *pcie) { return; }
718 static inline void disable_aspm_l11(struct tegra_pcie_dw *pcie) { return; }
719 static inline void init_host_aspm(struct tegra_pcie_dw *pcie) { return; }
720 static inline int init_debugfs(struct tegra_pcie_dw *pcie) { return 0; }
723 static void tegra_pcie_enable_system_interrupts(struct pcie_port *pp)
725 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
726 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
730 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
731 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
732 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
734 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
735 val |= APPL_INTR_EN_L1_0_0_LINK_REQ_RST_NOT_INT_EN;
736 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
738 if (pcie->enable_cdm_check) {
739 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
740 val |= APPL_INTR_EN_L0_0_CDM_REG_CHK_INT_EN;
741 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
743 val = appl_readl(pcie, APPL_INTR_EN_L1_18);
744 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_CMP_ERR;
745 val |= APPL_INTR_EN_L1_18_CDM_REG_CHK_LOGIC_ERR;
746 appl_writel(pcie, val, APPL_INTR_EN_L1_18);
749 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
751 pcie->init_link_width = (val_w & PCI_EXP_LNKSTA_NLW) >>
752 PCI_EXP_LNKSTA_NLW_SHIFT;
754 val_w = dw_pcie_readw_dbi(&pcie->pci, pcie->pcie_cap_base +
756 val_w |= PCI_EXP_LNKCTL_LBMIE;
757 dw_pcie_writew_dbi(&pcie->pci, pcie->pcie_cap_base + PCI_EXP_LNKCTL,
761 static void tegra_pcie_enable_legacy_interrupts(struct pcie_port *pp)
763 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
764 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
767 /* Enable legacy interrupt generation */
768 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
769 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
770 val |= APPL_INTR_EN_L0_0_INT_INT_EN;
771 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
773 val = appl_readl(pcie, APPL_INTR_EN_L1_8_0);
774 val |= APPL_INTR_EN_L1_8_INTX_EN;
775 val |= APPL_INTR_EN_L1_8_AUTO_BW_INT_EN;
776 val |= APPL_INTR_EN_L1_8_BW_MGT_INT_EN;
777 if (IS_ENABLED(CONFIG_PCIEAER))
778 val |= APPL_INTR_EN_L1_8_AER_INT_EN;
779 appl_writel(pcie, val, APPL_INTR_EN_L1_8_0);
782 static void tegra_pcie_enable_msi_interrupts(struct pcie_port *pp)
784 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
785 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
788 dw_pcie_msi_init(pp);
790 /* Enable MSI interrupt generation */
791 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
792 val |= APPL_INTR_EN_L0_0_SYS_MSI_INTR_EN;
793 val |= APPL_INTR_EN_L0_0_MSI_RCV_INT_EN;
794 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
797 static void tegra_pcie_enable_interrupts(struct pcie_port *pp)
799 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
800 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
802 /* Clear interrupt statuses before enabling interrupts */
803 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
804 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
805 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
806 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
807 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
808 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
809 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
810 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
811 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
812 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
813 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
814 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
815 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
816 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
817 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
819 tegra_pcie_enable_system_interrupts(pp);
820 tegra_pcie_enable_legacy_interrupts(pp);
821 if (IS_ENABLED(CONFIG_PCI_MSI))
822 tegra_pcie_enable_msi_interrupts(pp);
825 static void config_gen3_gen4_eq_presets(struct tegra_pcie_dw *pcie)
827 struct dw_pcie *pci = &pcie->pci;
830 /* Program init preset */
831 for (i = 0; i < pcie->num_lanes; i++) {
832 dw_pcie_read(pci->dbi_base + CAP_SPCIE_CAP_OFF
834 val &= ~CAP_SPCIE_CAP_OFF_DSP_TX_PRESET0_MASK;
835 val |= GEN3_GEN4_EQ_PRESET_INIT;
836 val &= ~CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_MASK;
837 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
838 CAP_SPCIE_CAP_OFF_USP_TX_PRESET0_SHIFT);
839 dw_pcie_write(pci->dbi_base + CAP_SPCIE_CAP_OFF
842 offset = dw_pcie_find_ext_capability(pci,
843 PCI_EXT_CAP_ID_PL_16GT) +
845 dw_pcie_read(pci->dbi_base + offset + i, 1, &val);
846 val &= ~PCI_PL_16GT_LE_CTRL_DSP_TX_PRESET_MASK;
847 val |= GEN3_GEN4_EQ_PRESET_INIT;
848 val &= ~PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_MASK;
849 val |= (GEN3_GEN4_EQ_PRESET_INIT <<
850 PCI_PL_16GT_LE_CTRL_USP_TX_PRESET_SHIFT);
851 dw_pcie_write(pci->dbi_base + offset + i, 1, val);
854 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
855 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
856 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
858 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
859 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
860 val |= (0x3ff << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
861 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
862 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
864 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
865 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
866 val |= (0x1 << GEN3_RELATED_OFF_RATE_SHADOW_SEL_SHIFT);
867 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
869 val = dw_pcie_readl_dbi(pci, GEN3_EQ_CONTROL_OFF);
870 val &= ~GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_MASK;
871 val |= (0x360 << GEN3_EQ_CONTROL_OFF_PSET_REQ_VEC_SHIFT);
872 val &= ~GEN3_EQ_CONTROL_OFF_FB_MODE_MASK;
873 dw_pcie_writel_dbi(pci, GEN3_EQ_CONTROL_OFF, val);
875 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
876 val &= ~GEN3_RELATED_OFF_RATE_SHADOW_SEL_MASK;
877 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
880 static void tegra_pcie_prepare_host(struct pcie_port *pp)
882 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
883 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
886 val = dw_pcie_readl_dbi(pci, PCI_IO_BASE);
887 val &= ~(IO_BASE_IO_DECODE | IO_BASE_IO_DECODE_BIT8);
888 dw_pcie_writel_dbi(pci, PCI_IO_BASE, val);
890 val = dw_pcie_readl_dbi(pci, PCI_PREF_MEMORY_BASE);
891 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_DECODE;
892 val |= CFG_PREF_MEM_LIMIT_BASE_MEM_LIMIT_DECODE;
893 dw_pcie_writel_dbi(pci, PCI_PREF_MEMORY_BASE, val);
895 dw_pcie_writel_dbi(pci, PCI_BASE_ADDRESS_0, 0);
898 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
899 val &= ~(N_FTS_MASK << N_FTS_SHIFT);
900 val |= N_FTS_VAL << N_FTS_SHIFT;
901 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
903 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
906 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
908 /* Enable as 0xFFFF0001 response for CRS */
909 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT);
910 val &= ~(AMBA_ERROR_RESPONSE_CRS_MASK << AMBA_ERROR_RESPONSE_CRS_SHIFT);
911 val |= (AMBA_ERROR_RESPONSE_CRS_OKAY_FFFF0001 <<
912 AMBA_ERROR_RESPONSE_CRS_SHIFT);
913 dw_pcie_writel_dbi(pci, PORT_LOGIC_AMBA_ERROR_RESPONSE_DEFAULT, val);
915 /* Configure Max Speed from DT */
916 if (pcie->max_speed && pcie->max_speed != -EINVAL) {
917 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
919 val &= ~PCI_EXP_LNKCAP_SLS;
920 val |= pcie->max_speed;
921 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
925 /* Configure Max lane width from DT */
926 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP);
927 val &= ~PCI_EXP_LNKCAP_MLW;
928 val |= (pcie->num_lanes << PCI_EXP_LNKSTA_NLW_SHIFT);
929 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP, val);
931 config_gen3_gen4_eq_presets(pcie);
933 init_host_aspm(pcie);
935 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
936 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
937 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
939 if (pcie->update_fc_fixup) {
940 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
941 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
942 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
945 dw_pcie_setup_rc(pp);
947 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
950 val = appl_readl(pcie, APPL_PINMUX);
951 val &= ~APPL_PINMUX_PEX_RST;
952 appl_writel(pcie, val, APPL_PINMUX);
954 usleep_range(100, 200);
957 val = appl_readl(pcie, APPL_CTRL);
958 val |= APPL_CTRL_LTSSM_EN;
959 appl_writel(pcie, val, APPL_CTRL);
962 val = appl_readl(pcie, APPL_PINMUX);
963 val |= APPL_PINMUX_PEX_RST;
964 appl_writel(pcie, val, APPL_PINMUX);
969 static int tegra_pcie_dw_host_init(struct pcie_port *pp)
971 struct dw_pcie *pci = to_dw_pcie_from_pp(pp);
972 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
973 u32 val, tmp, offset, speed;
975 pp->bridge->ops = &tegra_pci_ops;
977 tegra_pcie_prepare_host(pp);
979 if (dw_pcie_wait_for_link(pci)) {
981 * There are some endpoints which can't get the link up if
982 * root port has Data Link Feature (DLF) enabled.
983 * Refer Spec rev 4.0 ver 1.0 sec 3.4.2 & 7.7.4 for more info
984 * on Scaled Flow Control and DLF.
985 * So, need to confirm that is indeed the case here and attempt
986 * link up once again with DLF disabled.
988 val = appl_readl(pcie, APPL_DEBUG);
989 val &= APPL_DEBUG_LTSSM_STATE_MASK;
990 val >>= APPL_DEBUG_LTSSM_STATE_SHIFT;
991 tmp = appl_readl(pcie, APPL_LINK_STATUS);
992 tmp &= APPL_LINK_STATUS_RDLH_LINK_UP;
993 if (!(val == 0x11 && !tmp)) {
994 /* Link is down for all good reasons */
998 dev_info(pci->dev, "Link is down in DLL");
999 dev_info(pci->dev, "Trying again with DLFE disabled\n");
1001 val = appl_readl(pcie, APPL_CTRL);
1002 val &= ~APPL_CTRL_LTSSM_EN;
1003 appl_writel(pcie, val, APPL_CTRL);
1005 reset_control_assert(pcie->core_rst);
1006 reset_control_deassert(pcie->core_rst);
1008 offset = dw_pcie_find_ext_capability(pci, PCI_EXT_CAP_ID_DLF);
1009 val = dw_pcie_readl_dbi(pci, offset + PCI_DLF_CAP);
1010 val &= ~PCI_DLF_EXCHANGE_ENABLE;
1011 dw_pcie_writel_dbi(pci, offset, val);
1013 tegra_pcie_prepare_host(pp);
1015 if (dw_pcie_wait_for_link(pci))
1019 speed = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA) &
1021 clk_set_rate(pcie->core_clk, pcie_gen_freq[speed - 1]);
1023 tegra_pcie_enable_interrupts(pp);
1028 static int tegra_pcie_dw_link_up(struct dw_pcie *pci)
1030 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1031 u32 val = dw_pcie_readw_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKSTA);
1033 return !!(val & PCI_EXP_LNKSTA_DLLLA);
1036 static void tegra_pcie_set_msi_vec_num(struct pcie_port *pp)
1038 pp->num_vectors = MAX_MSI_IRQS;
1041 static int tegra_pcie_dw_start_link(struct dw_pcie *pci)
1043 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1045 enable_irq(pcie->pex_rst_irq);
1050 static void tegra_pcie_dw_stop_link(struct dw_pcie *pci)
1052 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1054 disable_irq(pcie->pex_rst_irq);
1057 static const struct dw_pcie_ops tegra_dw_pcie_ops = {
1058 .link_up = tegra_pcie_dw_link_up,
1059 .start_link = tegra_pcie_dw_start_link,
1060 .stop_link = tegra_pcie_dw_stop_link,
1063 static struct dw_pcie_host_ops tegra_pcie_dw_host_ops = {
1064 .host_init = tegra_pcie_dw_host_init,
1065 .set_num_vectors = tegra_pcie_set_msi_vec_num,
1068 static void tegra_pcie_disable_phy(struct tegra_pcie_dw *pcie)
1070 unsigned int phy_count = pcie->phy_count;
1072 while (phy_count--) {
1073 phy_power_off(pcie->phys[phy_count]);
1074 phy_exit(pcie->phys[phy_count]);
1078 static int tegra_pcie_enable_phy(struct tegra_pcie_dw *pcie)
1083 for (i = 0; i < pcie->phy_count; i++) {
1084 ret = phy_init(pcie->phys[i]);
1088 ret = phy_power_on(pcie->phys[i]);
1097 phy_power_off(pcie->phys[i]);
1099 phy_exit(pcie->phys[i]);
1105 static int tegra_pcie_dw_parse_dt(struct tegra_pcie_dw *pcie)
1107 struct device_node *np = pcie->dev->of_node;
1110 ret = of_property_read_u32(np, "nvidia,aspm-cmrt-us", &pcie->aspm_cmrt);
1112 dev_info(pcie->dev, "Failed to read ASPM T_cmrt: %d\n", ret);
1116 ret = of_property_read_u32(np, "nvidia,aspm-pwr-on-t-us",
1117 &pcie->aspm_pwr_on_t);
1119 dev_info(pcie->dev, "Failed to read ASPM Power On time: %d\n",
1122 ret = of_property_read_u32(np, "nvidia,aspm-l0s-entrance-latency-us",
1123 &pcie->aspm_l0s_enter_lat);
1126 "Failed to read ASPM L0s Entrance latency: %d\n", ret);
1128 ret = of_property_read_u32(np, "num-lanes", &pcie->num_lanes);
1130 dev_err(pcie->dev, "Failed to read num-lanes: %d\n", ret);
1134 pcie->max_speed = of_pci_get_max_link_speed(np);
1136 ret = of_property_read_u32_index(np, "nvidia,bpmp", 1, &pcie->cid);
1138 dev_err(pcie->dev, "Failed to read Controller-ID: %d\n", ret);
1142 ret = of_property_count_strings(np, "phy-names");
1144 dev_err(pcie->dev, "Failed to find PHY entries: %d\n",
1148 pcie->phy_count = ret;
1150 if (of_property_read_bool(np, "nvidia,update-fc-fixup"))
1151 pcie->update_fc_fixup = true;
1153 pcie->supports_clkreq =
1154 of_property_read_bool(pcie->dev->of_node, "supports-clkreq");
1156 pcie->enable_cdm_check =
1157 of_property_read_bool(np, "snps,enable-cdm-check");
1159 if (pcie->mode == DW_PCIE_RC_TYPE)
1162 /* Endpoint mode specific DT entries */
1163 pcie->pex_rst_gpiod = devm_gpiod_get(pcie->dev, "reset", GPIOD_IN);
1164 if (IS_ERR(pcie->pex_rst_gpiod)) {
1165 int err = PTR_ERR(pcie->pex_rst_gpiod);
1166 const char *level = KERN_ERR;
1168 if (err == -EPROBE_DEFER)
1171 dev_printk(level, pcie->dev,
1172 dev_fmt("Failed to get PERST GPIO: %d\n"),
1177 pcie->pex_refclk_sel_gpiod = devm_gpiod_get(pcie->dev,
1178 "nvidia,refclk-select",
1180 if (IS_ERR(pcie->pex_refclk_sel_gpiod)) {
1181 int err = PTR_ERR(pcie->pex_refclk_sel_gpiod);
1182 const char *level = KERN_ERR;
1184 if (err == -EPROBE_DEFER)
1187 dev_printk(level, pcie->dev,
1188 dev_fmt("Failed to get REFCLK select GPIOs: %d\n"),
1190 pcie->pex_refclk_sel_gpiod = NULL;
1196 static int tegra_pcie_bpmp_set_ctrl_state(struct tegra_pcie_dw *pcie,
1199 struct mrq_uphy_response resp;
1200 struct tegra_bpmp_message msg;
1201 struct mrq_uphy_request req;
1203 /* Controller-5 doesn't need to have its state set by BPMP-FW */
1207 memset(&req, 0, sizeof(req));
1208 memset(&resp, 0, sizeof(resp));
1210 req.cmd = CMD_UPHY_PCIE_CONTROLLER_STATE;
1211 req.controller_state.pcie_controller = pcie->cid;
1212 req.controller_state.enable = enable;
1214 memset(&msg, 0, sizeof(msg));
1217 msg.tx.size = sizeof(req);
1218 msg.rx.data = &resp;
1219 msg.rx.size = sizeof(resp);
1221 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1224 static int tegra_pcie_bpmp_set_pll_state(struct tegra_pcie_dw *pcie,
1227 struct mrq_uphy_response resp;
1228 struct tegra_bpmp_message msg;
1229 struct mrq_uphy_request req;
1231 memset(&req, 0, sizeof(req));
1232 memset(&resp, 0, sizeof(resp));
1235 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_INIT;
1236 req.ep_ctrlr_pll_init.ep_controller = pcie->cid;
1238 req.cmd = CMD_UPHY_PCIE_EP_CONTROLLER_PLL_OFF;
1239 req.ep_ctrlr_pll_off.ep_controller = pcie->cid;
1242 memset(&msg, 0, sizeof(msg));
1245 msg.tx.size = sizeof(req);
1246 msg.rx.data = &resp;
1247 msg.rx.size = sizeof(resp);
1249 return tegra_bpmp_transfer(pcie->bpmp, &msg);
1252 static void tegra_pcie_downstream_dev_to_D0(struct tegra_pcie_dw *pcie)
1254 struct pcie_port *pp = &pcie->pci.pp;
1255 struct pci_bus *child, *root_bus = NULL;
1256 struct pci_dev *pdev;
1259 * link doesn't go into L2 state with some of the endpoints with Tegra
1260 * if they are not in D0 state. So, need to make sure that immediate
1261 * downstream devices are in D0 state before sending PME_TurnOff to put
1262 * link into L2 state.
1263 * This is as per PCI Express Base r4.0 v1.0 September 27-2017,
1264 * 5.2 Link State Power Management (Page #428).
1267 list_for_each_entry(child, &pp->bridge->bus->children, node) {
1268 /* Bring downstream devices to D0 if they are not already in */
1269 if (child->parent == pp->bridge->bus) {
1276 dev_err(pcie->dev, "Failed to find downstream devices\n");
1280 list_for_each_entry(pdev, &root_bus->devices, bus_list) {
1281 if (PCI_SLOT(pdev->devfn) == 0) {
1282 if (pci_set_power_state(pdev, PCI_D0))
1284 "Failed to transition %s to D0 state\n",
1285 dev_name(&pdev->dev));
1290 static int tegra_pcie_get_slot_regulators(struct tegra_pcie_dw *pcie)
1292 pcie->slot_ctl_3v3 = devm_regulator_get_optional(pcie->dev, "vpcie3v3");
1293 if (IS_ERR(pcie->slot_ctl_3v3)) {
1294 if (PTR_ERR(pcie->slot_ctl_3v3) != -ENODEV)
1295 return PTR_ERR(pcie->slot_ctl_3v3);
1297 pcie->slot_ctl_3v3 = NULL;
1300 pcie->slot_ctl_12v = devm_regulator_get_optional(pcie->dev, "vpcie12v");
1301 if (IS_ERR(pcie->slot_ctl_12v)) {
1302 if (PTR_ERR(pcie->slot_ctl_12v) != -ENODEV)
1303 return PTR_ERR(pcie->slot_ctl_12v);
1305 pcie->slot_ctl_12v = NULL;
1311 static int tegra_pcie_enable_slot_regulators(struct tegra_pcie_dw *pcie)
1315 if (pcie->slot_ctl_3v3) {
1316 ret = regulator_enable(pcie->slot_ctl_3v3);
1319 "Failed to enable 3.3V slot supply: %d\n", ret);
1324 if (pcie->slot_ctl_12v) {
1325 ret = regulator_enable(pcie->slot_ctl_12v);
1328 "Failed to enable 12V slot supply: %d\n", ret);
1329 goto fail_12v_enable;
1334 * According to PCI Express Card Electromechanical Specification
1335 * Revision 1.1, Table-2.4, T_PVPERL (Power stable to PERST# inactive)
1336 * should be a minimum of 100ms.
1338 if (pcie->slot_ctl_3v3 || pcie->slot_ctl_12v)
1344 if (pcie->slot_ctl_3v3)
1345 regulator_disable(pcie->slot_ctl_3v3);
1349 static void tegra_pcie_disable_slot_regulators(struct tegra_pcie_dw *pcie)
1351 if (pcie->slot_ctl_12v)
1352 regulator_disable(pcie->slot_ctl_12v);
1353 if (pcie->slot_ctl_3v3)
1354 regulator_disable(pcie->slot_ctl_3v3);
1357 static int tegra_pcie_config_controller(struct tegra_pcie_dw *pcie,
1363 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, true);
1366 "Failed to enable controller %u: %d\n", pcie->cid, ret);
1370 ret = tegra_pcie_enable_slot_regulators(pcie);
1372 goto fail_slot_reg_en;
1374 ret = regulator_enable(pcie->pex_ctl_supply);
1376 dev_err(pcie->dev, "Failed to enable regulator: %d\n", ret);
1380 ret = clk_prepare_enable(pcie->core_clk);
1382 dev_err(pcie->dev, "Failed to enable core clock: %d\n", ret);
1386 ret = reset_control_deassert(pcie->core_apb_rst);
1388 dev_err(pcie->dev, "Failed to deassert core APB reset: %d\n",
1390 goto fail_core_apb_rst;
1393 if (en_hw_hot_rst) {
1394 /* Enable HW_HOT_RST mode */
1395 val = appl_readl(pcie, APPL_CTRL);
1396 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
1397 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
1398 val |= APPL_CTRL_HW_HOT_RST_EN;
1399 appl_writel(pcie, val, APPL_CTRL);
1402 ret = tegra_pcie_enable_phy(pcie);
1404 dev_err(pcie->dev, "Failed to enable PHY: %d\n", ret);
1408 /* Update CFG base address */
1409 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1410 APPL_CFG_BASE_ADDR);
1412 /* Configure this core for RP mode operation */
1413 appl_writel(pcie, APPL_DM_TYPE_RP, APPL_DM_TYPE);
1415 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1417 val = appl_readl(pcie, APPL_CTRL);
1418 appl_writel(pcie, val | APPL_CTRL_SYS_PRE_DET_STATE, APPL_CTRL);
1420 val = appl_readl(pcie, APPL_CFG_MISC);
1421 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1422 appl_writel(pcie, val, APPL_CFG_MISC);
1424 if (!pcie->supports_clkreq) {
1425 val = appl_readl(pcie, APPL_PINMUX);
1426 val |= APPL_PINMUX_CLKREQ_OVERRIDE_EN;
1427 val &= ~APPL_PINMUX_CLKREQ_OVERRIDE;
1428 appl_writel(pcie, val, APPL_PINMUX);
1431 /* Update iATU_DMA base address */
1433 pcie->atu_dma_res->start & APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1434 APPL_CFG_IATU_DMA_BASE_ADDR);
1436 reset_control_deassert(pcie->core_rst);
1438 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1441 /* Disable ASPM-L1SS advertisement as there is no CLKREQ routing */
1442 if (!pcie->supports_clkreq) {
1443 disable_aspm_l11(pcie);
1444 disable_aspm_l12(pcie);
1450 reset_control_assert(pcie->core_apb_rst);
1452 clk_disable_unprepare(pcie->core_clk);
1454 regulator_disable(pcie->pex_ctl_supply);
1456 tegra_pcie_disable_slot_regulators(pcie);
1458 tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1463 static int __deinit_controller(struct tegra_pcie_dw *pcie)
1467 ret = reset_control_assert(pcie->core_rst);
1469 dev_err(pcie->dev, "Failed to assert \"core\" reset: %d\n",
1474 tegra_pcie_disable_phy(pcie);
1476 ret = reset_control_assert(pcie->core_apb_rst);
1478 dev_err(pcie->dev, "Failed to assert APB reset: %d\n", ret);
1482 clk_disable_unprepare(pcie->core_clk);
1484 ret = regulator_disable(pcie->pex_ctl_supply);
1486 dev_err(pcie->dev, "Failed to disable regulator: %d\n", ret);
1490 tegra_pcie_disable_slot_regulators(pcie);
1492 ret = tegra_pcie_bpmp_set_ctrl_state(pcie, false);
1494 dev_err(pcie->dev, "Failed to disable controller %d: %d\n",
1502 static int tegra_pcie_init_controller(struct tegra_pcie_dw *pcie)
1504 struct dw_pcie *pci = &pcie->pci;
1505 struct pcie_port *pp = &pci->pp;
1508 ret = tegra_pcie_config_controller(pcie, false);
1512 pp->ops = &tegra_pcie_dw_host_ops;
1514 ret = dw_pcie_host_init(pp);
1516 dev_err(pcie->dev, "Failed to add PCIe port: %d\n", ret);
1517 goto fail_host_init;
1523 return __deinit_controller(pcie);
1526 static int tegra_pcie_try_link_l2(struct tegra_pcie_dw *pcie)
1530 if (!tegra_pcie_dw_link_up(&pcie->pci))
1533 val = appl_readl(pcie, APPL_RADM_STATUS);
1534 val |= APPL_PM_XMT_TURNOFF_STATE;
1535 appl_writel(pcie, val, APPL_RADM_STATUS);
1537 return readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG, val,
1538 val & APPL_DEBUG_PM_LINKST_IN_L2_LAT,
1539 1, PME_ACK_TIMEOUT);
1542 static void tegra_pcie_dw_pme_turnoff(struct tegra_pcie_dw *pcie)
1547 if (!tegra_pcie_dw_link_up(&pcie->pci)) {
1548 dev_dbg(pcie->dev, "PCIe link is not up...!\n");
1552 if (tegra_pcie_try_link_l2(pcie)) {
1553 dev_info(pcie->dev, "Link didn't transition to L2 state\n");
1555 * TX lane clock freq will reset to Gen1 only if link is in L2
1557 * So apply pex_rst to end point to force RP to go into detect
1560 data = appl_readl(pcie, APPL_PINMUX);
1561 data &= ~APPL_PINMUX_PEX_RST;
1562 appl_writel(pcie, data, APPL_PINMUX);
1564 err = readl_poll_timeout_atomic(pcie->appl_base + APPL_DEBUG,
1567 APPL_DEBUG_LTSSM_STATE_MASK) >>
1568 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1569 LTSSM_STATE_PRE_DETECT,
1572 dev_info(pcie->dev, "Link didn't go to detect state\n");
1574 /* Disable LTSSM after link is in detect state */
1575 data = appl_readl(pcie, APPL_CTRL);
1576 data &= ~APPL_CTRL_LTSSM_EN;
1577 appl_writel(pcie, data, APPL_CTRL);
1581 * DBI registers may not be accessible after this as PLL-E would be
1582 * down depending on how CLKREQ is pulled by end point
1584 data = appl_readl(pcie, APPL_PINMUX);
1585 data |= (APPL_PINMUX_CLKREQ_OVERRIDE_EN | APPL_PINMUX_CLKREQ_OVERRIDE);
1586 /* Cut REFCLK to slot */
1587 data |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1588 data &= ~APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1589 appl_writel(pcie, data, APPL_PINMUX);
1592 static int tegra_pcie_deinit_controller(struct tegra_pcie_dw *pcie)
1594 tegra_pcie_downstream_dev_to_D0(pcie);
1595 dw_pcie_host_deinit(&pcie->pci.pp);
1596 tegra_pcie_dw_pme_turnoff(pcie);
1598 return __deinit_controller(pcie);
1601 static int tegra_pcie_config_rp(struct tegra_pcie_dw *pcie)
1603 struct pcie_port *pp = &pcie->pci.pp;
1604 struct device *dev = pcie->dev;
1608 if (IS_ENABLED(CONFIG_PCI_MSI)) {
1609 pp->msi_irq = of_irq_get_byname(dev->of_node, "msi");
1611 dev_err(dev, "Failed to get MSI interrupt\n");
1616 pm_runtime_enable(dev);
1618 ret = pm_runtime_get_sync(dev);
1620 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1622 goto fail_pm_get_sync;
1625 ret = pinctrl_pm_select_default_state(dev);
1627 dev_err(dev, "Failed to configure sideband pins: %d\n", ret);
1628 goto fail_pm_get_sync;
1631 tegra_pcie_init_controller(pcie);
1633 pcie->link_state = tegra_pcie_dw_link_up(&pcie->pci);
1634 if (!pcie->link_state) {
1636 goto fail_host_init;
1639 name = devm_kasprintf(dev, GFP_KERNEL, "%pOFP", dev->of_node);
1642 goto fail_host_init;
1645 pcie->debugfs = debugfs_create_dir(name, NULL);
1647 dev_err(dev, "Failed to create debugfs\n");
1654 tegra_pcie_deinit_controller(pcie);
1656 pm_runtime_put_sync(dev);
1657 pm_runtime_disable(dev);
1661 static void pex_ep_event_pex_rst_assert(struct tegra_pcie_dw *pcie)
1666 if (pcie->ep_state == EP_STATE_DISABLED)
1670 val = appl_readl(pcie, APPL_CTRL);
1671 val &= ~APPL_CTRL_LTSSM_EN;
1672 appl_writel(pcie, val, APPL_CTRL);
1674 ret = readl_poll_timeout(pcie->appl_base + APPL_DEBUG, val,
1675 ((val & APPL_DEBUG_LTSSM_STATE_MASK) >>
1676 APPL_DEBUG_LTSSM_STATE_SHIFT) ==
1677 LTSSM_STATE_PRE_DETECT,
1680 dev_err(pcie->dev, "Failed to go Detect state: %d\n", ret);
1682 reset_control_assert(pcie->core_rst);
1684 tegra_pcie_disable_phy(pcie);
1686 reset_control_assert(pcie->core_apb_rst);
1688 clk_disable_unprepare(pcie->core_clk);
1690 pm_runtime_put_sync(pcie->dev);
1692 ret = tegra_pcie_bpmp_set_pll_state(pcie, false);
1694 dev_err(pcie->dev, "Failed to turn off UPHY: %d\n", ret);
1696 pcie->ep_state = EP_STATE_DISABLED;
1697 dev_dbg(pcie->dev, "Uninitialization of endpoint is completed\n");
1700 static void pex_ep_event_pex_rst_deassert(struct tegra_pcie_dw *pcie)
1702 struct dw_pcie *pci = &pcie->pci;
1703 struct dw_pcie_ep *ep = &pci->ep;
1704 struct device *dev = pcie->dev;
1708 if (pcie->ep_state == EP_STATE_ENABLED)
1711 ret = pm_runtime_get_sync(dev);
1713 dev_err(dev, "Failed to get runtime sync for PCIe dev: %d\n",
1718 ret = tegra_pcie_bpmp_set_pll_state(pcie, true);
1720 dev_err(dev, "Failed to init UPHY for PCIe EP: %d\n", ret);
1724 ret = clk_prepare_enable(pcie->core_clk);
1726 dev_err(dev, "Failed to enable core clock: %d\n", ret);
1727 goto fail_core_clk_enable;
1730 ret = reset_control_deassert(pcie->core_apb_rst);
1732 dev_err(dev, "Failed to deassert core APB reset: %d\n", ret);
1733 goto fail_core_apb_rst;
1736 ret = tegra_pcie_enable_phy(pcie);
1738 dev_err(dev, "Failed to enable PHY: %d\n", ret);
1742 /* Clear any stale interrupt statuses */
1743 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L0);
1744 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_0_0);
1745 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_1);
1746 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_2);
1747 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_3);
1748 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_6);
1749 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_7);
1750 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_8_0);
1751 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_9);
1752 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_10);
1753 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_11);
1754 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_13);
1755 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_14);
1756 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_15);
1757 appl_writel(pcie, 0xFFFFFFFF, APPL_INTR_STATUS_L1_17);
1759 /* configure this core for EP mode operation */
1760 val = appl_readl(pcie, APPL_DM_TYPE);
1761 val &= ~APPL_DM_TYPE_MASK;
1762 val |= APPL_DM_TYPE_EP;
1763 appl_writel(pcie, val, APPL_DM_TYPE);
1765 appl_writel(pcie, 0x0, APPL_CFG_SLCG_OVERRIDE);
1767 val = appl_readl(pcie, APPL_CTRL);
1768 val |= APPL_CTRL_SYS_PRE_DET_STATE;
1769 val |= APPL_CTRL_HW_HOT_RST_EN;
1770 appl_writel(pcie, val, APPL_CTRL);
1772 val = appl_readl(pcie, APPL_CFG_MISC);
1773 val |= APPL_CFG_MISC_SLV_EP_MODE;
1774 val |= (APPL_CFG_MISC_ARCACHE_VAL << APPL_CFG_MISC_ARCACHE_SHIFT);
1775 appl_writel(pcie, val, APPL_CFG_MISC);
1777 val = appl_readl(pcie, APPL_PINMUX);
1778 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE_EN;
1779 val |= APPL_PINMUX_CLK_OUTPUT_IN_OVERRIDE;
1780 appl_writel(pcie, val, APPL_PINMUX);
1782 appl_writel(pcie, pcie->dbi_res->start & APPL_CFG_BASE_ADDR_MASK,
1783 APPL_CFG_BASE_ADDR);
1785 appl_writel(pcie, pcie->atu_dma_res->start &
1786 APPL_CFG_IATU_DMA_BASE_ADDR_MASK,
1787 APPL_CFG_IATU_DMA_BASE_ADDR);
1789 val = appl_readl(pcie, APPL_INTR_EN_L0_0);
1790 val |= APPL_INTR_EN_L0_0_SYS_INTR_EN;
1791 val |= APPL_INTR_EN_L0_0_LINK_STATE_INT_EN;
1792 val |= APPL_INTR_EN_L0_0_PCI_CMD_EN_INT_EN;
1793 appl_writel(pcie, val, APPL_INTR_EN_L0_0);
1795 val = appl_readl(pcie, APPL_INTR_EN_L1_0_0);
1796 val |= APPL_INTR_EN_L1_0_0_HOT_RESET_DONE_INT_EN;
1797 val |= APPL_INTR_EN_L1_0_0_RDLH_LINK_UP_INT_EN;
1798 appl_writel(pcie, val, APPL_INTR_EN_L1_0_0);
1800 reset_control_deassert(pcie->core_rst);
1802 if (pcie->update_fc_fixup) {
1803 val = dw_pcie_readl_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF);
1804 val |= 0x1 << CFG_TIMER_CTRL_ACK_NAK_SHIFT;
1805 dw_pcie_writel_dbi(pci, CFG_TIMER_CTRL_MAX_FUNC_NUM_OFF, val);
1808 config_gen3_gen4_eq_presets(pcie);
1810 init_host_aspm(pcie);
1812 /* Disable ASPM-L1SS advertisement if there is no CLKREQ routing */
1813 if (!pcie->supports_clkreq) {
1814 disable_aspm_l11(pcie);
1815 disable_aspm_l12(pcie);
1818 val = dw_pcie_readl_dbi(pci, GEN3_RELATED_OFF);
1819 val &= ~GEN3_RELATED_OFF_GEN3_ZRXDC_NONCOMPL;
1820 dw_pcie_writel_dbi(pci, GEN3_RELATED_OFF, val);
1822 /* Configure N_FTS & FTS */
1823 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL);
1824 val &= ~(N_FTS_MASK << N_FTS_SHIFT);
1825 val |= N_FTS_VAL << N_FTS_SHIFT;
1826 dw_pcie_writel_dbi(pci, PORT_LOGIC_ACK_F_ASPM_CTRL, val);
1828 val = dw_pcie_readl_dbi(pci, PORT_LOGIC_GEN2_CTRL);
1831 dw_pcie_writel_dbi(pci, PORT_LOGIC_GEN2_CTRL, val);
1833 /* Configure Max Speed from DT */
1834 if (pcie->max_speed && pcie->max_speed != -EINVAL) {
1835 val = dw_pcie_readl_dbi(pci, pcie->pcie_cap_base +
1837 val &= ~PCI_EXP_LNKCAP_SLS;
1838 val |= pcie->max_speed;
1839 dw_pcie_writel_dbi(pci, pcie->pcie_cap_base + PCI_EXP_LNKCAP,
1843 pcie->pcie_cap_base = dw_pcie_find_capability(&pcie->pci,
1845 clk_set_rate(pcie->core_clk, GEN4_CORE_CLK_FREQ);
1847 val = (ep->msi_mem_phys & MSIX_ADDR_MATCH_LOW_OFF_MASK);
1848 val |= MSIX_ADDR_MATCH_LOW_OFF_EN;
1849 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_LOW_OFF, val);
1850 val = (lower_32_bits(ep->msi_mem_phys) & MSIX_ADDR_MATCH_HIGH_OFF_MASK);
1851 dw_pcie_writel_dbi(pci, MSIX_ADDR_MATCH_HIGH_OFF, val);
1853 ret = dw_pcie_ep_init_complete(ep);
1855 dev_err(dev, "Failed to complete initialization: %d\n", ret);
1856 goto fail_init_complete;
1859 dw_pcie_ep_init_notify(ep);
1862 val = appl_readl(pcie, APPL_CTRL);
1863 val |= APPL_CTRL_LTSSM_EN;
1864 appl_writel(pcie, val, APPL_CTRL);
1866 pcie->ep_state = EP_STATE_ENABLED;
1867 dev_dbg(dev, "Initialization of endpoint is completed\n");
1872 reset_control_assert(pcie->core_rst);
1873 tegra_pcie_disable_phy(pcie);
1875 reset_control_assert(pcie->core_apb_rst);
1877 clk_disable_unprepare(pcie->core_clk);
1878 fail_core_clk_enable:
1879 tegra_pcie_bpmp_set_pll_state(pcie, false);
1881 pm_runtime_put_sync(dev);
1884 static irqreturn_t tegra_pcie_ep_pex_rst_irq(int irq, void *arg)
1886 struct tegra_pcie_dw *pcie = arg;
1888 if (gpiod_get_value(pcie->pex_rst_gpiod))
1889 pex_ep_event_pex_rst_assert(pcie);
1891 pex_ep_event_pex_rst_deassert(pcie);
1896 static int tegra_pcie_ep_raise_legacy_irq(struct tegra_pcie_dw *pcie, u16 irq)
1898 /* Tegra194 supports only INTA */
1902 appl_writel(pcie, 1, APPL_LEGACY_INTX);
1903 usleep_range(1000, 2000);
1904 appl_writel(pcie, 0, APPL_LEGACY_INTX);
1908 static int tegra_pcie_ep_raise_msi_irq(struct tegra_pcie_dw *pcie, u16 irq)
1910 if (unlikely(irq > 31))
1913 appl_writel(pcie, (1 << irq), APPL_MSI_CTRL_1);
1918 static int tegra_pcie_ep_raise_msix_irq(struct tegra_pcie_dw *pcie, u16 irq)
1920 struct dw_pcie_ep *ep = &pcie->pci.ep;
1922 writel(irq, ep->msi_mem);
1927 static int tegra_pcie_ep_raise_irq(struct dw_pcie_ep *ep, u8 func_no,
1928 enum pci_epc_irq_type type,
1931 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
1932 struct tegra_pcie_dw *pcie = to_tegra_pcie(pci);
1935 case PCI_EPC_IRQ_LEGACY:
1936 return tegra_pcie_ep_raise_legacy_irq(pcie, interrupt_num);
1938 case PCI_EPC_IRQ_MSI:
1939 return tegra_pcie_ep_raise_msi_irq(pcie, interrupt_num);
1941 case PCI_EPC_IRQ_MSIX:
1942 return tegra_pcie_ep_raise_msix_irq(pcie, interrupt_num);
1945 dev_err(pci->dev, "Unknown IRQ type\n");
1952 static const struct pci_epc_features tegra_pcie_epc_features = {
1953 .linkup_notifier = true,
1954 .core_init_notifier = true,
1955 .msi_capable = false,
1956 .msix_capable = false,
1957 .reserved_bar = 1 << BAR_2 | 1 << BAR_3 | 1 << BAR_4 | 1 << BAR_5,
1958 .bar_fixed_64bit = 1 << BAR_0,
1959 .bar_fixed_size[0] = SZ_1M,
1962 static const struct pci_epc_features*
1963 tegra_pcie_ep_get_features(struct dw_pcie_ep *ep)
1965 return &tegra_pcie_epc_features;
1968 static struct dw_pcie_ep_ops pcie_ep_ops = {
1969 .raise_irq = tegra_pcie_ep_raise_irq,
1970 .get_features = tegra_pcie_ep_get_features,
1973 static int tegra_pcie_config_ep(struct tegra_pcie_dw *pcie,
1974 struct platform_device *pdev)
1976 struct dw_pcie *pci = &pcie->pci;
1977 struct device *dev = pcie->dev;
1978 struct dw_pcie_ep *ep;
1979 struct resource *res;
1984 ep->ops = &pcie_ep_ops;
1986 res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "addr_space");
1990 ep->phys_base = res->start;
1991 ep->addr_size = resource_size(res);
1992 ep->page_size = SZ_64K;
1994 ret = gpiod_set_debounce(pcie->pex_rst_gpiod, PERST_DEBOUNCE_TIME);
1996 dev_err(dev, "Failed to set PERST GPIO debounce time: %d\n",
2001 ret = gpiod_to_irq(pcie->pex_rst_gpiod);
2003 dev_err(dev, "Failed to get IRQ for PERST GPIO: %d\n", ret);
2006 pcie->pex_rst_irq = (unsigned int)ret;
2008 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_pex_rst_irq",
2011 dev_err(dev, "Failed to create PERST IRQ string\n");
2015 irq_set_status_flags(pcie->pex_rst_irq, IRQ_NOAUTOEN);
2017 pcie->ep_state = EP_STATE_DISABLED;
2019 ret = devm_request_threaded_irq(dev, pcie->pex_rst_irq, NULL,
2020 tegra_pcie_ep_pex_rst_irq,
2021 IRQF_TRIGGER_RISING |
2022 IRQF_TRIGGER_FALLING | IRQF_ONESHOT,
2023 name, (void *)pcie);
2025 dev_err(dev, "Failed to request IRQ for PERST: %d\n", ret);
2029 name = devm_kasprintf(dev, GFP_KERNEL, "tegra_pcie_%u_ep_work",
2032 dev_err(dev, "Failed to create PCIe EP work thread string\n");
2036 pm_runtime_enable(dev);
2038 ret = dw_pcie_ep_init(ep);
2040 dev_err(dev, "Failed to initialize DWC Endpoint subsystem: %d\n",
2048 static int tegra_pcie_dw_probe(struct platform_device *pdev)
2050 const struct tegra_pcie_dw_of_data *data;
2051 struct device *dev = &pdev->dev;
2052 struct resource *atu_dma_res;
2053 struct tegra_pcie_dw *pcie;
2054 struct resource *dbi_res;
2055 struct pcie_port *pp;
2056 struct dw_pcie *pci;
2062 data = of_device_get_match_data(dev);
2064 pcie = devm_kzalloc(dev, sizeof(*pcie), GFP_KERNEL);
2069 pci->dev = &pdev->dev;
2070 pci->ops = &tegra_dw_pcie_ops;
2072 pcie->dev = &pdev->dev;
2073 pcie->mode = (enum dw_pcie_device_mode)data->mode;
2075 ret = tegra_pcie_dw_parse_dt(pcie);
2077 const char *level = KERN_ERR;
2079 if (ret == -EPROBE_DEFER)
2082 dev_printk(level, dev,
2083 dev_fmt("Failed to parse device tree: %d\n"),
2088 ret = tegra_pcie_get_slot_regulators(pcie);
2090 const char *level = KERN_ERR;
2092 if (ret == -EPROBE_DEFER)
2095 dev_printk(level, dev,
2096 dev_fmt("Failed to get slot regulators: %d\n"),
2101 if (pcie->pex_refclk_sel_gpiod)
2102 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 1);
2104 pcie->pex_ctl_supply = devm_regulator_get(dev, "vddio-pex-ctl");
2105 if (IS_ERR(pcie->pex_ctl_supply)) {
2106 ret = PTR_ERR(pcie->pex_ctl_supply);
2107 if (ret != -EPROBE_DEFER)
2108 dev_err(dev, "Failed to get regulator: %ld\n",
2109 PTR_ERR(pcie->pex_ctl_supply));
2113 pcie->core_clk = devm_clk_get(dev, "core");
2114 if (IS_ERR(pcie->core_clk)) {
2115 dev_err(dev, "Failed to get core clock: %ld\n",
2116 PTR_ERR(pcie->core_clk));
2117 return PTR_ERR(pcie->core_clk);
2120 pcie->appl_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2122 if (!pcie->appl_res) {
2123 dev_err(dev, "Failed to find \"appl\" region\n");
2127 pcie->appl_base = devm_ioremap_resource(dev, pcie->appl_res);
2128 if (IS_ERR(pcie->appl_base))
2129 return PTR_ERR(pcie->appl_base);
2131 pcie->core_apb_rst = devm_reset_control_get(dev, "apb");
2132 if (IS_ERR(pcie->core_apb_rst)) {
2133 dev_err(dev, "Failed to get APB reset: %ld\n",
2134 PTR_ERR(pcie->core_apb_rst));
2135 return PTR_ERR(pcie->core_apb_rst);
2138 phys = devm_kcalloc(dev, pcie->phy_count, sizeof(*phys), GFP_KERNEL);
2142 for (i = 0; i < pcie->phy_count; i++) {
2143 name = kasprintf(GFP_KERNEL, "p2u-%u", i);
2145 dev_err(dev, "Failed to create P2U string\n");
2148 phys[i] = devm_phy_get(dev, name);
2150 if (IS_ERR(phys[i])) {
2151 ret = PTR_ERR(phys[i]);
2152 if (ret != -EPROBE_DEFER)
2153 dev_err(dev, "Failed to get PHY: %d\n", ret);
2160 dbi_res = platform_get_resource_byname(pdev, IORESOURCE_MEM, "dbi");
2162 dev_err(dev, "Failed to find \"dbi\" region\n");
2165 pcie->dbi_res = dbi_res;
2167 pci->dbi_base = devm_ioremap_resource(dev, dbi_res);
2168 if (IS_ERR(pci->dbi_base))
2169 return PTR_ERR(pci->dbi_base);
2171 /* Tegra HW locates DBI2 at a fixed offset from DBI */
2172 pci->dbi_base2 = pci->dbi_base + 0x1000;
2174 atu_dma_res = platform_get_resource_byname(pdev, IORESOURCE_MEM,
2177 dev_err(dev, "Failed to find \"atu_dma\" region\n");
2180 pcie->atu_dma_res = atu_dma_res;
2182 pci->atu_base = devm_ioremap_resource(dev, atu_dma_res);
2183 if (IS_ERR(pci->atu_base))
2184 return PTR_ERR(pci->atu_base);
2186 pcie->core_rst = devm_reset_control_get(dev, "core");
2187 if (IS_ERR(pcie->core_rst)) {
2188 dev_err(dev, "Failed to get core reset: %ld\n",
2189 PTR_ERR(pcie->core_rst));
2190 return PTR_ERR(pcie->core_rst);
2193 pp->irq = platform_get_irq_byname(pdev, "intr");
2197 pcie->bpmp = tegra_bpmp_get(dev);
2198 if (IS_ERR(pcie->bpmp))
2199 return PTR_ERR(pcie->bpmp);
2201 platform_set_drvdata(pdev, pcie);
2203 switch (pcie->mode) {
2204 case DW_PCIE_RC_TYPE:
2205 ret = devm_request_irq(dev, pp->irq, tegra_pcie_rp_irq_handler,
2206 IRQF_SHARED, "tegra-pcie-intr", pcie);
2208 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2213 ret = tegra_pcie_config_rp(pcie);
2214 if (ret && ret != -ENOMEDIUM)
2220 case DW_PCIE_EP_TYPE:
2221 ret = devm_request_threaded_irq(dev, pp->irq,
2222 tegra_pcie_ep_hard_irq,
2223 tegra_pcie_ep_irq_thread,
2224 IRQF_SHARED | IRQF_ONESHOT,
2225 "tegra-pcie-ep-intr", pcie);
2227 dev_err(dev, "Failed to request IRQ %d: %d\n", pp->irq,
2232 ret = tegra_pcie_config_ep(pcie, pdev);
2238 dev_err(dev, "Invalid PCIe device type %d\n", pcie->mode);
2242 tegra_bpmp_put(pcie->bpmp);
2246 static int tegra_pcie_dw_remove(struct platform_device *pdev)
2248 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2250 if (!pcie->link_state)
2253 debugfs_remove_recursive(pcie->debugfs);
2254 tegra_pcie_deinit_controller(pcie);
2255 pm_runtime_put_sync(pcie->dev);
2256 pm_runtime_disable(pcie->dev);
2257 tegra_bpmp_put(pcie->bpmp);
2258 if (pcie->pex_refclk_sel_gpiod)
2259 gpiod_set_value(pcie->pex_refclk_sel_gpiod, 0);
2264 static int tegra_pcie_dw_suspend_late(struct device *dev)
2266 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2269 if (!pcie->link_state)
2272 /* Enable HW_HOT_RST mode */
2273 val = appl_readl(pcie, APPL_CTRL);
2274 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2275 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2276 val |= APPL_CTRL_HW_HOT_RST_EN;
2277 appl_writel(pcie, val, APPL_CTRL);
2282 static int tegra_pcie_dw_suspend_noirq(struct device *dev)
2284 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2286 if (!pcie->link_state)
2289 /* Save MSI interrupt vector */
2290 pcie->msi_ctrl_int = dw_pcie_readl_dbi(&pcie->pci,
2291 PORT_LOGIC_MSI_CTRL_INT_0_EN);
2292 tegra_pcie_downstream_dev_to_D0(pcie);
2293 tegra_pcie_dw_pme_turnoff(pcie);
2295 return __deinit_controller(pcie);
2298 static int tegra_pcie_dw_resume_noirq(struct device *dev)
2300 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2303 if (!pcie->link_state)
2306 ret = tegra_pcie_config_controller(pcie, true);
2310 ret = tegra_pcie_dw_host_init(&pcie->pci.pp);
2312 dev_err(dev, "Failed to init host: %d\n", ret);
2313 goto fail_host_init;
2316 /* Restore MSI interrupt vector */
2317 dw_pcie_writel_dbi(&pcie->pci, PORT_LOGIC_MSI_CTRL_INT_0_EN,
2318 pcie->msi_ctrl_int);
2323 return __deinit_controller(pcie);
2326 static int tegra_pcie_dw_resume_early(struct device *dev)
2328 struct tegra_pcie_dw *pcie = dev_get_drvdata(dev);
2331 if (!pcie->link_state)
2334 /* Disable HW_HOT_RST mode */
2335 val = appl_readl(pcie, APPL_CTRL);
2336 val &= ~(APPL_CTRL_HW_HOT_RST_MODE_MASK <<
2337 APPL_CTRL_HW_HOT_RST_MODE_SHIFT);
2338 val |= APPL_CTRL_HW_HOT_RST_MODE_IMDT_RST <<
2339 APPL_CTRL_HW_HOT_RST_MODE_SHIFT;
2340 val &= ~APPL_CTRL_HW_HOT_RST_EN;
2341 appl_writel(pcie, val, APPL_CTRL);
2346 static void tegra_pcie_dw_shutdown(struct platform_device *pdev)
2348 struct tegra_pcie_dw *pcie = platform_get_drvdata(pdev);
2350 if (!pcie->link_state)
2353 debugfs_remove_recursive(pcie->debugfs);
2354 tegra_pcie_downstream_dev_to_D0(pcie);
2356 disable_irq(pcie->pci.pp.irq);
2357 if (IS_ENABLED(CONFIG_PCI_MSI))
2358 disable_irq(pcie->pci.pp.msi_irq);
2360 tegra_pcie_dw_pme_turnoff(pcie);
2361 __deinit_controller(pcie);
2364 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_rc_of_data = {
2365 .mode = DW_PCIE_RC_TYPE,
2368 static const struct tegra_pcie_dw_of_data tegra_pcie_dw_ep_of_data = {
2369 .mode = DW_PCIE_EP_TYPE,
2372 static const struct of_device_id tegra_pcie_dw_of_match[] = {
2374 .compatible = "nvidia,tegra194-pcie",
2375 .data = &tegra_pcie_dw_rc_of_data,
2378 .compatible = "nvidia,tegra194-pcie-ep",
2379 .data = &tegra_pcie_dw_ep_of_data,
2384 static const struct dev_pm_ops tegra_pcie_dw_pm_ops = {
2385 .suspend_late = tegra_pcie_dw_suspend_late,
2386 .suspend_noirq = tegra_pcie_dw_suspend_noirq,
2387 .resume_noirq = tegra_pcie_dw_resume_noirq,
2388 .resume_early = tegra_pcie_dw_resume_early,
2391 static struct platform_driver tegra_pcie_dw_driver = {
2392 .probe = tegra_pcie_dw_probe,
2393 .remove = tegra_pcie_dw_remove,
2394 .shutdown = tegra_pcie_dw_shutdown,
2396 .name = "tegra194-pcie",
2397 .pm = &tegra_pcie_dw_pm_ops,
2398 .of_match_table = tegra_pcie_dw_of_match,
2401 module_platform_driver(tegra_pcie_dw_driver);
2403 MODULE_DEVICE_TABLE(of, tegra_pcie_dw_of_match);
2405 MODULE_AUTHOR("Vidya Sagar <vidyas@nvidia.com>");
2406 MODULE_DESCRIPTION("NVIDIA PCIe host controller driver");
2407 MODULE_LICENSE("GPL v2");