1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #include "habanalabs.h"
9 #include "../include/hw_ip/pci/pci_general.h"
11 #include <linux/pci.h>
12 #include <linux/bitfield.h>
14 #define HL_PLDM_PCI_ELBI_TIMEOUT_MSEC (HL_PCI_ELBI_TIMEOUT_MSEC * 10)
16 #define IATU_REGION_CTRL_REGION_EN_MASK BIT(31)
17 #define IATU_REGION_CTRL_MATCH_MODE_MASK BIT(30)
18 #define IATU_REGION_CTRL_NUM_MATCH_EN_MASK BIT(19)
19 #define IATU_REGION_CTRL_BAR_NUM_MASK GENMASK(10, 8)
22 * hl_pci_bars_map() - Map PCI BARs.
23 * @hdev: Pointer to hl_device structure.
24 * @name: Array of BAR names.
25 * @is_wc: Array with flag per BAR whether a write-combined mapping is needed.
27 * Request PCI regions and map them to kernel virtual addresses.
29 * Return: 0 on success, non-zero for failure.
31 int hl_pci_bars_map(struct hl_device *hdev, const char * const name[3],
34 struct pci_dev *pdev = hdev->pdev;
37 rc = pci_request_regions(pdev, HL_NAME);
39 dev_err(hdev->dev, "Cannot obtain PCI resources\n");
43 for (i = 0 ; i < 3 ; i++) {
44 bar = i * 2; /* 64-bit BARs */
45 hdev->pcie_bar[bar] = is_wc[i] ?
46 pci_ioremap_wc_bar(pdev, bar) :
47 pci_ioremap_bar(pdev, bar);
48 if (!hdev->pcie_bar[bar]) {
49 dev_err(hdev->dev, "pci_ioremap%s_bar failed for %s\n",
50 is_wc[i] ? "_wc" : "", name[i]);
59 for (i = 2 ; i >= 0 ; i--) {
60 bar = i * 2; /* 64-bit BARs */
61 if (hdev->pcie_bar[bar])
62 iounmap(hdev->pcie_bar[bar]);
65 pci_release_regions(pdev);
71 * hl_pci_bars_unmap() - Unmap PCI BARS.
72 * @hdev: Pointer to hl_device structure.
74 * Release all PCI BARs and unmap their virtual addresses.
76 static void hl_pci_bars_unmap(struct hl_device *hdev)
78 struct pci_dev *pdev = hdev->pdev;
81 for (i = 2 ; i >= 0 ; i--) {
82 bar = i * 2; /* 64-bit BARs */
83 iounmap(hdev->pcie_bar[bar]);
86 pci_release_regions(pdev);
90 * hl_pci_elbi_write() - Write through the ELBI interface.
91 * @hdev: Pointer to hl_device structure.
92 * @addr: Address to write to
93 * @data: Data to write
95 * Return: 0 on success, negative value for failure.
97 static int hl_pci_elbi_write(struct hl_device *hdev, u64 addr, u32 data)
99 struct pci_dev *pdev = hdev->pdev;
105 msec = HL_PLDM_PCI_ELBI_TIMEOUT_MSEC;
107 msec = HL_PCI_ELBI_TIMEOUT_MSEC;
109 /* Clear previous status */
110 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, 0);
112 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_ADDR, (u32) addr);
113 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_DATA, data);
114 pci_write_config_dword(pdev, mmPCI_CONFIG_ELBI_CTRL,
115 PCI_CONFIG_ELBI_CTRL_WRITE);
117 timeout = ktime_add_ms(ktime_get(), msec);
119 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS, &val);
120 if (val & PCI_CONFIG_ELBI_STS_MASK)
122 if (ktime_compare(ktime_get(), timeout) > 0) {
123 pci_read_config_dword(pdev, mmPCI_CONFIG_ELBI_STS,
128 usleep_range(300, 500);
131 if ((val & PCI_CONFIG_ELBI_STS_MASK) == PCI_CONFIG_ELBI_STS_DONE)
134 if (val & PCI_CONFIG_ELBI_STS_ERR) {
135 dev_err(hdev->dev, "Error writing to ELBI\n");
139 if (!(val & PCI_CONFIG_ELBI_STS_MASK)) {
140 dev_err(hdev->dev, "ELBI write didn't finish in time\n");
144 dev_err(hdev->dev, "ELBI write has undefined bits in status\n");
149 * hl_pci_iatu_write() - iatu write routine.
150 * @hdev: Pointer to hl_device structure.
151 * @addr: Address to write to
152 * @data: Data to write
154 * Return: 0 on success, negative value for failure.
156 int hl_pci_iatu_write(struct hl_device *hdev, u32 addr, u32 data)
158 struct asic_fixed_properties *prop = &hdev->asic_prop;
162 dbi_offset = addr & 0xFFF;
164 rc = hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0x00300000);
165 rc |= hl_pci_elbi_write(hdev, prop->pcie_dbi_base_address + dbi_offset,
175 * hl_pci_reset_link_through_bridge() - Reset PCI link.
176 * @hdev: Pointer to hl_device structure.
178 static void hl_pci_reset_link_through_bridge(struct hl_device *hdev)
180 struct pci_dev *pdev = hdev->pdev;
181 struct pci_dev *parent_port;
184 parent_port = pdev->bus->self;
185 pci_read_config_word(parent_port, PCI_BRIDGE_CONTROL, &val);
186 val |= PCI_BRIDGE_CTL_BUS_RESET;
187 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
190 val &= ~(PCI_BRIDGE_CTL_BUS_RESET);
191 pci_write_config_word(parent_port, PCI_BRIDGE_CONTROL, val);
196 * hl_pci_set_inbound_region() - Configure inbound region
197 * @hdev: Pointer to hl_device structure.
198 * @region: Inbound region number.
199 * @pci_region: Inbound region parameters.
201 * Configure the iATU inbound region.
203 * Return: 0 on success, negative value for failure.
205 int hl_pci_set_inbound_region(struct hl_device *hdev, u8 region,
206 struct hl_inbound_pci_region *pci_region)
208 struct asic_fixed_properties *prop = &hdev->asic_prop;
209 u64 bar_phys_base, region_base, region_end_address;
210 u32 offset, ctrl_reg_val;
214 offset = (0x200 * region) + 0x100;
216 if (pci_region->mode == PCI_ADDRESS_MATCH_MODE) {
217 bar_phys_base = hdev->pcie_bar_phys[pci_region->bar];
218 region_base = bar_phys_base + pci_region->offset_in_bar;
219 region_end_address = region_base + pci_region->size - 1;
221 rc |= hl_pci_iatu_write(hdev, offset + 0x8,
222 lower_32_bits(region_base));
223 rc |= hl_pci_iatu_write(hdev, offset + 0xC,
224 upper_32_bits(region_base));
225 rc |= hl_pci_iatu_write(hdev, offset + 0x10,
226 lower_32_bits(region_end_address));
229 /* Point to the specified address */
230 rc |= hl_pci_iatu_write(hdev, offset + 0x14,
231 lower_32_bits(pci_region->addr));
232 rc |= hl_pci_iatu_write(hdev, offset + 0x18,
233 upper_32_bits(pci_region->addr));
234 rc |= hl_pci_iatu_write(hdev, offset + 0x0, 0);
236 /* Enable + bar/address match + match enable + bar number */
237 ctrl_reg_val = FIELD_PREP(IATU_REGION_CTRL_REGION_EN_MASK, 1);
238 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_MATCH_MODE_MASK,
240 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_NUM_MATCH_EN_MASK, 1);
242 if (pci_region->mode == PCI_BAR_MATCH_MODE)
243 ctrl_reg_val |= FIELD_PREP(IATU_REGION_CTRL_BAR_NUM_MASK,
246 rc |= hl_pci_iatu_write(hdev, offset + 0x4, ctrl_reg_val);
248 /* Return the DBI window to the default location */
249 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
250 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
253 dev_err(hdev->dev, "failed to map bar %u to 0x%08llx\n",
254 pci_region->bar, pci_region->addr);
260 * hl_pci_set_outbound_region() - Configure outbound region 0
261 * @hdev: Pointer to hl_device structure.
262 * @pci_region: Outbound region parameters.
264 * Configure the iATU outbound region 0.
266 * Return: 0 on success, negative value for failure.
268 int hl_pci_set_outbound_region(struct hl_device *hdev,
269 struct hl_outbound_pci_region *pci_region)
271 struct asic_fixed_properties *prop = &hdev->asic_prop;
272 u64 outbound_region_end_address;
275 /* Outbound Region 0 */
276 outbound_region_end_address =
277 pci_region->addr + pci_region->size - 1;
278 rc |= hl_pci_iatu_write(hdev, 0x008,
279 lower_32_bits(pci_region->addr));
280 rc |= hl_pci_iatu_write(hdev, 0x00C,
281 upper_32_bits(pci_region->addr));
282 rc |= hl_pci_iatu_write(hdev, 0x010,
283 lower_32_bits(outbound_region_end_address));
284 rc |= hl_pci_iatu_write(hdev, 0x014, 0);
286 if ((hdev->power9_64bit_dma_enable) && (hdev->dma_mask == 64))
287 rc |= hl_pci_iatu_write(hdev, 0x018, 0x08000000);
289 rc |= hl_pci_iatu_write(hdev, 0x018, 0);
291 rc |= hl_pci_iatu_write(hdev, 0x020,
292 upper_32_bits(outbound_region_end_address));
293 /* Increase region size */
294 rc |= hl_pci_iatu_write(hdev, 0x000, 0x00002000);
296 rc |= hl_pci_iatu_write(hdev, 0x004, 0x80000000);
298 /* Return the DBI window to the default location */
299 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr, 0);
300 rc |= hl_pci_elbi_write(hdev, prop->pcie_aux_dbi_reg_addr + 4, 0);
306 * hl_pci_set_dma_mask() - Set DMA masks for the device.
307 * @hdev: Pointer to hl_device structure.
309 * This function sets the DMA masks (regular and consistent) for a specified
310 * value. If it doesn't succeed, it tries to set it to a fall-back value
312 * Return: 0 on success, non-zero for failure.
314 static int hl_pci_set_dma_mask(struct hl_device *hdev)
316 struct pci_dev *pdev = hdev->pdev;
320 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
323 "Failed to set pci dma mask to %d bits, error %d\n",
328 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(hdev->dma_mask));
331 "Failed to set pci consistent dma mask to %d bits, error %d\n",
340 * hl_pci_init() - PCI initialization code.
341 * @hdev: Pointer to hl_device structure.
343 * Set DMA masks, initialize the PCI controller and map the PCI BARs.
345 * Return: 0 on success, non-zero for failure.
347 int hl_pci_init(struct hl_device *hdev)
349 struct pci_dev *pdev = hdev->pdev;
352 if (hdev->reset_pcilink)
353 hl_pci_reset_link_through_bridge(hdev);
355 rc = pci_enable_device_mem(pdev);
357 dev_err(hdev->dev, "can't enable PCI device\n");
361 pci_set_master(pdev);
363 rc = hdev->asic_funcs->pci_bars_map(hdev);
365 dev_err(hdev->dev, "Failed to initialize PCI BARs\n");
369 rc = hdev->asic_funcs->init_iatu(hdev);
371 dev_err(hdev->dev, "Failed to initialize iATU\n");
375 rc = hl_pci_set_dma_mask(hdev);
382 hl_pci_bars_unmap(hdev);
384 pci_clear_master(pdev);
385 pci_disable_device(pdev);
391 * hl_fw_fini() - PCI finalization code.
392 * @hdev: Pointer to hl_device structure
394 * Unmap PCI bars and disable PCI device.
396 void hl_pci_fini(struct hl_device *hdev)
398 hl_pci_bars_unmap(hdev);
400 pci_clear_master(hdev->pdev);
401 pci_disable_device(hdev->pdev);