1 // SPDX-License-Identifier: GPL-2.0-only
3 #include <linux/delay.h>
4 #include <linux/firmware.h>
5 #include <linux/list.h>
6 #include <linux/module.h>
7 #include <linux/mutex.h>
9 #include <linux/pci_ids.h>
11 #include "nitrox_dev.h"
12 #include "nitrox_common.h"
13 #include "nitrox_csr.h"
14 #include "nitrox_hal.h"
15 #include "nitrox_isr.h"
16 #include "nitrox_debugfs.h"
18 #define CNN55XX_DEV_ID 0x12
20 #define DEFAULT_SE_GROUP 0
21 #define DEFAULT_AE_GROUP 0
23 #define DRIVER_VERSION "1.2"
24 #define CNN55XX_UCD_BLOCK_SIZE 32768
25 #define CNN55XX_MAX_UCODE_SIZE (CNN55XX_UCD_BLOCK_SIZE * 2)
26 #define FW_DIR "cavium/"
28 #define SE_FW FW_DIR "cnn55xx_se.fw"
30 #define AE_FW FW_DIR "cnn55xx_ae.fw"
32 static const char nitrox_driver_name[] = "CNN55XX";
34 static LIST_HEAD(ndevlist);
35 static DEFINE_MUTEX(devlist_lock);
36 static unsigned int num_devices;
39 * nitrox_pci_tbl - PCI Device ID Table
41 static const struct pci_device_id nitrox_pci_tbl[] = {
42 {PCI_VDEVICE(CAVIUM, CNN55XX_DEV_ID), 0},
43 /* required last entry */
46 MODULE_DEVICE_TABLE(pci, nitrox_pci_tbl);
48 static unsigned int qlen = DEFAULT_CMD_QLEN;
49 module_param(qlen, uint, 0644);
50 MODULE_PARM_DESC(qlen, "Command queue length - default 2048");
53 * struct ucode - Firmware Header
55 * @version: firmware version
56 * @code_size: code section size
62 char version[VERSION_LEN - 1];
69 * write_to_ucd_unit - Write Firmware to NITROX UCD unit
71 static void write_to_ucd_unit(struct nitrox_device *ndev, u32 ucode_size,
72 u64 *ucode_data, int block_num)
90 * Total of 8 blocks, each size 32KB
93 /* set the block number */
94 offset = UCD_UCODE_LOAD_BLOCK_NUM;
95 nitrox_write_csr(ndev, offset, block_num);
97 code_size = roundup(ucode_size, 16);
100 /* write 8 bytes at a time */
101 offset = UCD_UCODE_LOAD_IDX_DATAX(i);
102 nitrox_write_csr(ndev, offset, data);
107 usleep_range(300, 400);
110 static int nitrox_load_fw(struct nitrox_device *ndev)
112 const struct firmware *fw;
117 union ucd_core_eid_ucode_block_num core_2_eid_val;
118 union aqm_grp_execmsk_lo aqm_grp_execmask_lo;
119 union aqm_grp_execmsk_hi aqm_grp_execmask_hi;
124 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
126 ret = request_firmware(&fw, fw_name, DEV(ndev));
128 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
132 ucode = (struct ucode *)fw->data;
134 ucode_size = be32_to_cpu(ucode->code_size) * 2;
135 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
136 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
137 ucode_size, fw_name);
138 release_firmware(fw);
141 ucode_data = ucode->code;
143 /* copy the firmware version */
144 memcpy(&ndev->hw.fw_name[0][0], ucode->version, (VERSION_LEN - 2));
145 ndev->hw.fw_name[0][VERSION_LEN - 1] = '\0';
147 /* Load SE Firmware on UCD Block 0 */
148 write_to_ucd_unit(ndev, ucode_size, ucode_data, 0);
150 release_firmware(fw);
152 /* put all SE cores in DEFAULT_SE_GROUP */
153 offset = POM_GRP_EXECMASKX(DEFAULT_SE_GROUP);
154 nitrox_write_csr(ndev, offset, (~0ULL));
156 /* write block number and firmware length
157 * bit:<2:0> block number
158 * bit:3 is set SE uses 32KB microcode
159 * bit:3 is clear SE uses 64KB microcode
161 core_2_eid_val.value = 0ULL;
162 core_2_eid_val.ucode_blk = 0;
163 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
164 core_2_eid_val.ucode_len = 1;
166 core_2_eid_val.ucode_len = 0;
168 for (i = 0; i < ndev->hw.se_cores; i++) {
169 offset = UCD_SE_EID_UCODE_BLOCK_NUMX(i);
170 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
175 dev_info(DEV(ndev), "Loading firmware \"%s\"\n", fw_name);
177 ret = request_firmware(&fw, fw_name, DEV(ndev));
179 dev_err(DEV(ndev), "failed to get firmware %s\n", fw_name);
183 ucode = (struct ucode *)fw->data;
185 ucode_size = be32_to_cpu(ucode->code_size) * 2;
186 if (!ucode_size || ucode_size > CNN55XX_MAX_UCODE_SIZE) {
187 dev_err(DEV(ndev), "Invalid ucode size: %u for firmware %s\n",
188 ucode_size, fw_name);
189 release_firmware(fw);
192 ucode_data = ucode->code;
194 /* copy the firmware version */
195 memcpy(&ndev->hw.fw_name[1][0], ucode->version, (VERSION_LEN - 2));
196 ndev->hw.fw_name[1][VERSION_LEN - 1] = '\0';
198 /* Load AE Firmware on UCD Block 2 */
199 write_to_ucd_unit(ndev, ucode_size, ucode_data, 2);
201 release_firmware(fw);
203 /* put all AE cores in DEFAULT_AE_GROUP */
204 offset = AQM_GRP_EXECMSK_LOX(DEFAULT_AE_GROUP);
205 aqm_grp_execmask_lo.exec_0_to_39 = 0xFFFFFFFFFFULL;
206 nitrox_write_csr(ndev, offset, aqm_grp_execmask_lo.value);
207 offset = AQM_GRP_EXECMSK_HIX(DEFAULT_AE_GROUP);
208 aqm_grp_execmask_hi.exec_40_to_79 = 0xFFFFFFFFFFULL;
209 nitrox_write_csr(ndev, offset, aqm_grp_execmask_hi.value);
211 /* write block number and firmware length
212 * bit:<2:0> block number
213 * bit:3 is set AE uses 32KB microcode
214 * bit:3 is clear AE uses 64KB microcode
216 core_2_eid_val.value = 0ULL;
217 core_2_eid_val.ucode_blk = 2;
218 if (ucode_size <= CNN55XX_UCD_BLOCK_SIZE)
219 core_2_eid_val.ucode_len = 1;
221 core_2_eid_val.ucode_len = 0;
223 for (i = 0; i < ndev->hw.ae_cores; i++) {
224 offset = UCD_AE_EID_UCODE_BLOCK_NUMX(i);
225 nitrox_write_csr(ndev, offset, core_2_eid_val.value);
232 * nitrox_add_to_devlist - add NITROX device to global device list
233 * @ndev: NITROX device
235 static int nitrox_add_to_devlist(struct nitrox_device *ndev)
237 struct nitrox_device *dev;
240 INIT_LIST_HEAD(&ndev->list);
241 refcount_set(&ndev->refcnt, 1);
243 mutex_lock(&devlist_lock);
244 list_for_each_entry(dev, &ndevlist, list) {
250 ndev->idx = num_devices++;
251 list_add_tail(&ndev->list, &ndevlist);
253 mutex_unlock(&devlist_lock);
258 * nitrox_remove_from_devlist - remove NITROX device from
260 * @ndev: NITROX device
262 static void nitrox_remove_from_devlist(struct nitrox_device *ndev)
264 mutex_lock(&devlist_lock);
265 list_del(&ndev->list);
267 mutex_unlock(&devlist_lock);
270 struct nitrox_device *nitrox_get_first_device(void)
272 struct nitrox_device *ndev;
274 mutex_lock(&devlist_lock);
275 list_for_each_entry(ndev, &ndevlist, list) {
276 if (nitrox_ready(ndev))
279 mutex_unlock(&devlist_lock);
280 if (&ndev->list == &ndevlist)
283 refcount_inc(&ndev->refcnt);
284 /* barrier to sync with other cpus */
285 smp_mb__after_atomic();
289 void nitrox_put_device(struct nitrox_device *ndev)
294 refcount_dec(&ndev->refcnt);
295 /* barrier to sync with other cpus */
296 smp_mb__after_atomic();
299 static int nitrox_device_flr(struct pci_dev *pdev)
303 pos = pci_save_state(pdev);
305 dev_err(&pdev->dev, "Failed to save pci state\n");
309 /* check flr support */
310 if (pcie_has_flr(pdev))
313 pci_restore_state(pdev);
318 static int nitrox_pf_sw_init(struct nitrox_device *ndev)
322 err = nitrox_common_sw_init(ndev);
326 err = nitrox_register_interrupts(ndev);
328 nitrox_common_sw_cleanup(ndev);
333 static void nitrox_pf_sw_cleanup(struct nitrox_device *ndev)
335 nitrox_unregister_interrupts(ndev);
336 nitrox_common_sw_cleanup(ndev);
340 * nitrox_bist_check - Check NITROX BIST registers status
341 * @ndev: NITROX device
343 static int nitrox_bist_check(struct nitrox_device *ndev)
348 for (i = 0; i < NR_CLUSTERS; i++) {
349 value += nitrox_read_csr(ndev, EMU_BIST_STATUSX(i));
350 value += nitrox_read_csr(ndev, EFL_CORE_BIST_REGX(i));
352 value += nitrox_read_csr(ndev, UCD_BIST_STATUS);
353 value += nitrox_read_csr(ndev, NPS_CORE_BIST_REG);
354 value += nitrox_read_csr(ndev, NPS_CORE_NPC_BIST_REG);
355 value += nitrox_read_csr(ndev, NPS_PKT_SLC_BIST_REG);
356 value += nitrox_read_csr(ndev, NPS_PKT_IN_BIST_REG);
357 value += nitrox_read_csr(ndev, POM_BIST_REG);
358 value += nitrox_read_csr(ndev, BMI_BIST_REG);
359 value += nitrox_read_csr(ndev, EFL_TOP_BIST_STAT);
360 value += nitrox_read_csr(ndev, BMO_BIST_REG);
361 value += nitrox_read_csr(ndev, LBC_BIST_STATUS);
362 value += nitrox_read_csr(ndev, PEM_BIST_STATUSX(0));
368 static int nitrox_pf_hw_init(struct nitrox_device *ndev)
372 err = nitrox_bist_check(ndev);
374 dev_err(&ndev->pdev->dev, "BIST check failed\n");
377 /* get cores information */
378 nitrox_get_hwinfo(ndev);
380 nitrox_config_nps_core_unit(ndev);
381 nitrox_config_aqm_unit(ndev);
382 nitrox_config_nps_pkt_unit(ndev);
383 nitrox_config_pom_unit(ndev);
384 nitrox_config_efl_unit(ndev);
385 /* configure IO units */
386 nitrox_config_bmi_unit(ndev);
387 nitrox_config_bmo_unit(ndev);
388 /* configure Local Buffer Cache */
389 nitrox_config_lbc_unit(ndev);
390 nitrox_config_rand_unit(ndev);
392 /* load firmware on cores */
393 err = nitrox_load_fw(ndev);
397 nitrox_config_emu_unit(ndev);
403 * nitrox_probe - NITROX Initialization function.
404 * @pdev: PCI device information struct
405 * @id: entry in nitrox_pci_tbl
407 * Return: 0, if the driver is bound to the device, or
408 * a negative error if there is failure.
410 static int nitrox_probe(struct pci_dev *pdev,
411 const struct pci_device_id *id)
413 struct nitrox_device *ndev;
416 dev_info_once(&pdev->dev, "%s driver version %s\n",
417 nitrox_driver_name, DRIVER_VERSION);
419 err = pci_enable_device_mem(pdev);
424 err = nitrox_device_flr(pdev);
426 dev_err(&pdev->dev, "FLR failed\n");
427 pci_disable_device(pdev);
431 if (!dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64))) {
432 dev_dbg(&pdev->dev, "DMA to 64-BIT address\n");
434 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
436 dev_err(&pdev->dev, "DMA configuration failed\n");
437 pci_disable_device(pdev);
442 err = pci_request_mem_regions(pdev, nitrox_driver_name);
444 pci_disable_device(pdev);
447 pci_set_master(pdev);
449 ndev = kzalloc(sizeof(*ndev), GFP_KERNEL);
455 pci_set_drvdata(pdev, ndev);
458 /* add to device list */
459 nitrox_add_to_devlist(ndev);
461 ndev->hw.vendor_id = pdev->vendor;
462 ndev->hw.device_id = pdev->device;
463 ndev->hw.revision_id = pdev->revision;
464 /* command timeout in jiffies */
465 ndev->timeout = msecs_to_jiffies(CMD_TIMEOUT);
466 ndev->node = dev_to_node(&pdev->dev);
467 if (ndev->node == NUMA_NO_NODE)
470 ndev->bar_addr = ioremap(pci_resource_start(pdev, 0),
471 pci_resource_len(pdev, 0));
472 if (!ndev->bar_addr) {
476 /* allocate command queus based on cpus, max queues are 64 */
477 ndev->nr_queues = min_t(u32, MAX_PF_QUEUES, num_online_cpus());
480 err = nitrox_pf_sw_init(ndev);
484 err = nitrox_pf_hw_init(ndev);
488 nitrox_debugfs_init(ndev);
490 /* clear the statistics */
491 atomic64_set(&ndev->stats.posted, 0);
492 atomic64_set(&ndev->stats.completed, 0);
493 atomic64_set(&ndev->stats.dropped, 0);
495 atomic_set(&ndev->state, __NDEV_READY);
496 /* barrier to sync with other cpus */
497 smp_mb__after_atomic();
499 err = nitrox_crypto_register();
506 nitrox_debugfs_exit(ndev);
507 atomic_set(&ndev->state, __NDEV_NOT_READY);
508 /* barrier to sync with other cpus */
509 smp_mb__after_atomic();
511 nitrox_pf_sw_cleanup(ndev);
513 nitrox_remove_from_devlist(ndev);
515 pci_set_drvdata(pdev, NULL);
517 pci_release_mem_regions(pdev);
518 pci_disable_device(pdev);
523 * nitrox_remove - Unbind the driver from the device.
524 * @pdev: PCI device information struct
526 static void nitrox_remove(struct pci_dev *pdev)
528 struct nitrox_device *ndev = pci_get_drvdata(pdev);
533 if (!refcount_dec_and_test(&ndev->refcnt)) {
534 dev_err(DEV(ndev), "Device refcnt not zero (%d)\n",
535 refcount_read(&ndev->refcnt));
539 dev_info(DEV(ndev), "Removing Device %x:%x\n",
540 ndev->hw.vendor_id, ndev->hw.device_id);
542 atomic_set(&ndev->state, __NDEV_NOT_READY);
543 /* barrier to sync with other cpus */
544 smp_mb__after_atomic();
546 nitrox_remove_from_devlist(ndev);
549 nitrox_sriov_configure(pdev, 0);
550 nitrox_crypto_unregister();
551 nitrox_debugfs_exit(ndev);
552 nitrox_pf_sw_cleanup(ndev);
554 iounmap(ndev->bar_addr);
557 pci_set_drvdata(pdev, NULL);
558 pci_release_mem_regions(pdev);
559 pci_disable_device(pdev);
562 static void nitrox_shutdown(struct pci_dev *pdev)
564 pci_set_drvdata(pdev, NULL);
565 pci_release_mem_regions(pdev);
566 pci_disable_device(pdev);
569 static struct pci_driver nitrox_driver = {
570 .name = nitrox_driver_name,
571 .id_table = nitrox_pci_tbl,
572 .probe = nitrox_probe,
573 .remove = nitrox_remove,
574 .shutdown = nitrox_shutdown,
575 .sriov_configure = nitrox_sriov_configure,
578 module_pci_driver(nitrox_driver);
580 MODULE_AUTHOR("Srikanth Jampala <Jampala.Srikanth@cavium.com>");
581 MODULE_DESCRIPTION("Cavium CNN55XX PF Driver" DRIVER_VERSION " ");
582 MODULE_LICENSE("GPL");
583 MODULE_VERSION(DRIVER_VERSION);
584 MODULE_FIRMWARE(SE_FW);