1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
9 #define pr_fmt(fmt) "habanalabs: " fmt
11 #include "habanalabs.h"
13 #include <linux/pci.h>
14 #include <linux/aer.h>
15 #include <linux/module.h>
17 #define HL_DRIVER_AUTHOR "HabanaLabs Kernel Driver Team"
19 #define HL_DRIVER_DESC "Driver for HabanaLabs's AI Accelerators"
21 MODULE_AUTHOR(HL_DRIVER_AUTHOR);
22 MODULE_DESCRIPTION(HL_DRIVER_DESC);
23 MODULE_LICENSE("GPL v2");
26 static struct class *hl_class;
27 static DEFINE_IDR(hl_devs_idr);
28 static DEFINE_MUTEX(hl_devs_idr_lock);
30 static int timeout_locked = 30;
31 static int reset_on_lockup = 1;
32 static int memory_scrub;
33 static ulong boot_error_status_mask = ULONG_MAX;
35 module_param(timeout_locked, int, 0444);
36 MODULE_PARM_DESC(timeout_locked,
37 "Device lockup timeout in seconds (0 = disabled, default 30s)");
39 module_param(reset_on_lockup, int, 0444);
40 MODULE_PARM_DESC(reset_on_lockup,
41 "Do device reset on lockup (0 = no, 1 = yes, default yes)");
43 module_param(memory_scrub, int, 0444);
44 MODULE_PARM_DESC(memory_scrub,
45 "Scrub device memory in various states (0 = no, 1 = yes, default no)");
47 module_param(boot_error_status_mask, ulong, 0444);
48 MODULE_PARM_DESC(boot_error_status_mask,
49 "Mask of the error status during device CPU boot (If bitX is cleared then error X is masked. Default all 1's)");
51 #define PCI_VENDOR_ID_HABANALABS 0x1da3
53 #define PCI_IDS_GOYA 0x0001
54 #define PCI_IDS_GAUDI 0x1000
55 #define PCI_IDS_GAUDI_SEC 0x1010
57 static const struct pci_device_id ids[] = {
58 { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GOYA), },
59 { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI), },
60 { PCI_DEVICE(PCI_VENDOR_ID_HABANALABS, PCI_IDS_GAUDI_SEC), },
63 MODULE_DEVICE_TABLE(pci, ids);
66 * get_asic_type - translate device id to asic type
68 * @device: id of the PCI device
70 * Translate device id to asic type.
71 * In case of unidentified device, return -1
73 static enum hl_asic_type get_asic_type(u16 device)
75 enum hl_asic_type asic_type;
79 asic_type = ASIC_GOYA;
82 asic_type = ASIC_GAUDI;
84 case PCI_IDS_GAUDI_SEC:
85 asic_type = ASIC_GAUDI_SEC;
88 asic_type = ASIC_INVALID;
95 static bool is_asic_secured(enum hl_asic_type asic_type)
106 * hl_device_open - open function for habanalabs device
108 * @inode: pointer to inode structure
109 * @filp: pointer to file structure
111 * Called when process opens an habanalabs device.
113 int hl_device_open(struct inode *inode, struct file *filp)
115 enum hl_device_status status;
116 struct hl_device *hdev;
117 struct hl_fpriv *hpriv;
120 mutex_lock(&hl_devs_idr_lock);
121 hdev = idr_find(&hl_devs_idr, iminor(inode));
122 mutex_unlock(&hl_devs_idr_lock);
125 pr_err("Couldn't find device %d:%d\n",
126 imajor(inode), iminor(inode));
130 hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
135 filp->private_data = hpriv;
137 mutex_init(&hpriv->restore_phase_mutex);
138 kref_init(&hpriv->refcount);
139 nonseekable_open(inode, filp);
141 hl_cb_mgr_init(&hpriv->cb_mgr);
142 hl_ctx_mgr_init(&hpriv->ctx_mgr);
144 hpriv->taskpid = find_get_pid(current->pid);
146 mutex_lock(&hdev->fpriv_list_lock);
148 if (!hl_device_operational(hdev, &status)) {
149 dev_err_ratelimited(hdev->dev,
150 "Can't open %s because it is %s\n",
151 dev_name(hdev->dev), hdev->status[status]);
156 if (hdev->in_debug) {
157 dev_err_ratelimited(hdev->dev,
158 "Can't open %s because it is being debugged by another user\n",
159 dev_name(hdev->dev));
164 if (hdev->compute_ctx) {
165 dev_dbg_ratelimited(hdev->dev,
166 "Can't open %s because another user is working on it\n",
167 dev_name(hdev->dev));
172 rc = hl_ctx_create(hdev, hpriv);
174 dev_err(hdev->dev, "Failed to create context %d\n", rc);
178 /* Device is IDLE at this point so it is legal to change PLLs.
179 * There is no need to check anything because if the PLL is
180 * already HIGH, the set function will return without doing
183 hl_device_set_frequency(hdev, PLL_HIGH);
185 list_add(&hpriv->dev_node, &hdev->fpriv_list);
186 mutex_unlock(&hdev->fpriv_list_lock);
188 hl_debugfs_add_file(hpriv);
190 hdev->open_counter++;
191 hdev->last_successful_open_jif = jiffies;
196 mutex_unlock(&hdev->fpriv_list_lock);
198 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
199 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
200 filp->private_data = NULL;
201 mutex_destroy(&hpriv->restore_phase_mutex);
202 put_pid(hpriv->taskpid);
209 int hl_device_open_ctrl(struct inode *inode, struct file *filp)
211 struct hl_device *hdev;
212 struct hl_fpriv *hpriv;
215 mutex_lock(&hl_devs_idr_lock);
216 hdev = idr_find(&hl_devs_idr, iminor(inode));
217 mutex_unlock(&hl_devs_idr_lock);
220 pr_err("Couldn't find device %d:%d\n",
221 imajor(inode), iminor(inode));
225 hpriv = kzalloc(sizeof(*hpriv), GFP_KERNEL);
229 mutex_lock(&hdev->fpriv_list_lock);
231 if (!hl_device_operational(hdev, NULL)) {
232 dev_err_ratelimited(hdev->dev_ctrl,
233 "Can't open %s because it is disabled or in reset\n",
234 dev_name(hdev->dev_ctrl));
239 list_add(&hpriv->dev_node, &hdev->fpriv_list);
240 mutex_unlock(&hdev->fpriv_list_lock);
243 filp->private_data = hpriv;
245 hpriv->is_control = true;
246 nonseekable_open(inode, filp);
248 hpriv->taskpid = find_get_pid(current->pid);
253 mutex_unlock(&hdev->fpriv_list_lock);
258 static void set_driver_behavior_per_device(struct hl_device *hdev)
260 hdev->fw_components = FW_TYPE_ALL_TYPES;
261 hdev->cpu_queues_enable = 1;
263 hdev->mmu_enable = 1;
264 hdev->clock_gating_mask = ULONG_MAX;
265 hdev->sram_scrambler_enable = 1;
266 hdev->dram_scrambler_enable = 1;
267 hdev->bmc_enable = 1;
268 hdev->hard_reset_on_fw_events = 1;
269 hdev->reset_on_preboot_fail = 1;
270 hdev->reset_if_device_not_idle = 1;
272 hdev->reset_pcilink = 0;
277 * create_hdev - create habanalabs device instance
279 * @dev: will hold the pointer to the new habanalabs device structure
280 * @pdev: pointer to the pci device
281 * @asic_type: in case of simulator device, which device is it
282 * @minor: in case of simulator device, the minor of the device
284 * Allocate memory for habanalabs device and initialize basic fields
285 * Identify the ASIC type
286 * Allocate ID (minor) for the device (only for real devices)
288 int create_hdev(struct hl_device **dev, struct pci_dev *pdev,
289 enum hl_asic_type asic_type, int minor)
291 struct hl_device *hdev;
292 int rc, main_id, ctrl_id = 0;
296 hdev = kzalloc(sizeof(*hdev), GFP_KERNEL);
300 /* First, we must find out which ASIC are we handling. This is needed
301 * to configure the behavior of the driver (kernel parameters)
304 hdev->asic_type = get_asic_type(pdev->device);
305 if (hdev->asic_type == ASIC_INVALID) {
306 dev_err(&pdev->dev, "Unsupported ASIC\n");
311 hdev->asic_type = asic_type;
315 hdev->asic_prop.fw_security_enabled =
316 is_asic_secured(hdev->asic_type);
318 hdev->asic_prop.fw_security_enabled = false;
320 /* Assign status description string */
321 strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION],
322 "disabled", HL_STR_MAX);
323 strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET],
324 "in reset", HL_STR_MAX);
325 strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET],
326 "needs reset", HL_STR_MAX);
328 hdev->major = hl_major;
329 hdev->reset_on_lockup = reset_on_lockup;
330 hdev->memory_scrub = memory_scrub;
331 hdev->boot_error_status_mask = boot_error_status_mask;
332 hdev->stop_on_err = true;
336 set_driver_behavior_per_device(hdev);
338 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
341 hdev->timeout_jiffies = msecs_to_jiffies(timeout_locked * 1000);
343 hdev->timeout_jiffies = MAX_SCHEDULE_TIMEOUT;
345 hdev->disabled = true;
346 hdev->pdev = pdev; /* can be NULL in case of simulator device */
348 /* Set default DMA mask to 32 bits */
351 mutex_lock(&hl_devs_idr_lock);
353 /* Always save 2 numbers, 1 for main device and 1 for control.
354 * They must be consecutive
356 main_id = idr_alloc(&hl_devs_idr, hdev, 0, HL_MAX_MINORS,
360 ctrl_id = idr_alloc(&hl_devs_idr, hdev, main_id + 1,
361 main_id + 2, GFP_KERNEL);
363 mutex_unlock(&hl_devs_idr_lock);
365 if ((main_id < 0) || (ctrl_id < 0)) {
366 if ((main_id == -ENOSPC) || (ctrl_id == -ENOSPC))
367 pr_err("too many devices in the system\n");
370 mutex_lock(&hl_devs_idr_lock);
371 idr_remove(&hl_devs_idr, main_id);
372 mutex_unlock(&hl_devs_idr_lock);
380 hdev->id_control = ctrl_id;
392 * destroy_hdev - destroy habanalabs device instance
394 * @dev: pointer to the habanalabs device structure
397 void destroy_hdev(struct hl_device *hdev)
399 /* Remove device from the device list */
400 mutex_lock(&hl_devs_idr_lock);
401 idr_remove(&hl_devs_idr, hdev->id);
402 idr_remove(&hl_devs_idr, hdev->id_control);
403 mutex_unlock(&hl_devs_idr_lock);
408 static int hl_pmops_suspend(struct device *dev)
410 struct hl_device *hdev = dev_get_drvdata(dev);
412 pr_debug("Going to suspend PCI device\n");
415 pr_err("device pointer is NULL in suspend\n");
419 return hl_device_suspend(hdev);
422 static int hl_pmops_resume(struct device *dev)
424 struct hl_device *hdev = dev_get_drvdata(dev);
426 pr_debug("Going to resume PCI device\n");
429 pr_err("device pointer is NULL in resume\n");
433 return hl_device_resume(hdev);
437 * hl_pci_probe - probe PCI habanalabs devices
439 * @pdev: pointer to pci device
440 * @id: pointer to pci device id structure
442 * Standard PCI probe function for habanalabs device.
443 * Create a new habanalabs device and initialize it according to the
446 static int hl_pci_probe(struct pci_dev *pdev,
447 const struct pci_device_id *id)
449 struct hl_device *hdev;
452 dev_info(&pdev->dev, HL_NAME
453 " device found [%04x:%04x] (rev %x)\n",
454 (int)pdev->vendor, (int)pdev->device, (int)pdev->revision);
456 rc = create_hdev(&hdev, pdev, ASIC_INVALID, -1);
460 pci_set_drvdata(pdev, hdev);
462 pci_enable_pcie_error_reporting(pdev);
464 rc = hl_device_init(hdev, hl_class);
466 dev_err(&pdev->dev, "Fatal error during habanalabs device init\n");
474 pci_disable_pcie_error_reporting(pdev);
475 pci_set_drvdata(pdev, NULL);
482 * hl_pci_remove - remove PCI habanalabs devices
484 * @pdev: pointer to pci device
486 * Standard PCI remove function for habanalabs device
488 static void hl_pci_remove(struct pci_dev *pdev)
490 struct hl_device *hdev;
492 hdev = pci_get_drvdata(pdev);
496 hl_device_fini(hdev);
497 pci_disable_pcie_error_reporting(pdev);
498 pci_set_drvdata(pdev, NULL);
503 * hl_pci_err_detected - a PCI bus error detected on this device
505 * @pdev: pointer to pci device
506 * @state: PCI error type
508 * Called by the PCI subsystem whenever a non-correctable
509 * PCI bus error is detected
511 static pci_ers_result_t
512 hl_pci_err_detected(struct pci_dev *pdev, pci_channel_state_t state)
514 struct hl_device *hdev = pci_get_drvdata(pdev);
515 enum pci_ers_result result;
518 case pci_channel_io_normal:
519 return PCI_ERS_RESULT_CAN_RECOVER;
521 case pci_channel_io_frozen:
522 dev_warn(hdev->dev, "frozen state error detected\n");
523 result = PCI_ERS_RESULT_NEED_RESET;
526 case pci_channel_io_perm_failure:
527 dev_warn(hdev->dev, "failure state error detected\n");
528 result = PCI_ERS_RESULT_DISCONNECT;
532 result = PCI_ERS_RESULT_NONE;
535 hdev->asic_funcs->halt_engines(hdev, true);
541 * hl_pci_err_resume - resume after a PCI slot reset
543 * @pdev: pointer to pci device
546 static void hl_pci_err_resume(struct pci_dev *pdev)
548 struct hl_device *hdev = pci_get_drvdata(pdev);
550 dev_warn(hdev->dev, "Resuming device after PCI slot reset\n");
551 hl_device_resume(hdev);
555 * hl_pci_err_slot_reset - a PCI slot reset has just happened
557 * @pdev: pointer to pci device
559 * Determine if the driver can recover from the PCI slot reset
561 static pci_ers_result_t hl_pci_err_slot_reset(struct pci_dev *pdev)
563 return PCI_ERS_RESULT_RECOVERED;
566 static const struct dev_pm_ops hl_pm_ops = {
567 .suspend = hl_pmops_suspend,
568 .resume = hl_pmops_resume,
571 static const struct pci_error_handlers hl_pci_err_handler = {
572 .error_detected = hl_pci_err_detected,
573 .slot_reset = hl_pci_err_slot_reset,
574 .resume = hl_pci_err_resume,
577 static struct pci_driver hl_pci_driver = {
580 .probe = hl_pci_probe,
581 .remove = hl_pci_remove,
582 .shutdown = hl_pci_remove,
586 .probe_type = PROBE_PREFER_ASYNCHRONOUS,
588 .err_handler = &hl_pci_err_handler,
592 * hl_init - Initialize the habanalabs kernel driver
594 static int __init hl_init(void)
599 pr_info("loading driver\n");
601 rc = alloc_chrdev_region(&dev, 0, HL_MAX_MINORS, HL_NAME);
603 pr_err("unable to get major\n");
607 hl_major = MAJOR(dev);
609 hl_class = class_create(THIS_MODULE, HL_NAME);
610 if (IS_ERR(hl_class)) {
611 pr_err("failed to allocate class\n");
612 rc = PTR_ERR(hl_class);
618 rc = pci_register_driver(&hl_pci_driver);
620 pr_err("failed to register pci device\n");
624 pr_debug("driver loaded\n");
630 class_destroy(hl_class);
632 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
637 * hl_exit - Release all resources of the habanalabs kernel driver
639 static void __exit hl_exit(void)
641 pci_unregister_driver(&hl_pci_driver);
644 * Removing debugfs must be after all devices or simulator devices
645 * have been removed because otherwise we get a bug in the
646 * debugfs module for referencing NULL objects
650 class_destroy(hl_class);
651 unregister_chrdev_region(MKDEV(hl_major, 0), HL_MAX_MINORS);
653 idr_destroy(&hl_devs_idr);
655 pr_debug("driver removed\n");
658 module_init(hl_init);
659 module_exit(hl_exit);