1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #define pr_fmt(fmt) "habanalabs: " fmt
10 #include <uapi/misc/habanalabs.h>
11 #include "habanalabs.h"
13 #include <linux/pci.h>
14 #include <linux/hwmon.h>
16 enum hl_device_status hl_device_status(struct hl_device *hdev)
18 enum hl_device_status status;
20 if (atomic_read(&hdev->in_reset))
21 status = HL_DEVICE_STATUS_IN_RESET;
22 else if (hdev->needs_reset)
23 status = HL_DEVICE_STATUS_NEEDS_RESET;
24 else if (hdev->disabled)
25 status = HL_DEVICE_STATUS_MALFUNCTION;
26 else if (!hdev->init_done)
27 status = HL_DEVICE_STATUS_IN_DEVICE_CREATION;
29 status = HL_DEVICE_STATUS_OPERATIONAL;
34 bool hl_device_operational(struct hl_device *hdev,
35 enum hl_device_status *status)
37 enum hl_device_status current_status;
39 current_status = hl_device_status(hdev);
41 *status = current_status;
43 switch (current_status) {
44 case HL_DEVICE_STATUS_IN_RESET:
45 case HL_DEVICE_STATUS_MALFUNCTION:
46 case HL_DEVICE_STATUS_NEEDS_RESET:
48 case HL_DEVICE_STATUS_OPERATIONAL:
49 case HL_DEVICE_STATUS_IN_DEVICE_CREATION:
55 static void hpriv_release(struct kref *ref)
57 u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
58 bool device_is_idle = true;
59 struct hl_fpriv *hpriv;
60 struct hl_device *hdev;
62 hpriv = container_of(ref, struct hl_fpriv, refcount);
66 put_pid(hpriv->taskpid);
68 hl_debugfs_remove_file(hpriv);
70 mutex_destroy(&hpriv->restore_phase_mutex);
72 mutex_lock(&hdev->fpriv_list_lock);
73 list_del(&hpriv->dev_node);
74 hdev->compute_ctx = NULL;
75 mutex_unlock(&hdev->fpriv_list_lock);
79 if ((!hdev->pldm) && (hdev->pdev) &&
80 (!hdev->asic_funcs->is_device_idle(hdev,
82 HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL))) {
84 "device not idle after user context is closed (0x%llx_%llx)\n",
85 idle_mask[1], idle_mask[0]);
87 device_is_idle = false;
90 if ((hdev->reset_if_device_not_idle && !device_is_idle)
91 || hdev->reset_upon_device_release)
92 hl_device_reset(hdev, HL_RESET_DEVICE_RELEASE);
95 void hl_hpriv_get(struct hl_fpriv *hpriv)
97 kref_get(&hpriv->refcount);
100 int hl_hpriv_put(struct hl_fpriv *hpriv)
102 return kref_put(&hpriv->refcount, hpriv_release);
106 * hl_device_release - release function for habanalabs device
108 * @inode: pointer to inode structure
109 * @filp: pointer to file structure
111 * Called when process closes an habanalabs device
113 static int hl_device_release(struct inode *inode, struct file *filp)
115 struct hl_fpriv *hpriv = filp->private_data;
116 struct hl_device *hdev = hpriv->hdev;
118 filp->private_data = NULL;
121 pr_crit("Closing FD after device was removed. Memory leak will occur and it is advised to reboot.\n");
122 put_pid(hpriv->taskpid);
126 /* Each pending user interrupt holds the user's context, hence we
127 * must release them all before calling hl_ctx_mgr_fini().
129 hl_release_pending_user_interrupts(hpriv->hdev);
131 hl_cb_mgr_fini(hdev, &hpriv->cb_mgr);
132 hl_ctx_mgr_fini(hdev, &hpriv->ctx_mgr);
134 if (!hl_hpriv_put(hpriv))
135 dev_notice(hdev->dev,
136 "User process closed FD but device still in use\n");
138 hdev->last_open_session_duration_jif =
139 jiffies - hdev->last_successful_open_jif;
144 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
146 struct hl_fpriv *hpriv = filp->private_data;
147 struct hl_device *hdev = hpriv->hdev;
149 filp->private_data = NULL;
152 pr_err("Closing FD after device was removed\n");
156 mutex_lock(&hdev->fpriv_list_lock);
157 list_del(&hpriv->dev_node);
158 mutex_unlock(&hdev->fpriv_list_lock);
160 put_pid(hpriv->taskpid);
168 * hl_mmap - mmap function for habanalabs device
170 * @*filp: pointer to file structure
171 * @*vma: pointer to vm_area_struct of the process
173 * Called when process does an mmap on habanalabs device. Call the device's mmap
174 * function at the end of the common code.
176 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
178 struct hl_fpriv *hpriv = filp->private_data;
179 struct hl_device *hdev = hpriv->hdev;
180 unsigned long vm_pgoff;
183 pr_err_ratelimited("Trying to mmap after device was removed! Please close FD\n");
187 vm_pgoff = vma->vm_pgoff;
188 vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
190 switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
191 case HL_MMAP_TYPE_CB:
192 return hl_cb_mmap(hpriv, vma);
194 case HL_MMAP_TYPE_BLOCK:
195 return hl_hw_block_mmap(hpriv, vma);
201 static const struct file_operations hl_ops = {
202 .owner = THIS_MODULE,
203 .open = hl_device_open,
204 .release = hl_device_release,
206 .unlocked_ioctl = hl_ioctl,
207 .compat_ioctl = hl_ioctl
210 static const struct file_operations hl_ctrl_ops = {
211 .owner = THIS_MODULE,
212 .open = hl_device_open_ctrl,
213 .release = hl_device_release_ctrl,
214 .unlocked_ioctl = hl_ioctl_control,
215 .compat_ioctl = hl_ioctl_control
218 static void device_release_func(struct device *dev)
224 * device_init_cdev - Initialize cdev and device for habanalabs device
226 * @hdev: pointer to habanalabs device structure
227 * @hclass: pointer to the class object of the device
228 * @minor: minor number of the specific device
229 * @fpos: file operations to install for this device
230 * @name: name of the device as it will appear in the filesystem
231 * @cdev: pointer to the char device object that will be initialized
232 * @dev: pointer to the device object that will be initialized
234 * Initialize a cdev and a Linux device for habanalabs's device.
236 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
237 int minor, const struct file_operations *fops,
238 char *name, struct cdev *cdev,
241 cdev_init(cdev, fops);
242 cdev->owner = THIS_MODULE;
244 *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
248 device_initialize(*dev);
249 (*dev)->devt = MKDEV(hdev->major, minor);
250 (*dev)->class = hclass;
251 (*dev)->release = device_release_func;
252 dev_set_drvdata(*dev, hdev);
253 dev_set_name(*dev, "%s", name);
258 static int device_cdev_sysfs_add(struct hl_device *hdev)
262 rc = cdev_device_add(&hdev->cdev, hdev->dev);
265 "failed to add a char device to the system\n");
269 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
272 "failed to add a control char device to the system\n");
273 goto delete_cdev_device;
276 /* hl_sysfs_init() must be done after adding the device to the system */
277 rc = hl_sysfs_init(hdev);
279 dev_err(hdev->dev, "failed to initialize sysfs\n");
280 goto delete_ctrl_cdev_device;
283 hdev->cdev_sysfs_created = true;
287 delete_ctrl_cdev_device:
288 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
290 cdev_device_del(&hdev->cdev, hdev->dev);
294 static void device_cdev_sysfs_del(struct hl_device *hdev)
296 if (!hdev->cdev_sysfs_created)
300 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
301 cdev_device_del(&hdev->cdev, hdev->dev);
304 put_device(hdev->dev);
305 put_device(hdev->dev_ctrl);
308 static void device_hard_reset_pending(struct work_struct *work)
310 struct hl_device_reset_work *device_reset_work =
311 container_of(work, struct hl_device_reset_work,
313 struct hl_device *hdev = device_reset_work->hdev;
317 flags = HL_RESET_HARD | HL_RESET_FROM_RESET_THREAD;
319 if (device_reset_work->fw_reset)
320 flags |= HL_RESET_FW;
322 rc = hl_device_reset(hdev, flags);
323 if ((rc == -EBUSY) && !hdev->device_fini_pending) {
325 "Could not reset device. will try again in %u seconds",
326 HL_PENDING_RESET_PER_SEC);
328 queue_delayed_work(device_reset_work->wq,
329 &device_reset_work->reset_work,
330 msecs_to_jiffies(HL_PENDING_RESET_PER_SEC * 1000));
335 * device_early_init - do some early initialization for the habanalabs device
337 * @hdev: pointer to habanalabs device structure
339 * Install the relevant function pointers and call the early_init function,
340 * if such a function exists
342 static int device_early_init(struct hl_device *hdev)
347 switch (hdev->asic_type) {
349 goya_set_asic_funcs(hdev);
350 strscpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
353 gaudi_set_asic_funcs(hdev);
354 strscpy(hdev->asic_name, "GAUDI", sizeof(hdev->asic_name));
357 gaudi_set_asic_funcs(hdev);
358 strscpy(hdev->asic_name, "GAUDI SEC", sizeof(hdev->asic_name));
361 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
366 rc = hdev->asic_funcs->early_init(hdev);
370 rc = hl_asid_init(hdev);
374 if (hdev->asic_prop.completion_queues_count) {
375 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
376 sizeof(*hdev->cq_wq),
384 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
385 snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
386 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
387 if (hdev->cq_wq[i] == NULL) {
388 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
394 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
395 if (hdev->eq_wq == NULL) {
396 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
401 hdev->sob_reset_wq = alloc_workqueue("hl-sob-reset", WQ_UNBOUND, 0);
402 if (!hdev->sob_reset_wq) {
404 "Failed to allocate SOB reset workqueue\n");
409 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
411 if (!hdev->hl_chip_info) {
413 goto free_sob_reset_wq;
416 rc = hl_mmu_if_set_funcs(hdev);
420 hl_cb_mgr_init(&hdev->kernel_cb_mgr);
422 hdev->device_reset_work.wq =
423 create_singlethread_workqueue("hl_device_reset");
424 if (!hdev->device_reset_work.wq) {
426 dev_err(hdev->dev, "Failed to create device reset WQ\n");
430 INIT_DELAYED_WORK(&hdev->device_reset_work.reset_work,
431 device_hard_reset_pending);
432 hdev->device_reset_work.hdev = hdev;
433 hdev->device_fini_pending = 0;
435 mutex_init(&hdev->send_cpu_message_lock);
436 mutex_init(&hdev->debug_lock);
437 INIT_LIST_HEAD(&hdev->cs_mirror_list);
438 spin_lock_init(&hdev->cs_mirror_lock);
439 INIT_LIST_HEAD(&hdev->fpriv_list);
440 mutex_init(&hdev->fpriv_list_lock);
441 atomic_set(&hdev->in_reset, 0);
446 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
448 kfree(hdev->hl_chip_info);
450 destroy_workqueue(hdev->sob_reset_wq);
452 destroy_workqueue(hdev->eq_wq);
454 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
456 destroy_workqueue(hdev->cq_wq[i]);
461 if (hdev->asic_funcs->early_fini)
462 hdev->asic_funcs->early_fini(hdev);
468 * device_early_fini - finalize all that was done in device_early_init
470 * @hdev: pointer to habanalabs device structure
473 static void device_early_fini(struct hl_device *hdev)
477 mutex_destroy(&hdev->debug_lock);
478 mutex_destroy(&hdev->send_cpu_message_lock);
480 mutex_destroy(&hdev->fpriv_list_lock);
482 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
484 kfree(hdev->hl_chip_info);
486 destroy_workqueue(hdev->sob_reset_wq);
487 destroy_workqueue(hdev->eq_wq);
488 destroy_workqueue(hdev->device_reset_work.wq);
490 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
491 destroy_workqueue(hdev->cq_wq[i]);
496 if (hdev->asic_funcs->early_fini)
497 hdev->asic_funcs->early_fini(hdev);
500 static void set_freq_to_low_job(struct work_struct *work)
502 struct hl_device *hdev = container_of(work, struct hl_device,
505 mutex_lock(&hdev->fpriv_list_lock);
507 if (!hdev->compute_ctx)
508 hl_device_set_frequency(hdev, PLL_LOW);
510 mutex_unlock(&hdev->fpriv_list_lock);
512 schedule_delayed_work(&hdev->work_freq,
513 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
516 static void hl_device_heartbeat(struct work_struct *work)
518 struct hl_device *hdev = container_of(work, struct hl_device,
519 work_heartbeat.work);
521 if (!hl_device_operational(hdev, NULL))
524 if (!hdev->asic_funcs->send_heartbeat(hdev))
527 dev_err(hdev->dev, "Device heartbeat failed!\n");
528 hl_device_reset(hdev, HL_RESET_HARD | HL_RESET_HEARTBEAT);
533 schedule_delayed_work(&hdev->work_heartbeat,
534 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
538 * device_late_init - do late stuff initialization for the habanalabs device
540 * @hdev: pointer to habanalabs device structure
542 * Do stuff that either needs the device H/W queues to be active or needs
543 * to happen after all the rest of the initialization is finished
545 static int device_late_init(struct hl_device *hdev)
549 if (hdev->asic_funcs->late_init) {
550 rc = hdev->asic_funcs->late_init(hdev);
553 "failed late initialization for the H/W\n");
558 hdev->high_pll = hdev->asic_prop.high_pll;
560 /* force setting to low frequency */
561 hdev->curr_pll_profile = PLL_LOW;
563 if (hdev->pm_mng_profile == PM_AUTO)
564 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
566 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
568 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
569 schedule_delayed_work(&hdev->work_freq,
570 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
572 if (hdev->heartbeat) {
573 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
574 schedule_delayed_work(&hdev->work_heartbeat,
575 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
578 hdev->late_init_done = true;
584 * device_late_fini - finalize all that was done in device_late_init
586 * @hdev: pointer to habanalabs device structure
589 static void device_late_fini(struct hl_device *hdev)
591 if (!hdev->late_init_done)
594 cancel_delayed_work_sync(&hdev->work_freq);
596 cancel_delayed_work_sync(&hdev->work_heartbeat);
598 if (hdev->asic_funcs->late_fini)
599 hdev->asic_funcs->late_fini(hdev);
601 hdev->late_init_done = false;
604 int hl_device_utilization(struct hl_device *hdev, u32 *utilization)
606 u64 max_power, curr_power, dc_power, dividend;
609 max_power = hdev->asic_prop.max_power_default;
610 dc_power = hdev->asic_prop.dc_power_default;
611 rc = hl_fw_cpucp_power_get(hdev, &curr_power);
616 curr_power = clamp(curr_power, dc_power, max_power);
618 dividend = (curr_power - dc_power) * 100;
619 *utilization = (u32) div_u64(dividend, (max_power - dc_power));
625 * hl_device_set_frequency - set the frequency of the device
627 * @hdev: pointer to habanalabs device structure
628 * @freq: the new frequency value
630 * Change the frequency if needed. This function has no protection against
631 * concurrency, therefore it is assumed that the calling function has protected
632 * itself against the case of calling this function from multiple threads with
635 * Returns 0 if no change was done, otherwise returns 1
637 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
639 if ((hdev->pm_mng_profile == PM_MANUAL) ||
640 (hdev->curr_pll_profile == freq))
643 dev_dbg(hdev->dev, "Changing device frequency to %s\n",
644 freq == PLL_HIGH ? "high" : "low");
646 hdev->asic_funcs->set_pll_profile(hdev, freq);
648 hdev->curr_pll_profile = freq;
653 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
657 mutex_lock(&hdev->debug_lock);
660 if (!hdev->in_debug) {
662 "Failed to disable debug mode because device was not in debug mode\n");
667 if (!hdev->hard_reset_pending)
668 hdev->asic_funcs->halt_coresight(hdev);
672 if (!hdev->hard_reset_pending)
673 hdev->asic_funcs->set_clock_gating(hdev);
678 if (hdev->in_debug) {
680 "Failed to enable debug mode because device is already in debug mode\n");
685 hdev->asic_funcs->disable_clock_gating(hdev);
689 mutex_unlock(&hdev->debug_lock);
694 static void take_release_locks(struct hl_device *hdev)
696 /* Flush anyone that is inside the critical section of enqueue
699 hdev->asic_funcs->hw_queues_lock(hdev);
700 hdev->asic_funcs->hw_queues_unlock(hdev);
702 /* Flush processes that are sending message to CPU */
703 mutex_lock(&hdev->send_cpu_message_lock);
704 mutex_unlock(&hdev->send_cpu_message_lock);
706 /* Flush anyone that is inside device open */
707 mutex_lock(&hdev->fpriv_list_lock);
708 mutex_unlock(&hdev->fpriv_list_lock);
711 static void cleanup_resources(struct hl_device *hdev, bool hard_reset, bool fw_reset)
714 device_late_fini(hdev);
717 * Halt the engines and disable interrupts so we won't get any more
718 * completions from H/W and we won't have any accesses from the
719 * H/W to the host machine
721 hdev->asic_funcs->halt_engines(hdev, hard_reset, fw_reset);
723 /* Go over all the queues, release all CS and their jobs */
724 hl_cs_rollback_all(hdev);
726 /* Release all pending user interrupts, each pending user interrupt
727 * holds a reference to user context
729 hl_release_pending_user_interrupts(hdev);
733 * hl_device_suspend - initiate device suspend
735 * @hdev: pointer to habanalabs device structure
737 * Puts the hw in the suspend state (all asics).
738 * Returns 0 for success or an error on failure.
739 * Called at driver suspend.
741 int hl_device_suspend(struct hl_device *hdev)
745 pci_save_state(hdev->pdev);
747 /* Block future CS/VM/JOB completion operations */
748 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
750 dev_err(hdev->dev, "Can't suspend while in reset\n");
754 /* This blocks all other stuff that is not blocked by in_reset */
755 hdev->disabled = true;
757 take_release_locks(hdev);
759 rc = hdev->asic_funcs->suspend(hdev);
762 "Failed to disable PCI access of device CPU\n");
764 /* Shut down the device */
765 pci_disable_device(hdev->pdev);
766 pci_set_power_state(hdev->pdev, PCI_D3hot);
772 * hl_device_resume - initiate device resume
774 * @hdev: pointer to habanalabs device structure
776 * Bring the hw back to operating state (all asics).
777 * Returns 0 for success or an error on failure.
778 * Called at driver resume.
780 int hl_device_resume(struct hl_device *hdev)
784 pci_set_power_state(hdev->pdev, PCI_D0);
785 pci_restore_state(hdev->pdev);
786 rc = pci_enable_device_mem(hdev->pdev);
789 "Failed to enable PCI device in resume\n");
793 pci_set_master(hdev->pdev);
795 rc = hdev->asic_funcs->resume(hdev);
797 dev_err(hdev->dev, "Failed to resume device after suspend\n");
802 hdev->disabled = false;
803 atomic_set(&hdev->in_reset, 0);
805 rc = hl_device_reset(hdev, HL_RESET_HARD);
807 dev_err(hdev->dev, "Failed to reset device during resume\n");
814 pci_clear_master(hdev->pdev);
815 pci_disable_device(hdev->pdev);
820 static int device_kill_open_processes(struct hl_device *hdev, u32 timeout)
822 struct hl_fpriv *hpriv;
823 struct task_struct *task = NULL;
827 /* Giving time for user to close FD, and for processes that are inside
828 * hl_device_open to finish
830 if (!list_empty(&hdev->fpriv_list))
834 pending_cnt = timeout;
836 if (hdev->process_kill_trial_cnt) {
837 /* Processes have been already killed */
839 goto wait_for_processes;
841 /* Wait a small period after process kill */
842 pending_cnt = HL_PENDING_RESET_PER_SEC;
846 mutex_lock(&hdev->fpriv_list_lock);
848 /* This section must be protected because we are dereferencing
849 * pointers that are freed if the process exits
851 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
852 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
854 dev_info(hdev->dev, "Killing user process pid=%d\n",
856 send_sig(SIGKILL, task, 1);
857 usleep_range(1000, 10000);
859 put_task_struct(task);
862 "Can't get task struct for PID so giving up on killing process\n");
863 mutex_unlock(&hdev->fpriv_list_lock);
868 mutex_unlock(&hdev->fpriv_list_lock);
871 * We killed the open users, but that doesn't mean they are closed.
872 * It could be that they are running a long cleanup phase in the driver
873 * e.g. MMU unmappings, or running other long teardown flow even before
875 * Therefore we need to wait again to make sure they are closed before
876 * continuing with the reset.
880 while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
882 "Waiting for all unmap operations to finish before hard reset\n");
889 /* All processes exited successfully */
890 if (list_empty(&hdev->fpriv_list))
893 /* Give up waiting for processes to exit */
894 if (hdev->process_kill_trial_cnt == HL_PENDING_RESET_MAX_TRIALS)
897 hdev->process_kill_trial_cnt++;
902 static void device_disable_open_processes(struct hl_device *hdev)
904 struct hl_fpriv *hpriv;
906 mutex_lock(&hdev->fpriv_list_lock);
907 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node)
909 mutex_unlock(&hdev->fpriv_list_lock);
913 * hl_device_reset - reset the device
915 * @hdev: pointer to habanalabs device structure
916 * @flags: reset flags.
918 * Block future CS and wait for pending CS to be enqueued
920 * Flush all completions
921 * Re-initialize all internal data structures
922 * Call ASIC H/W init, late_init
926 * Returns 0 for success or an error on failure.
928 int hl_device_reset(struct hl_device *hdev, u32 flags)
930 u64 idle_mask[HL_BUSY_ENGINES_MASK_EXT_SIZE] = {0};
931 bool hard_reset, from_hard_reset_thread, fw_reset, hard_instead_soft = false;
934 if (!hdev->init_done) {
936 "Can't reset before initialization is done\n");
940 hard_reset = !!(flags & HL_RESET_HARD);
941 from_hard_reset_thread = !!(flags & HL_RESET_FROM_RESET_THREAD);
942 fw_reset = !!(flags & HL_RESET_FW);
944 if (!hard_reset && !hdev->supports_soft_reset) {
945 hard_instead_soft = true;
949 if (hdev->reset_upon_device_release &&
950 (flags & HL_RESET_DEVICE_RELEASE)) {
952 "Perform %s-reset upon device release\n",
953 hard_reset ? "hard" : "soft");
957 if (!hard_reset && !hdev->allow_external_soft_reset) {
958 hard_instead_soft = true;
962 if (hard_instead_soft)
963 dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
966 /* Re-entry of reset thread */
967 if (from_hard_reset_thread && hdev->process_kill_trial_cnt)
971 * Prevent concurrency in this function - only one reset should be
972 * done at any given time. Only need to perform this if we didn't
973 * get from the dedicated hard reset thread
975 if (!from_hard_reset_thread) {
976 /* Block future CS/VM/JOB completion operations */
977 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
982 * 'reset cause' is being updated here, because getting here
983 * means that it's the 1st time and the last time we're here
984 * ('in_reset' makes sure of it). This makes sure that
985 * 'reset_cause' will continue holding its 1st recorded reason!
987 if (flags & HL_RESET_HEARTBEAT)
988 hdev->curr_reset_cause = HL_RESET_CAUSE_HEARTBEAT;
989 else if (flags & HL_RESET_TDR)
990 hdev->curr_reset_cause = HL_RESET_CAUSE_TDR;
992 hdev->curr_reset_cause = HL_RESET_CAUSE_UNKNOWN;
994 /* If reset is due to heartbeat, device CPU is no responsive in
995 * which case no point sending PCI disable message to it.
997 * If F/W is performing the reset, no need to send it a message to disable
1000 if (hard_reset && !(flags & (HL_RESET_HEARTBEAT | HL_RESET_FW))) {
1001 /* Disable PCI access from device F/W so he won't send
1002 * us additional interrupts. We disable MSI/MSI-X at
1003 * the halt_engines function and we can't have the F/W
1004 * sending us interrupts after that. We need to disable
1005 * the access here because if the device is marked
1006 * disable, the message won't be send. Also, in case
1007 * of heartbeat, the device CPU is marked as disable
1008 * so this message won't be sent
1010 if (hl_fw_send_pci_access_msg(hdev,
1011 CPUCP_PACKET_DISABLE_PCI_ACCESS))
1013 "Failed to disable PCI access by F/W\n");
1016 /* This also blocks future CS/VM/JOB completion operations */
1017 hdev->disabled = true;
1019 take_release_locks(hdev);
1021 dev_err(hdev->dev, "Going to RESET device!\n");
1025 if ((hard_reset) && (!from_hard_reset_thread)) {
1026 hdev->hard_reset_pending = true;
1028 hdev->process_kill_trial_cnt = 0;
1030 hdev->device_reset_work.fw_reset = fw_reset;
1033 * Because the reset function can't run from heartbeat work,
1034 * we need to call the reset function from a dedicated work.
1036 queue_delayed_work(hdev->device_reset_work.wq,
1037 &hdev->device_reset_work.reset_work, 0);
1042 cleanup_resources(hdev, hard_reset, fw_reset);
1046 /* Kill processes here after CS rollback. This is because the
1047 * process can't really exit until all its CSs are done, which
1048 * is what we do in cs rollback
1050 rc = device_kill_open_processes(hdev, 0);
1053 if (hdev->device_fini_pending) {
1055 "Failed to kill all open processes, stopping hard reset\n");
1059 /* signal reset thread to reschedule */
1065 "Failed to kill all open processes, stopping hard reset\n");
1069 /* Flush the Event queue workers to make sure no other thread is
1070 * reading or writing to registers during the reset
1072 flush_workqueue(hdev->eq_wq);
1075 /* Reset the H/W. It will be in idle state after this returns */
1076 hdev->asic_funcs->hw_fini(hdev, hard_reset, fw_reset);
1079 hdev->fw_loader.linux_loaded = false;
1081 /* Release kernel context */
1082 if (hdev->kernel_ctx && hl_ctx_put(hdev->kernel_ctx) == 1)
1083 hdev->kernel_ctx = NULL;
1087 hl_eq_reset(hdev, &hdev->event_queue);
1090 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
1091 hl_hw_queue_reset(hdev, hard_reset);
1092 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1093 hl_cq_reset(hdev, &hdev->completion_queue[i]);
1095 mutex_lock(&hdev->fpriv_list_lock);
1097 /* Make sure the context switch phase will run again */
1098 if (hdev->compute_ctx) {
1099 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
1100 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
1103 mutex_unlock(&hdev->fpriv_list_lock);
1105 /* Finished tear-down, starting to re-initialize */
1108 hdev->device_cpu_disabled = false;
1109 hdev->hard_reset_pending = false;
1111 if (hdev->kernel_ctx) {
1113 "kernel ctx was alive during hard reset, something is terribly wrong\n");
1118 rc = hl_mmu_init(hdev);
1121 "Failed to initialize MMU S/W after hard reset\n");
1125 /* Allocate the kernel context */
1126 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1128 if (!hdev->kernel_ctx) {
1134 hdev->compute_ctx = NULL;
1136 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1139 "failed to init kernel ctx in hard reset\n");
1140 kfree(hdev->kernel_ctx);
1141 hdev->kernel_ctx = NULL;
1147 /* Device is now enabled as part of the initialization requires
1148 * communication with the device firmware to get information that
1149 * is required for the initialization itself
1151 hdev->disabled = false;
1153 rc = hdev->asic_funcs->hw_init(hdev);
1156 "failed to initialize the H/W after reset\n");
1160 /* If device is not idle fail the reset process */
1161 if (!hdev->asic_funcs->is_device_idle(hdev, idle_mask,
1162 HL_BUSY_ENGINES_MASK_EXT_SIZE, NULL)) {
1164 "device is not idle (mask 0x%llx_%llx) after reset\n",
1165 idle_mask[1], idle_mask[0]);
1170 /* Check that the communication with the device is working */
1171 rc = hdev->asic_funcs->test_queues(hdev);
1174 "Failed to detect if device is alive after reset\n");
1179 rc = device_late_init(hdev);
1182 "Failed late init after hard reset\n");
1186 rc = hl_vm_init(hdev);
1189 "Failed to init memory module after hard reset\n");
1193 hl_set_max_power(hdev);
1195 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
1198 "Failed late init after soft reset\n");
1203 atomic_set(&hdev->in_reset, 0);
1204 hdev->needs_reset = false;
1206 dev_notice(hdev->dev, "Successfully finished resetting the device\n");
1209 hdev->hard_reset_cnt++;
1211 /* After reset is done, we are ready to receive events from
1212 * the F/W. We can't do it before because we will ignore events
1213 * and if those events are fatal, we won't know about it and
1214 * the device will be operational although it shouldn't be
1216 hdev->asic_funcs->enable_events_from_fw(hdev);
1218 hdev->soft_reset_cnt++;
1224 hdev->disabled = true;
1228 "Failed to reset! Device is NOT usable\n");
1229 hdev->hard_reset_cnt++;
1232 "Failed to do soft-reset, trying hard reset\n");
1233 hdev->soft_reset_cnt++;
1238 atomic_set(&hdev->in_reset, 0);
1244 * hl_device_init - main initialization function for habanalabs device
1246 * @hdev: pointer to habanalabs device structure
1248 * Allocate an id for the device, do early initialization and then call the
1249 * ASIC specific initialization functions. Finally, create the cdev and the
1250 * Linux device to expose it to the user
1252 int hl_device_init(struct hl_device *hdev, struct class *hclass)
1254 int i, rc, cq_cnt, user_interrupt_cnt, cq_ready_cnt;
1256 bool add_cdev_sysfs_on_err = false;
1258 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
1264 /* Initialize cdev and device structures */
1265 rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
1266 &hdev->cdev, &hdev->dev);
1273 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
1279 /* Initialize cdev and device structures for control device */
1280 rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
1281 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
1288 /* Initialize ASIC function pointers and perform early init */
1289 rc = device_early_init(hdev);
1293 user_interrupt_cnt = hdev->asic_prop.user_interrupt_count;
1295 if (user_interrupt_cnt) {
1296 hdev->user_interrupt = kcalloc(user_interrupt_cnt,
1297 sizeof(*hdev->user_interrupt),
1300 if (!hdev->user_interrupt) {
1307 * Start calling ASIC initialization. First S/W then H/W and finally
1310 rc = hdev->asic_funcs->sw_init(hdev);
1312 goto user_interrupts_fini;
1315 /* initialize completion structure for multi CS wait */
1316 hl_multi_cs_completion_init(hdev);
1319 * Initialize the H/W queues. Must be done before hw_init, because
1320 * there the addresses of the kernel queue are being written to the
1321 * registers of the device
1323 rc = hl_hw_queues_create(hdev);
1325 dev_err(hdev->dev, "failed to initialize kernel queues\n");
1329 cq_cnt = hdev->asic_prop.completion_queues_count;
1332 * Initialize the completion queues. Must be done before hw_init,
1333 * because there the addresses of the completion queues are being
1334 * passed as arguments to request_irq
1337 hdev->completion_queue = kcalloc(cq_cnt,
1338 sizeof(*hdev->completion_queue),
1341 if (!hdev->completion_queue) {
1343 "failed to allocate completion queues\n");
1345 goto hw_queues_destroy;
1349 for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
1350 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
1351 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
1354 "failed to initialize completion queue\n");
1357 hdev->completion_queue[i].cq_idx = i;
1361 * Initialize the event queue. Must be done before hw_init,
1362 * because there the address of the event queue is being
1363 * passed as argument to request_irq
1365 rc = hl_eq_init(hdev, &hdev->event_queue);
1367 dev_err(hdev->dev, "failed to initialize event queue\n");
1371 /* MMU S/W must be initialized before kernel context is created */
1372 rc = hl_mmu_init(hdev);
1374 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
1378 /* Allocate the kernel context */
1379 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1380 if (!hdev->kernel_ctx) {
1385 hdev->compute_ctx = NULL;
1387 hdev->asic_funcs->state_dump_init(hdev);
1389 hl_debugfs_add_device(hdev);
1391 /* debugfs nodes are created in hl_ctx_init so it must be called after
1392 * hl_debugfs_add_device.
1394 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1396 dev_err(hdev->dev, "failed to initialize kernel context\n");
1397 kfree(hdev->kernel_ctx);
1398 goto remove_device_from_debugfs;
1401 rc = hl_cb_pool_init(hdev);
1403 dev_err(hdev->dev, "failed to initialize CB pool\n");
1408 * From this point, override rc (=0) in case of an error to allow
1409 * debugging (by adding char devices and create sysfs nodes as part of
1412 add_cdev_sysfs_on_err = true;
1414 /* Device is now enabled as part of the initialization requires
1415 * communication with the device firmware to get information that
1416 * is required for the initialization itself
1418 hdev->disabled = false;
1420 rc = hdev->asic_funcs->hw_init(hdev);
1422 dev_err(hdev->dev, "failed to initialize the H/W\n");
1427 /* Check that the communication with the device is working */
1428 rc = hdev->asic_funcs->test_queues(hdev);
1430 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1435 rc = device_late_init(hdev);
1437 dev_err(hdev->dev, "Failed late initialization\n");
1442 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1444 hdev->asic_prop.dram_size / SZ_1G);
1446 rc = hl_vm_init(hdev);
1448 dev_err(hdev->dev, "Failed to initialize memory module\n");
1454 * Expose devices and sysfs nodes to user.
1455 * From here there is no need to add char devices and create sysfs nodes
1456 * in case of an error.
1458 add_cdev_sysfs_on_err = false;
1459 rc = device_cdev_sysfs_add(hdev);
1462 "Failed to add char devices and sysfs nodes\n");
1467 /* Need to call this again because the max power might change,
1468 * depending on card type for certain ASICs
1470 hl_set_max_power(hdev);
1473 * hl_hwmon_init() must be called after device_late_init(), because only
1474 * there we get the information from the device about which
1475 * hwmon-related sensors the device supports.
1476 * Furthermore, it must be done after adding the device to the system.
1478 rc = hl_hwmon_init(hdev);
1480 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1485 dev_notice(hdev->dev,
1486 "Successfully added device to habanalabs driver\n");
1488 hdev->init_done = true;
1490 /* After initialization is done, we are ready to receive events from
1491 * the F/W. We can't do it before because we will ignore events and if
1492 * those events are fatal, we won't know about it and the device will
1493 * be operational although it shouldn't be
1495 hdev->asic_funcs->enable_events_from_fw(hdev);
1500 if (hl_ctx_put(hdev->kernel_ctx) != 1)
1502 "kernel ctx is still alive on initialization failure\n");
1503 remove_device_from_debugfs:
1504 hl_debugfs_remove_device(hdev);
1508 hl_eq_fini(hdev, &hdev->event_queue);
1510 for (i = 0 ; i < cq_ready_cnt ; i++)
1511 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1512 kfree(hdev->completion_queue);
1514 hl_hw_queues_destroy(hdev);
1516 hdev->asic_funcs->sw_fini(hdev);
1517 user_interrupts_fini:
1518 kfree(hdev->user_interrupt);
1520 device_early_fini(hdev);
1522 put_device(hdev->dev_ctrl);
1524 put_device(hdev->dev);
1526 hdev->disabled = true;
1527 if (add_cdev_sysfs_on_err)
1528 device_cdev_sysfs_add(hdev);
1530 dev_err(&hdev->pdev->dev,
1531 "Failed to initialize hl%d. Device is NOT usable !\n",
1534 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1541 * hl_device_fini - main tear-down function for habanalabs device
1543 * @hdev: pointer to habanalabs device structure
1545 * Destroy the device, call ASIC fini functions and release the id
1547 void hl_device_fini(struct hl_device *hdev)
1553 dev_info(hdev->dev, "Removing device\n");
1555 hdev->device_fini_pending = 1;
1556 flush_delayed_work(&hdev->device_reset_work.reset_work);
1559 reset_sec = HL_PLDM_HARD_RESET_MAX_TIMEOUT;
1561 reset_sec = HL_HARD_RESET_MAX_TIMEOUT;
1564 * This function is competing with the reset function, so try to
1565 * take the reset atomic and if we are already in middle of reset,
1566 * wait until reset function is finished. Reset function is designed
1567 * to always finish. However, in Gaudi, because of all the network
1568 * ports, the hard reset could take between 10-30 seconds
1571 timeout = ktime_add_us(ktime_get(), reset_sec * 1000 * 1000);
1572 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1574 usleep_range(50, 200);
1575 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1576 if (ktime_compare(ktime_get(), timeout) > 0) {
1578 "Failed to remove device because reset function did not finish\n");
1583 /* Disable PCI access from device F/W so it won't send us additional
1584 * interrupts. We disable MSI/MSI-X at the halt_engines function and we
1585 * can't have the F/W sending us interrupts after that. We need to
1586 * disable the access here because if the device is marked disable, the
1587 * message won't be send. Also, in case of heartbeat, the device CPU is
1588 * marked as disable so this message won't be sent
1590 hl_fw_send_pci_access_msg(hdev, CPUCP_PACKET_DISABLE_PCI_ACCESS);
1592 /* Mark device as disabled */
1593 hdev->disabled = true;
1595 take_release_locks(hdev);
1597 hdev->hard_reset_pending = true;
1599 hl_hwmon_fini(hdev);
1601 cleanup_resources(hdev, true, false);
1603 /* Kill processes here after CS rollback. This is because the process
1604 * can't really exit until all its CSs are done, which is what we
1608 "Waiting for all processes to exit (timeout of %u seconds)",
1609 HL_PENDING_RESET_LONG_SEC);
1611 rc = device_kill_open_processes(hdev, HL_PENDING_RESET_LONG_SEC);
1613 dev_crit(hdev->dev, "Failed to kill all open processes\n");
1614 device_disable_open_processes(hdev);
1617 hl_cb_pool_fini(hdev);
1619 /* Reset the H/W. It will be in idle state after this returns */
1620 hdev->asic_funcs->hw_fini(hdev, true, false);
1622 hdev->fw_loader.linux_loaded = false;
1624 /* Release kernel context */
1625 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1626 dev_err(hdev->dev, "kernel ctx is still alive\n");
1628 hl_debugfs_remove_device(hdev);
1634 hl_eq_fini(hdev, &hdev->event_queue);
1636 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1637 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1638 kfree(hdev->completion_queue);
1639 kfree(hdev->user_interrupt);
1641 hl_hw_queues_destroy(hdev);
1643 /* Call ASIC S/W finalize function */
1644 hdev->asic_funcs->sw_fini(hdev);
1646 device_early_fini(hdev);
1648 /* Hide devices and sysfs nodes from user */
1649 device_cdev_sysfs_del(hdev);
1651 pr_info("removed device successfully\n");
1655 * MMIO register access helper functions.
1659 * hl_rreg - Read an MMIO register
1661 * @hdev: pointer to habanalabs device structure
1662 * @reg: MMIO register offset (in bytes)
1664 * Returns the value of the MMIO register we are asked to read
1667 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1669 return readl(hdev->rmmio + reg);
1673 * hl_wreg - Write to an MMIO register
1675 * @hdev: pointer to habanalabs device structure
1676 * @reg: MMIO register offset (in bytes)
1677 * @val: 32-bit value
1679 * Writes the 32-bit value into the MMIO register
1682 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1684 writel(val, hdev->rmmio + reg);