1 // SPDX-License-Identifier: GPL-2.0
4 * Copyright 2016-2019 HabanaLabs, Ltd.
8 #define pr_fmt(fmt) "habanalabs: " fmt
10 #include "habanalabs.h"
12 #include <linux/pci.h>
13 #include <linux/sched/signal.h>
14 #include <linux/hwmon.h>
15 #include <uapi/misc/habanalabs.h>
17 #define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10)
19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
21 if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
27 enum hl_device_status hl_device_status(struct hl_device *hdev)
29 enum hl_device_status status;
32 status = HL_DEVICE_STATUS_MALFUNCTION;
33 else if (atomic_read(&hdev->in_reset))
34 status = HL_DEVICE_STATUS_IN_RESET;
36 status = HL_DEVICE_STATUS_OPERATIONAL;
41 static void hpriv_release(struct kref *ref)
43 struct hl_fpriv *hpriv;
44 struct hl_device *hdev;
46 hpriv = container_of(ref, struct hl_fpriv, refcount);
50 put_pid(hpriv->taskpid);
52 hl_debugfs_remove_file(hpriv);
54 mutex_destroy(&hpriv->restore_phase_mutex);
56 mutex_lock(&hdev->fpriv_list_lock);
57 list_del(&hpriv->dev_node);
58 hdev->compute_ctx = NULL;
59 mutex_unlock(&hdev->fpriv_list_lock);
64 void hl_hpriv_get(struct hl_fpriv *hpriv)
66 kref_get(&hpriv->refcount);
69 void hl_hpriv_put(struct hl_fpriv *hpriv)
71 kref_put(&hpriv->refcount, hpriv_release);
75 * hl_device_release - release function for habanalabs device
77 * @inode: pointer to inode structure
78 * @filp: pointer to file structure
80 * Called when process closes an habanalabs device
82 static int hl_device_release(struct inode *inode, struct file *filp)
84 struct hl_fpriv *hpriv = filp->private_data;
86 hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
87 hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
89 filp->private_data = NULL;
96 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
98 struct hl_fpriv *hpriv = filp->private_data;
99 struct hl_device *hdev;
101 filp->private_data = NULL;
105 mutex_lock(&hdev->fpriv_list_lock);
106 list_del(&hpriv->dev_node);
107 mutex_unlock(&hdev->fpriv_list_lock);
115 * hl_mmap - mmap function for habanalabs device
117 * @*filp: pointer to file structure
118 * @*vma: pointer to vm_area_struct of the process
120 * Called when process does an mmap on habanalabs device. Call the device's mmap
121 * function at the end of the common code.
123 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
125 struct hl_fpriv *hpriv = filp->private_data;
126 unsigned long vm_pgoff;
128 vm_pgoff = vma->vm_pgoff;
129 vma->vm_pgoff = HL_MMAP_OFFSET_VALUE_GET(vm_pgoff);
131 switch (vm_pgoff & HL_MMAP_TYPE_MASK) {
132 case HL_MMAP_TYPE_CB:
133 return hl_cb_mmap(hpriv, vma);
139 static const struct file_operations hl_ops = {
140 .owner = THIS_MODULE,
141 .open = hl_device_open,
142 .release = hl_device_release,
144 .unlocked_ioctl = hl_ioctl,
145 .compat_ioctl = hl_ioctl
148 static const struct file_operations hl_ctrl_ops = {
149 .owner = THIS_MODULE,
150 .open = hl_device_open_ctrl,
151 .release = hl_device_release_ctrl,
152 .unlocked_ioctl = hl_ioctl_control,
153 .compat_ioctl = hl_ioctl_control
156 static void device_release_func(struct device *dev)
162 * device_init_cdev - Initialize cdev and device for habanalabs device
164 * @hdev: pointer to habanalabs device structure
165 * @hclass: pointer to the class object of the device
166 * @minor: minor number of the specific device
167 * @fpos: file operations to install for this device
168 * @name: name of the device as it will appear in the filesystem
169 * @cdev: pointer to the char device object that will be initialized
170 * @dev: pointer to the device object that will be initialized
172 * Initialize a cdev and a Linux device for habanalabs's device.
174 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
175 int minor, const struct file_operations *fops,
176 char *name, struct cdev *cdev,
179 cdev_init(cdev, fops);
180 cdev->owner = THIS_MODULE;
182 *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
186 device_initialize(*dev);
187 (*dev)->devt = MKDEV(hdev->major, minor);
188 (*dev)->class = hclass;
189 (*dev)->release = device_release_func;
190 dev_set_drvdata(*dev, hdev);
191 dev_set_name(*dev, "%s", name);
196 static int device_cdev_sysfs_add(struct hl_device *hdev)
200 rc = cdev_device_add(&hdev->cdev, hdev->dev);
203 "failed to add a char device to the system\n");
207 rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
210 "failed to add a control char device to the system\n");
211 goto delete_cdev_device;
214 /* hl_sysfs_init() must be done after adding the device to the system */
215 rc = hl_sysfs_init(hdev);
217 dev_err(hdev->dev, "failed to initialize sysfs\n");
218 goto delete_ctrl_cdev_device;
221 hdev->cdev_sysfs_created = true;
225 delete_ctrl_cdev_device:
226 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
228 cdev_device_del(&hdev->cdev, hdev->dev);
232 static void device_cdev_sysfs_del(struct hl_device *hdev)
234 /* device_release() won't be called so must free devices explicitly */
235 if (!hdev->cdev_sysfs_created) {
236 kfree(hdev->dev_ctrl);
242 cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
243 cdev_device_del(&hdev->cdev, hdev->dev);
247 * device_early_init - do some early initialization for the habanalabs device
249 * @hdev: pointer to habanalabs device structure
251 * Install the relevant function pointers and call the early_init function,
252 * if such a function exists
254 static int device_early_init(struct hl_device *hdev)
259 switch (hdev->asic_type) {
261 goya_set_asic_funcs(hdev);
262 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
265 gaudi_set_asic_funcs(hdev);
266 sprintf(hdev->asic_name, "GAUDI");
269 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
274 rc = hdev->asic_funcs->early_init(hdev);
278 rc = hl_asid_init(hdev);
282 if (hdev->asic_prop.completion_queues_count) {
283 hdev->cq_wq = kcalloc(hdev->asic_prop.completion_queues_count,
284 sizeof(*hdev->cq_wq),
292 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++) {
293 snprintf(workq_name, 32, "hl-free-jobs-%u", (u32) i);
294 hdev->cq_wq[i] = create_singlethread_workqueue(workq_name);
295 if (hdev->cq_wq[i] == NULL) {
296 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
302 hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
303 if (hdev->eq_wq == NULL) {
304 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
309 hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
311 if (!hdev->hl_chip_info) {
316 hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,
317 sizeof(struct hl_device_idle_busy_ts),
318 (GFP_KERNEL | __GFP_ZERO));
319 if (!hdev->idle_busy_ts_arr) {
324 rc = hl_mmu_if_set_funcs(hdev);
326 goto free_idle_busy_ts_arr;
328 hl_cb_mgr_init(&hdev->kernel_cb_mgr);
330 mutex_init(&hdev->send_cpu_message_lock);
331 mutex_init(&hdev->debug_lock);
332 mutex_init(&hdev->mmu_cache_lock);
333 INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
334 spin_lock_init(&hdev->hw_queues_mirror_lock);
335 INIT_LIST_HEAD(&hdev->fpriv_list);
336 mutex_init(&hdev->fpriv_list_lock);
337 atomic_set(&hdev->in_reset, 0);
341 free_idle_busy_ts_arr:
342 kfree(hdev->idle_busy_ts_arr);
344 kfree(hdev->hl_chip_info);
346 destroy_workqueue(hdev->eq_wq);
348 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
350 destroy_workqueue(hdev->cq_wq[i]);
355 if (hdev->asic_funcs->early_fini)
356 hdev->asic_funcs->early_fini(hdev);
362 * device_early_fini - finalize all that was done in device_early_init
364 * @hdev: pointer to habanalabs device structure
367 static void device_early_fini(struct hl_device *hdev)
371 mutex_destroy(&hdev->mmu_cache_lock);
372 mutex_destroy(&hdev->debug_lock);
373 mutex_destroy(&hdev->send_cpu_message_lock);
375 mutex_destroy(&hdev->fpriv_list_lock);
377 hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
379 kfree(hdev->idle_busy_ts_arr);
380 kfree(hdev->hl_chip_info);
382 destroy_workqueue(hdev->eq_wq);
384 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
385 destroy_workqueue(hdev->cq_wq[i]);
390 if (hdev->asic_funcs->early_fini)
391 hdev->asic_funcs->early_fini(hdev);
394 static void set_freq_to_low_job(struct work_struct *work)
396 struct hl_device *hdev = container_of(work, struct hl_device,
399 mutex_lock(&hdev->fpriv_list_lock);
401 if (!hdev->compute_ctx)
402 hl_device_set_frequency(hdev, PLL_LOW);
404 mutex_unlock(&hdev->fpriv_list_lock);
406 schedule_delayed_work(&hdev->work_freq,
407 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
410 static void hl_device_heartbeat(struct work_struct *work)
412 struct hl_device *hdev = container_of(work, struct hl_device,
413 work_heartbeat.work);
415 if (hl_device_disabled_or_in_reset(hdev))
418 if (!hdev->asic_funcs->send_heartbeat(hdev))
421 dev_err(hdev->dev, "Device heartbeat failed!\n");
422 hl_device_reset(hdev, true, false);
427 schedule_delayed_work(&hdev->work_heartbeat,
428 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
432 * device_late_init - do late stuff initialization for the habanalabs device
434 * @hdev: pointer to habanalabs device structure
436 * Do stuff that either needs the device H/W queues to be active or needs
437 * to happen after all the rest of the initialization is finished
439 static int device_late_init(struct hl_device *hdev)
443 if (hdev->asic_funcs->late_init) {
444 rc = hdev->asic_funcs->late_init(hdev);
447 "failed late initialization for the H/W\n");
452 hdev->high_pll = hdev->asic_prop.high_pll;
454 /* force setting to low frequency */
455 hdev->curr_pll_profile = PLL_LOW;
457 if (hdev->pm_mng_profile == PM_AUTO)
458 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
460 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
462 INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
463 schedule_delayed_work(&hdev->work_freq,
464 usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
466 if (hdev->heartbeat) {
467 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
468 schedule_delayed_work(&hdev->work_heartbeat,
469 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
472 hdev->late_init_done = true;
478 * device_late_fini - finalize all that was done in device_late_init
480 * @hdev: pointer to habanalabs device structure
483 static void device_late_fini(struct hl_device *hdev)
485 if (!hdev->late_init_done)
488 cancel_delayed_work_sync(&hdev->work_freq);
490 cancel_delayed_work_sync(&hdev->work_heartbeat);
492 if (hdev->asic_funcs->late_fini)
493 hdev->asic_funcs->late_fini(hdev);
495 hdev->late_init_done = false;
498 uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)
500 struct hl_device_idle_busy_ts *ts;
501 ktime_t zero_ktime, curr = ktime_get();
502 u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;
503 s64 period_us, last_start_us, last_end_us, last_busy_time_us,
504 total_busy_time_us = 0, total_busy_time_ms;
506 zero_ktime = ktime_set(0, 0);
507 period_us = period_ms * USEC_PER_MSEC;
508 ts = &hdev->idle_busy_ts_arr[last_index];
510 /* check case that device is currently in idle */
511 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) &&
512 !ktime_compare(ts->idle_to_busy_ts, zero_ktime)) {
515 /* Handle case idle_busy_ts_idx was 0 */
516 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
517 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
519 ts = &hdev->idle_busy_ts_arr[last_index];
522 while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) {
523 /* Check if we are in last sample case. i.e. if the sample
524 * begun before the sampling period. This could be a real
525 * sample or 0 so need to handle both cases
527 last_start_us = ktime_to_us(
528 ktime_sub(curr, ts->idle_to_busy_ts));
530 if (last_start_us > period_us) {
532 /* First check two cases:
533 * 1. If the device is currently busy
534 * 2. If the device was idle during the whole sampling
538 if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) {
539 /* Check if the device is currently busy */
540 if (ktime_compare(ts->idle_to_busy_ts,
544 /* We either didn't have any activity or we
545 * reached an entry which is 0. Either way,
546 * exit and return what was accumulated so far
551 /* If sample has finished, check it is relevant */
552 last_end_us = ktime_to_us(
553 ktime_sub(curr, ts->busy_to_idle_ts));
555 if (last_end_us > period_us)
558 /* It is relevant so add it but with adjustment */
559 last_busy_time_us = ktime_to_us(
560 ktime_sub(ts->busy_to_idle_ts,
561 ts->idle_to_busy_ts));
562 total_busy_time_us += last_busy_time_us -
563 (last_start_us - period_us);
567 /* Check if the sample is finished or still open */
568 if (ktime_compare(ts->busy_to_idle_ts, zero_ktime))
569 last_busy_time_us = ktime_to_us(
570 ktime_sub(ts->busy_to_idle_ts,
571 ts->idle_to_busy_ts));
573 last_busy_time_us = ktime_to_us(
574 ktime_sub(curr, ts->idle_to_busy_ts));
576 total_busy_time_us += last_busy_time_us;
579 /* Handle case idle_busy_ts_idx was 0 */
580 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
581 last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
583 ts = &hdev->idle_busy_ts_arr[last_index];
588 total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us,
591 return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms);
595 * hl_device_set_frequency - set the frequency of the device
597 * @hdev: pointer to habanalabs device structure
598 * @freq: the new frequency value
600 * Change the frequency if needed. This function has no protection against
601 * concurrency, therefore it is assumed that the calling function has protected
602 * itself against the case of calling this function from multiple threads with
605 * Returns 0 if no change was done, otherwise returns 1
607 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
609 if ((hdev->pm_mng_profile == PM_MANUAL) ||
610 (hdev->curr_pll_profile == freq))
613 dev_dbg(hdev->dev, "Changing device frequency to %s\n",
614 freq == PLL_HIGH ? "high" : "low");
616 hdev->asic_funcs->set_pll_profile(hdev, freq);
618 hdev->curr_pll_profile = freq;
623 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
627 mutex_lock(&hdev->debug_lock);
630 if (!hdev->in_debug) {
632 "Failed to disable debug mode because device was not in debug mode\n");
637 if (!hdev->hard_reset_pending)
638 hdev->asic_funcs->halt_coresight(hdev);
642 if (!hdev->hard_reset_pending)
643 hdev->asic_funcs->set_clock_gating(hdev);
648 if (hdev->in_debug) {
650 "Failed to enable debug mode because device is already in debug mode\n");
655 hdev->asic_funcs->disable_clock_gating(hdev);
659 mutex_unlock(&hdev->debug_lock);
665 * hl_device_suspend - initiate device suspend
667 * @hdev: pointer to habanalabs device structure
669 * Puts the hw in the suspend state (all asics).
670 * Returns 0 for success or an error on failure.
671 * Called at driver suspend.
673 int hl_device_suspend(struct hl_device *hdev)
677 pci_save_state(hdev->pdev);
679 /* Block future CS/VM/JOB completion operations */
680 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
682 dev_err(hdev->dev, "Can't suspend while in reset\n");
686 /* This blocks all other stuff that is not blocked by in_reset */
687 hdev->disabled = true;
690 * Flush anyone that is inside the critical section of enqueue
693 hdev->asic_funcs->hw_queues_lock(hdev);
694 hdev->asic_funcs->hw_queues_unlock(hdev);
696 /* Flush processes that are sending message to CPU */
697 mutex_lock(&hdev->send_cpu_message_lock);
698 mutex_unlock(&hdev->send_cpu_message_lock);
700 rc = hdev->asic_funcs->suspend(hdev);
703 "Failed to disable PCI access of device CPU\n");
705 /* Shut down the device */
706 pci_disable_device(hdev->pdev);
707 pci_set_power_state(hdev->pdev, PCI_D3hot);
713 * hl_device_resume - initiate device resume
715 * @hdev: pointer to habanalabs device structure
717 * Bring the hw back to operating state (all asics).
718 * Returns 0 for success or an error on failure.
719 * Called at driver resume.
721 int hl_device_resume(struct hl_device *hdev)
725 pci_set_power_state(hdev->pdev, PCI_D0);
726 pci_restore_state(hdev->pdev);
727 rc = pci_enable_device_mem(hdev->pdev);
730 "Failed to enable PCI device in resume\n");
734 pci_set_master(hdev->pdev);
736 rc = hdev->asic_funcs->resume(hdev);
738 dev_err(hdev->dev, "Failed to resume device after suspend\n");
743 hdev->disabled = false;
744 atomic_set(&hdev->in_reset, 0);
746 rc = hl_device_reset(hdev, true, false);
748 dev_err(hdev->dev, "Failed to reset device during resume\n");
755 pci_clear_master(hdev->pdev);
756 pci_disable_device(hdev->pdev);
761 static int device_kill_open_processes(struct hl_device *hdev)
763 u16 pending_total, pending_cnt;
764 struct hl_fpriv *hpriv;
765 struct task_struct *task = NULL;
768 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
770 pending_total = HL_PENDING_RESET_PER_SEC;
772 /* Giving time for user to close FD, and for processes that are inside
773 * hl_device_open to finish
775 if (!list_empty(&hdev->fpriv_list))
778 mutex_lock(&hdev->fpriv_list_lock);
780 /* This section must be protected because we are dereferencing
781 * pointers that are freed if the process exits
783 list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
784 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
786 dev_info(hdev->dev, "Killing user process pid=%d\n",
788 send_sig(SIGKILL, task, 1);
789 usleep_range(1000, 10000);
791 put_task_struct(task);
795 mutex_unlock(&hdev->fpriv_list_lock);
797 /* We killed the open users, but because the driver cleans up after the
798 * user contexts are closed (e.g. mmu mappings), we need to wait again
799 * to make sure the cleaning phase is finished before continuing with
803 pending_cnt = pending_total;
805 while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
807 "Waiting for all unmap operations to finish before hard reset\n");
814 return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
817 static void device_hard_reset_pending(struct work_struct *work)
819 struct hl_device_reset_work *device_reset_work =
820 container_of(work, struct hl_device_reset_work, reset_work);
821 struct hl_device *hdev = device_reset_work->hdev;
823 hl_device_reset(hdev, true, true);
825 kfree(device_reset_work);
829 * hl_device_reset - reset the device
831 * @hdev: pointer to habanalabs device structure
832 * @hard_reset: should we do hard reset to all engines or just reset the
833 * compute/dma engines
834 * @from_hard_reset_thread: is the caller the hard-reset thread
836 * Block future CS and wait for pending CS to be enqueued
838 * Flush all completions
839 * Re-initialize all internal data structures
840 * Call ASIC H/W init, late_init
844 * Returns 0 for success or an error on failure.
846 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
847 bool from_hard_reset_thread)
851 if (!hdev->init_done) {
853 "Can't reset before initialization is done\n");
857 if ((!hard_reset) && (!hdev->supports_soft_reset)) {
858 dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
863 * Prevent concurrency in this function - only one reset should be
864 * done at any given time. Only need to perform this if we didn't
865 * get from the dedicated hard reset thread
867 if (!from_hard_reset_thread) {
868 /* Block future CS/VM/JOB completion operations */
869 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
874 /* Disable PCI access from device F/W so he won't send
875 * us additional interrupts. We disable MSI/MSI-X at
876 * the halt_engines function and we can't have the F/W
877 * sending us interrupts after that. We need to disable
878 * the access here because if the device is marked
879 * disable, the message won't be send. Also, in case
880 * of heartbeat, the device CPU is marked as disable
881 * so this message won't be sent
883 if (hl_fw_send_pci_access_msg(hdev,
884 CPUCP_PACKET_DISABLE_PCI_ACCESS))
886 "Failed to disable PCI access by F/W\n");
889 /* This also blocks future CS/VM/JOB completion operations */
890 hdev->disabled = true;
892 /* Flush anyone that is inside the critical section of enqueue
895 hdev->asic_funcs->hw_queues_lock(hdev);
896 hdev->asic_funcs->hw_queues_unlock(hdev);
898 /* Flush anyone that is inside device open */
899 mutex_lock(&hdev->fpriv_list_lock);
900 mutex_unlock(&hdev->fpriv_list_lock);
902 dev_err(hdev->dev, "Going to RESET device!\n");
906 if ((hard_reset) && (!from_hard_reset_thread)) {
907 struct hl_device_reset_work *device_reset_work;
909 hdev->hard_reset_pending = true;
911 device_reset_work = kzalloc(sizeof(*device_reset_work),
913 if (!device_reset_work) {
919 * Because the reset function can't run from interrupt or
920 * from heartbeat work, we need to call the reset function
921 * from a dedicated work
923 INIT_WORK(&device_reset_work->reset_work,
924 device_hard_reset_pending);
925 device_reset_work->hdev = hdev;
926 schedule_work(&device_reset_work->reset_work);
932 device_late_fini(hdev);
935 * Now that the heartbeat thread is closed, flush processes
936 * which are sending messages to CPU
938 mutex_lock(&hdev->send_cpu_message_lock);
939 mutex_unlock(&hdev->send_cpu_message_lock);
943 * Halt the engines and disable interrupts so we won't get any more
944 * completions from H/W and we won't have any accesses from the
945 * H/W to the host machine
947 hdev->asic_funcs->halt_engines(hdev, hard_reset);
949 /* Go over all the queues, release all CS and their jobs */
950 hl_cs_rollback_all(hdev);
953 /* Kill processes here after CS rollback. This is because the
954 * process can't really exit until all its CSs are done, which
955 * is what we do in cs rollback
957 rc = device_kill_open_processes(hdev);
960 "Failed to kill all open processes, stopping hard reset\n");
964 /* Flush the Event queue workers to make sure no other thread is
965 * reading or writing to registers during the reset
967 flush_workqueue(hdev->eq_wq);
970 /* Reset the H/W. It will be in idle state after this returns */
971 hdev->asic_funcs->hw_fini(hdev, hard_reset);
974 /* Release kernel context */
975 if (hl_ctx_put(hdev->kernel_ctx) == 1)
976 hdev->kernel_ctx = NULL;
979 hl_eq_reset(hdev, &hdev->event_queue);
982 /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
983 hl_hw_queue_reset(hdev, hard_reset);
984 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
985 hl_cq_reset(hdev, &hdev->completion_queue[i]);
987 hdev->idle_busy_ts_idx = 0;
988 hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);
989 hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);
991 if (hdev->cs_active_cnt)
992 dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",
993 hdev->cs_active_cnt);
995 mutex_lock(&hdev->fpriv_list_lock);
997 /* Make sure the context switch phase will run again */
998 if (hdev->compute_ctx) {
999 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
1000 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
1003 mutex_unlock(&hdev->fpriv_list_lock);
1005 /* Finished tear-down, starting to re-initialize */
1008 hdev->device_cpu_disabled = false;
1009 hdev->hard_reset_pending = false;
1011 if (hdev->kernel_ctx) {
1013 "kernel ctx was alive during hard reset, something is terribly wrong\n");
1018 rc = hl_mmu_init(hdev);
1021 "Failed to initialize MMU S/W after hard reset\n");
1025 /* Allocate the kernel context */
1026 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
1028 if (!hdev->kernel_ctx) {
1033 hdev->compute_ctx = NULL;
1035 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1038 "failed to init kernel ctx in hard reset\n");
1039 kfree(hdev->kernel_ctx);
1040 hdev->kernel_ctx = NULL;
1045 /* Device is now enabled as part of the initialization requires
1046 * communication with the device firmware to get information that
1047 * is required for the initialization itself
1049 hdev->disabled = false;
1051 rc = hdev->asic_funcs->hw_init(hdev);
1054 "failed to initialize the H/W after reset\n");
1058 /* Check that the communication with the device is working */
1059 rc = hdev->asic_funcs->test_queues(hdev);
1062 "Failed to detect if device is alive after reset\n");
1067 rc = device_late_init(hdev);
1070 "Failed late init after hard reset\n");
1074 rc = hl_vm_init(hdev);
1077 "Failed to init memory module after hard reset\n");
1081 hl_set_max_power(hdev);
1083 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
1086 "Failed late init after soft reset\n");
1091 atomic_set(&hdev->in_reset, 0);
1094 hdev->hard_reset_cnt++;
1096 hdev->soft_reset_cnt++;
1098 dev_warn(hdev->dev, "Successfully finished resetting the device\n");
1103 hdev->disabled = true;
1107 "Failed to reset! Device is NOT usable\n");
1108 hdev->hard_reset_cnt++;
1111 "Failed to do soft-reset, trying hard reset\n");
1112 hdev->soft_reset_cnt++;
1117 atomic_set(&hdev->in_reset, 0);
1123 * hl_device_init - main initialization function for habanalabs device
1125 * @hdev: pointer to habanalabs device structure
1127 * Allocate an id for the device, do early initialization and then call the
1128 * ASIC specific initialization functions. Finally, create the cdev and the
1129 * Linux device to expose it to the user
1131 int hl_device_init(struct hl_device *hdev, struct class *hclass)
1133 int i, rc, cq_cnt, cq_ready_cnt;
1135 bool add_cdev_sysfs_on_err = false;
1137 name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
1143 /* Initialize cdev and device structures */
1144 rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
1145 &hdev->cdev, &hdev->dev);
1152 name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
1158 /* Initialize cdev and device structures for control device */
1159 rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
1160 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
1167 /* Initialize ASIC function pointers and perform early init */
1168 rc = device_early_init(hdev);
1173 * Start calling ASIC initialization. First S/W then H/W and finally
1176 rc = hdev->asic_funcs->sw_init(hdev);
1181 * Initialize the H/W queues. Must be done before hw_init, because
1182 * there the addresses of the kernel queue are being written to the
1183 * registers of the device
1185 rc = hl_hw_queues_create(hdev);
1187 dev_err(hdev->dev, "failed to initialize kernel queues\n");
1191 cq_cnt = hdev->asic_prop.completion_queues_count;
1194 * Initialize the completion queues. Must be done before hw_init,
1195 * because there the addresses of the completion queues are being
1196 * passed as arguments to request_irq
1199 hdev->completion_queue = kcalloc(cq_cnt,
1200 sizeof(*hdev->completion_queue),
1203 if (!hdev->completion_queue) {
1205 "failed to allocate completion queues\n");
1207 goto hw_queues_destroy;
1211 for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
1212 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
1213 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
1216 "failed to initialize completion queue\n");
1219 hdev->completion_queue[i].cq_idx = i;
1223 * Initialize the event queue. Must be done before hw_init,
1224 * because there the address of the event queue is being
1225 * passed as argument to request_irq
1227 rc = hl_eq_init(hdev, &hdev->event_queue);
1229 dev_err(hdev->dev, "failed to initialize event queue\n");
1233 /* MMU S/W must be initialized before kernel context is created */
1234 rc = hl_mmu_init(hdev);
1236 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
1240 /* Allocate the kernel context */
1241 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1242 if (!hdev->kernel_ctx) {
1247 hdev->compute_ctx = NULL;
1249 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1251 dev_err(hdev->dev, "failed to initialize kernel context\n");
1252 kfree(hdev->kernel_ctx);
1256 rc = hl_cb_pool_init(hdev);
1258 dev_err(hdev->dev, "failed to initialize CB pool\n");
1262 hl_debugfs_add_device(hdev);
1264 if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
1266 "H/W state is dirty, must reset before initializing\n");
1267 hdev->asic_funcs->halt_engines(hdev, true);
1268 hdev->asic_funcs->hw_fini(hdev, true);
1272 * From this point, in case of an error, add char devices and create
1273 * sysfs nodes as part of the error flow, to allow debugging.
1275 add_cdev_sysfs_on_err = true;
1277 /* Device is now enabled as part of the initialization requires
1278 * communication with the device firmware to get information that
1279 * is required for the initialization itself
1281 hdev->disabled = false;
1283 rc = hdev->asic_funcs->hw_init(hdev);
1285 dev_err(hdev->dev, "failed to initialize the H/W\n");
1290 /* Check that the communication with the device is working */
1291 rc = hdev->asic_funcs->test_queues(hdev);
1293 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1298 rc = device_late_init(hdev);
1300 dev_err(hdev->dev, "Failed late initialization\n");
1305 dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1307 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1309 rc = hl_vm_init(hdev);
1311 dev_err(hdev->dev, "Failed to initialize memory module\n");
1317 * Expose devices and sysfs nodes to user.
1318 * From here there is no need to add char devices and create sysfs nodes
1319 * in case of an error.
1321 add_cdev_sysfs_on_err = false;
1322 rc = device_cdev_sysfs_add(hdev);
1325 "Failed to add char devices and sysfs nodes\n");
1330 /* Need to call this again because the max power might change,
1331 * depending on card type for certain ASICs
1333 hl_set_max_power(hdev);
1336 * hl_hwmon_init() must be called after device_late_init(), because only
1337 * there we get the information from the device about which
1338 * hwmon-related sensors the device supports.
1339 * Furthermore, it must be done after adding the device to the system.
1341 rc = hl_hwmon_init(hdev);
1343 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1348 dev_notice(hdev->dev,
1349 "Successfully added device to habanalabs driver\n");
1351 hdev->init_done = true;
1356 if (hl_ctx_put(hdev->kernel_ctx) != 1)
1358 "kernel ctx is still alive on initialization failure\n");
1362 hl_eq_fini(hdev, &hdev->event_queue);
1364 for (i = 0 ; i < cq_ready_cnt ; i++)
1365 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1366 kfree(hdev->completion_queue);
1368 hl_hw_queues_destroy(hdev);
1370 hdev->asic_funcs->sw_fini(hdev);
1372 device_early_fini(hdev);
1374 kfree(hdev->dev_ctrl);
1378 hdev->disabled = true;
1379 if (add_cdev_sysfs_on_err)
1380 device_cdev_sysfs_add(hdev);
1382 dev_err(&hdev->pdev->dev,
1383 "Failed to initialize hl%d. Device is NOT usable !\n",
1386 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1393 * hl_device_fini - main tear-down function for habanalabs device
1395 * @hdev: pointer to habanalabs device structure
1397 * Destroy the device, call ASIC fini functions and release the id
1399 void hl_device_fini(struct hl_device *hdev)
1404 dev_info(hdev->dev, "Removing device\n");
1407 * This function is competing with the reset function, so try to
1408 * take the reset atomic and if we are already in middle of reset,
1409 * wait until reset function is finished. Reset function is designed
1410 * to always finish. However, in Gaudi, because of all the network
1411 * ports, the hard reset could take between 10-30 seconds
1414 timeout = ktime_add_us(ktime_get(),
1415 HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
1416 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1418 usleep_range(50, 200);
1419 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1420 if (ktime_compare(ktime_get(), timeout) > 0) {
1421 WARN(1, "Failed to remove device because reset function did not finish\n");
1426 /* Mark device as disabled */
1427 hdev->disabled = true;
1429 /* Flush anyone that is inside the critical section of enqueue
1432 hdev->asic_funcs->hw_queues_lock(hdev);
1433 hdev->asic_funcs->hw_queues_unlock(hdev);
1435 /* Flush anyone that is inside device open */
1436 mutex_lock(&hdev->fpriv_list_lock);
1437 mutex_unlock(&hdev->fpriv_list_lock);
1439 hdev->hard_reset_pending = true;
1441 hl_hwmon_fini(hdev);
1443 device_late_fini(hdev);
1445 hl_debugfs_remove_device(hdev);
1448 * Halt the engines and disable interrupts so we won't get any more
1449 * completions from H/W and we won't have any accesses from the
1450 * H/W to the host machine
1452 hdev->asic_funcs->halt_engines(hdev, true);
1454 /* Go over all the queues, release all CS and their jobs */
1455 hl_cs_rollback_all(hdev);
1457 /* Kill processes here after CS rollback. This is because the process
1458 * can't really exit until all its CSs are done, which is what we
1461 rc = device_kill_open_processes(hdev);
1463 dev_crit(hdev->dev, "Failed to kill all open processes\n");
1465 hl_cb_pool_fini(hdev);
1467 /* Reset the H/W. It will be in idle state after this returns */
1468 hdev->asic_funcs->hw_fini(hdev, true);
1470 /* Release kernel context */
1471 if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1472 dev_err(hdev->dev, "kernel ctx is still alive\n");
1478 hl_eq_fini(hdev, &hdev->event_queue);
1480 for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1481 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1482 kfree(hdev->completion_queue);
1484 hl_hw_queues_destroy(hdev);
1486 /* Call ASIC S/W finalize function */
1487 hdev->asic_funcs->sw_fini(hdev);
1489 device_early_fini(hdev);
1491 /* Hide devices and sysfs nodes from user */
1492 device_cdev_sysfs_del(hdev);
1494 pr_info("removed device successfully\n");
1498 * MMIO register access helper functions.
1502 * hl_rreg - Read an MMIO register
1504 * @hdev: pointer to habanalabs device structure
1505 * @reg: MMIO register offset (in bytes)
1507 * Returns the value of the MMIO register we are asked to read
1510 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1512 return readl(hdev->rmmio + reg);
1516 * hl_wreg - Write to an MMIO register
1518 * @hdev: pointer to habanalabs device structure
1519 * @reg: MMIO register offset (in bytes)
1520 * @val: 32-bit value
1522 * Writes the 32-bit value into the MMIO register
1525 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1527 writel(val, hdev->rmmio + reg);