drivers/misc/habanalabs/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #define pr_fmt(fmt)                     "habanalabs: " fmt
   9
  10 #include "habanalabs.h"
  11
  12 #include <linux/pci.h>
  13 #include <linux/sched/signal.h>
  14 #include <linux/hwmon.h>
  15 #include <uapi/misc/habanalabs.h>
  16
  17 #define HL_PLDM_PENDING_RESET_PER_SEC   (HL_PENDING_RESET_PER_SEC * 10)
  18
  19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
  20 {
  21         if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
  22                 return true;
  23         else
  24                 return false;
  25 }
  26
  27 enum hl_device_status hl_device_status(struct hl_device *hdev)
  28 {
  29         enum hl_device_status status;
  30
  31         if (hdev->disabled)
  32                 status = HL_DEVICE_STATUS_MALFUNCTION;
  33         else if (atomic_read(&hdev->in_reset))
  34                 status = HL_DEVICE_STATUS_IN_RESET;
  35         else
  36                 status = HL_DEVICE_STATUS_OPERATIONAL;
  37
  38         return status;
  39 }
  40
  41 static void hpriv_release(struct kref *ref)
  42 {
  43         struct hl_fpriv *hpriv;
  44         struct hl_device *hdev;
  45
  46         hpriv = container_of(ref, struct hl_fpriv, refcount);
  47
  48         hdev = hpriv->hdev;
  49
  50         put_pid(hpriv->taskpid);
  51
  52         hl_debugfs_remove_file(hpriv);
  53
  54         mutex_destroy(&hpriv->restore_phase_mutex);
  55
  56         mutex_lock(&hdev->fpriv_list_lock);
  57         list_del(&hpriv->dev_node);
  58         hdev->compute_ctx = NULL;
  59         mutex_unlock(&hdev->fpriv_list_lock);
  60
  61         kfree(hpriv);
  62 }
  63
  64 void hl_hpriv_get(struct hl_fpriv *hpriv)
  65 {
  66         kref_get(&hpriv->refcount);
  67 }
  68
  69 void hl_hpriv_put(struct hl_fpriv *hpriv)
  70 {
  71         kref_put(&hpriv->refcount, hpriv_release);
  72 }
  73
  74 /*
  75  * hl_device_release - release function for habanalabs device
  76  *
  77  * @inode: pointer to inode structure
  78  * @filp: pointer to file structure
  79  *
  80  * Called when process closes an habanalabs device
  81  */
  82 static int hl_device_release(struct inode *inode, struct file *filp)
  83 {
  84         struct hl_fpriv *hpriv = filp->private_data;
  85
  86         hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
  87         hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
  88
  89         filp->private_data = NULL;
  90
  91         hl_hpriv_put(hpriv);
  92
  93         return 0;
  94 }
  95
  96 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
  97 {
  98         struct hl_fpriv *hpriv = filp->private_data;
  99         struct hl_device *hdev;
 100
 101         filp->private_data = NULL;
 102
 103         hdev = hpriv->hdev;
 104
 105         mutex_lock(&hdev->fpriv_list_lock);
 106         list_del(&hpriv->dev_node);
 107         mutex_unlock(&hdev->fpriv_list_lock);
 108
 109         kfree(hpriv);
 110
 111         return 0;
 112 }
 113
 114 /*
 115  * hl_mmap - mmap function for habanalabs device
 116  *
 117  * @*filp: pointer to file structure
 118  * @*vma: pointer to vm_area_struct of the process
 119  *
 120  * Called when process does an mmap on habanalabs device. Call the device's mmap
 121  * function at the end of the common code.
 122  */
 123 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
 124 {
 125         struct hl_fpriv *hpriv = filp->private_data;
 126
 127         if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
 128                 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
 129                 return hl_cb_mmap(hpriv, vma);
 130         }
 131
 132         return -EINVAL;
 133 }
 134
 135 static const struct file_operations hl_ops = {
 136         .owner = THIS_MODULE,
 137         .open = hl_device_open,
 138         .release = hl_device_release,
 139         .mmap = hl_mmap,
 140         .unlocked_ioctl = hl_ioctl,
 141         .compat_ioctl = hl_ioctl
 142 };
 143
 144 static const struct file_operations hl_ctrl_ops = {
 145         .owner = THIS_MODULE,
 146         .open = hl_device_open_ctrl,
 147         .release = hl_device_release_ctrl,
 148         .unlocked_ioctl = hl_ioctl_control,
 149         .compat_ioctl = hl_ioctl_control
 150 };
 151
 152 static void device_release_func(struct device *dev)
 153 {
 154         kfree(dev);
 155 }
 156
 157 /*
 158  * device_init_cdev - Initialize cdev and device for habanalabs device
 159  *
 160  * @hdev: pointer to habanalabs device structure
 161  * @hclass: pointer to the class object of the device
 162  * @minor: minor number of the specific device
 163  * @fpos: file operations to install for this device
 164  * @name: name of the device as it will appear in the filesystem
 165  * @cdev: pointer to the char device object that will be initialized
 166  * @dev: pointer to the device object that will be initialized
 167  *
 168  * Initialize a cdev and a Linux device for habanalabs's device.
 169  */
 170 static int device_init_cdev(struct hl_device *hdev, struct class *hclass,
 171                                 int minor, const struct file_operations *fops,
 172                                 char *name, struct cdev *cdev,
 173                                 struct device **dev)
 174 {
 175         cdev_init(cdev, fops);
 176         cdev->owner = THIS_MODULE;
 177
 178         *dev = kzalloc(sizeof(**dev), GFP_KERNEL);
 179         if (!*dev)
 180                 return -ENOMEM;
 181
 182         device_initialize(*dev);
 183         (*dev)->devt = MKDEV(hdev->major, minor);
 184         (*dev)->class = hclass;
 185         (*dev)->release = device_release_func;
 186         dev_set_drvdata(*dev, hdev);
 187         dev_set_name(*dev, "%s", name);
 188
 189         return 0;
 190 }
 191
 192 static int device_cdev_sysfs_add(struct hl_device *hdev)
 193 {
 194         int rc;
 195
 196         rc = cdev_device_add(&hdev->cdev, hdev->dev);
 197         if (rc) {
 198                 dev_err(hdev->dev,
 199                         "failed to add a char device to the system\n");
 200                 return rc;
 201         }
 202
 203         rc = cdev_device_add(&hdev->cdev_ctrl, hdev->dev_ctrl);
 204         if (rc) {
 205                 dev_err(hdev->dev,
 206                         "failed to add a control char device to the system\n");
 207                 goto delete_cdev_device;
 208         }
 209
 210         /* hl_sysfs_init() must be done after adding the device to the system */
 211         rc = hl_sysfs_init(hdev);
 212         if (rc) {
 213                 dev_err(hdev->dev, "failed to initialize sysfs\n");
 214                 goto delete_ctrl_cdev_device;
 215         }
 216
 217         hdev->cdev_sysfs_created = true;
 218
 219         return 0;
 220
 221 delete_ctrl_cdev_device:
 222         cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
 223 delete_cdev_device:
 224         cdev_device_del(&hdev->cdev, hdev->dev);
 225         return rc;
 226 }
 227
 228 static void device_cdev_sysfs_del(struct hl_device *hdev)
 229 {
 230         /* device_release() won't be called so must free devices explicitly */
 231         if (!hdev->cdev_sysfs_created) {
 232                 kfree(hdev->dev_ctrl);
 233                 kfree(hdev->dev);
 234                 return;
 235         }
 236
 237         hl_sysfs_fini(hdev);
 238         cdev_device_del(&hdev->cdev_ctrl, hdev->dev_ctrl);
 239         cdev_device_del(&hdev->cdev, hdev->dev);
 240 }
 241
 242 /*
 243  * device_early_init - do some early initialization for the habanalabs device
 244  *
 245  * @hdev: pointer to habanalabs device structure
 246  *
 247  * Install the relevant function pointers and call the early_init function,
 248  * if such a function exists
 249  */
 250 static int device_early_init(struct hl_device *hdev)
 251 {
 252         int rc;
 253
 254         switch (hdev->asic_type) {
 255         case ASIC_GOYA:
 256                 goya_set_asic_funcs(hdev);
 257                 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 258                 break;
 259         case ASIC_GAUDI:
 260                 gaudi_set_asic_funcs(hdev);
 261                 sprintf(hdev->asic_name, "GAUDI");
 262                 break;
 263         default:
 264                 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 265                         hdev->asic_type);
 266                 return -EINVAL;
 267         }
 268
 269         rc = hdev->asic_funcs->early_init(hdev);
 270         if (rc)
 271                 return rc;
 272
 273         rc = hl_asid_init(hdev);
 274         if (rc)
 275                 goto early_fini;
 276
 277         hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
 278         if (hdev->cq_wq == NULL) {
 279                 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
 280                 rc = -ENOMEM;
 281                 goto asid_fini;
 282         }
 283
 284         hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
 285         if (hdev->eq_wq == NULL) {
 286                 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
 287                 rc = -ENOMEM;
 288                 goto free_cq_wq;
 289         }
 290
 291         hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
 292                                         GFP_KERNEL);
 293         if (!hdev->hl_chip_info) {
 294                 rc = -ENOMEM;
 295                 goto free_eq_wq;
 296         }
 297
 298         hdev->idle_busy_ts_arr = kmalloc_array(HL_IDLE_BUSY_TS_ARR_SIZE,
 299                                         sizeof(struct hl_device_idle_busy_ts),
 300                                         (GFP_KERNEL | __GFP_ZERO));
 301         if (!hdev->idle_busy_ts_arr) {
 302                 rc = -ENOMEM;
 303                 goto free_chip_info;
 304         }
 305
 306         hl_cb_mgr_init(&hdev->kernel_cb_mgr);
 307
 308         mutex_init(&hdev->send_cpu_message_lock);
 309         mutex_init(&hdev->debug_lock);
 310         mutex_init(&hdev->mmu_cache_lock);
 311         INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
 312         spin_lock_init(&hdev->hw_queues_mirror_lock);
 313         INIT_LIST_HEAD(&hdev->fpriv_list);
 314         mutex_init(&hdev->fpriv_list_lock);
 315         atomic_set(&hdev->in_reset, 0);
 316
 317         return 0;
 318
 319 free_chip_info:
 320         kfree(hdev->hl_chip_info);
 321 free_eq_wq:
 322         destroy_workqueue(hdev->eq_wq);
 323 free_cq_wq:
 324         destroy_workqueue(hdev->cq_wq);
 325 asid_fini:
 326         hl_asid_fini(hdev);
 327 early_fini:
 328         if (hdev->asic_funcs->early_fini)
 329                 hdev->asic_funcs->early_fini(hdev);
 330
 331         return rc;
 332 }
 333
 334 /*
 335  * device_early_fini - finalize all that was done in device_early_init
 336  *
 337  * @hdev: pointer to habanalabs device structure
 338  *
 339  */
 340 static void device_early_fini(struct hl_device *hdev)
 341 {
 342         mutex_destroy(&hdev->mmu_cache_lock);
 343         mutex_destroy(&hdev->debug_lock);
 344         mutex_destroy(&hdev->send_cpu_message_lock);
 345
 346         mutex_destroy(&hdev->fpriv_list_lock);
 347
 348         hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
 349
 350         kfree(hdev->idle_busy_ts_arr);
 351         kfree(hdev->hl_chip_info);
 352
 353         destroy_workqueue(hdev->eq_wq);
 354         destroy_workqueue(hdev->cq_wq);
 355
 356         hl_asid_fini(hdev);
 357
 358         if (hdev->asic_funcs->early_fini)
 359                 hdev->asic_funcs->early_fini(hdev);
 360 }
 361
 362 static void set_freq_to_low_job(struct work_struct *work)
 363 {
 364         struct hl_device *hdev = container_of(work, struct hl_device,
 365                                                 work_freq.work);
 366
 367         mutex_lock(&hdev->fpriv_list_lock);
 368
 369         if (!hdev->compute_ctx)
 370                 hl_device_set_frequency(hdev, PLL_LOW);
 371
 372         mutex_unlock(&hdev->fpriv_list_lock);
 373
 374         schedule_delayed_work(&hdev->work_freq,
 375                         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 376 }
 377
 378 static void hl_device_heartbeat(struct work_struct *work)
 379 {
 380         struct hl_device *hdev = container_of(work, struct hl_device,
 381                                                 work_heartbeat.work);
 382
 383         if (hl_device_disabled_or_in_reset(hdev))
 384                 goto reschedule;
 385
 386         if (!hdev->asic_funcs->send_heartbeat(hdev))
 387                 goto reschedule;
 388
 389         dev_err(hdev->dev, "Device heartbeat failed!\n");
 390         hl_device_reset(hdev, true, false);
 391
 392         return;
 393
 394 reschedule:
 395         schedule_delayed_work(&hdev->work_heartbeat,
 396                         usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 397 }
 398
 399 /*
 400  * device_late_init - do late stuff initialization for the habanalabs device
 401  *
 402  * @hdev: pointer to habanalabs device structure
 403  *
 404  * Do stuff that either needs the device H/W queues to be active or needs
 405  * to happen after all the rest of the initialization is finished
 406  */
 407 static int device_late_init(struct hl_device *hdev)
 408 {
 409         int rc;
 410
 411         if (hdev->asic_funcs->late_init) {
 412                 rc = hdev->asic_funcs->late_init(hdev);
 413                 if (rc) {
 414                         dev_err(hdev->dev,
 415                                 "failed late initialization for the H/W\n");
 416                         return rc;
 417                 }
 418         }
 419
 420         hdev->high_pll = hdev->asic_prop.high_pll;
 421
 422         /* force setting to low frequency */
 423         hdev->curr_pll_profile = PLL_LOW;
 424
 425         if (hdev->pm_mng_profile == PM_AUTO)
 426                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
 427         else
 428                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
 429
 430         INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
 431         schedule_delayed_work(&hdev->work_freq,
 432         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 433
 434         if (hdev->heartbeat) {
 435                 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
 436                 schedule_delayed_work(&hdev->work_heartbeat,
 437                                 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 438         }
 439
 440         hdev->late_init_done = true;
 441
 442         return 0;
 443 }
 444
 445 /*
 446  * device_late_fini - finalize all that was done in device_late_init
 447  *
 448  * @hdev: pointer to habanalabs device structure
 449  *
 450  */
 451 static void device_late_fini(struct hl_device *hdev)
 452 {
 453         if (!hdev->late_init_done)
 454                 return;
 455
 456         cancel_delayed_work_sync(&hdev->work_freq);
 457         if (hdev->heartbeat)
 458                 cancel_delayed_work_sync(&hdev->work_heartbeat);
 459
 460         if (hdev->asic_funcs->late_fini)
 461                 hdev->asic_funcs->late_fini(hdev);
 462
 463         hdev->late_init_done = false;
 464 }
 465
 466 uint32_t hl_device_utilization(struct hl_device *hdev, uint32_t period_ms)
 467 {
 468         struct hl_device_idle_busy_ts *ts;
 469         ktime_t zero_ktime, curr = ktime_get();
 470         u32 overlap_cnt = 0, last_index = hdev->idle_busy_ts_idx;
 471         s64 period_us, last_start_us, last_end_us, last_busy_time_us,
 472                 total_busy_time_us = 0, total_busy_time_ms;
 473
 474         zero_ktime = ktime_set(0, 0);
 475         period_us = period_ms * USEC_PER_MSEC;
 476         ts = &hdev->idle_busy_ts_arr[last_index];
 477
 478         /* check case that device is currently in idle */
 479         if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime) &&
 480                         !ktime_compare(ts->idle_to_busy_ts, zero_ktime)) {
 481
 482                 last_index--;
 483                 /* Handle case idle_busy_ts_idx was 0 */
 484                 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
 485                         last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
 486
 487                 ts = &hdev->idle_busy_ts_arr[last_index];
 488         }
 489
 490         while (overlap_cnt < HL_IDLE_BUSY_TS_ARR_SIZE) {
 491                 /* Check if we are in last sample case. i.e. if the sample
 492                  * begun before the sampling period. This could be a real
 493                  * sample or 0 so need to handle both cases
 494                  */
 495                 last_start_us = ktime_to_us(
 496                                 ktime_sub(curr, ts->idle_to_busy_ts));
 497
 498                 if (last_start_us > period_us) {
 499
 500                         /* First check two cases:
 501                          * 1. If the device is currently busy
 502                          * 2. If the device was idle during the whole sampling
 503                          *    period
 504                          */
 505
 506                         if (!ktime_compare(ts->busy_to_idle_ts, zero_ktime)) {
 507                                 /* Check if the device is currently busy */
 508                                 if (ktime_compare(ts->idle_to_busy_ts,
 509                                                 zero_ktime))
 510                                         return 100;
 511
 512                                 /* We either didn't have any activity or we
 513                                  * reached an entry which is 0. Either way,
 514                                  * exit and return what was accumulated so far
 515                                  */
 516                                 break;
 517                         }
 518
 519                         /* If sample has finished, check it is relevant */
 520                         last_end_us = ktime_to_us(
 521                                         ktime_sub(curr, ts->busy_to_idle_ts));
 522
 523                         if (last_end_us > period_us)
 524                                 break;
 525
 526                         /* It is relevant so add it but with adjustment */
 527                         last_busy_time_us = ktime_to_us(
 528                                                 ktime_sub(ts->busy_to_idle_ts,
 529                                                 ts->idle_to_busy_ts));
 530                         total_busy_time_us += last_busy_time_us -
 531                                         (last_start_us - period_us);
 532                         break;
 533                 }
 534
 535                 /* Check if the sample is finished or still open */
 536                 if (ktime_compare(ts->busy_to_idle_ts, zero_ktime))
 537                         last_busy_time_us = ktime_to_us(
 538                                                 ktime_sub(ts->busy_to_idle_ts,
 539                                                 ts->idle_to_busy_ts));
 540                 else
 541                         last_busy_time_us = ktime_to_us(
 542                                         ktime_sub(curr, ts->idle_to_busy_ts));
 543
 544                 total_busy_time_us += last_busy_time_us;
 545
 546                 last_index--;
 547                 /* Handle case idle_busy_ts_idx was 0 */
 548                 if (last_index > HL_IDLE_BUSY_TS_ARR_SIZE)
 549                         last_index = HL_IDLE_BUSY_TS_ARR_SIZE - 1;
 550
 551                 ts = &hdev->idle_busy_ts_arr[last_index];
 552
 553                 overlap_cnt++;
 554         }
 555
 556         total_busy_time_ms = DIV_ROUND_UP_ULL(total_busy_time_us,
 557                                                 USEC_PER_MSEC);
 558
 559         return DIV_ROUND_UP_ULL(total_busy_time_ms * 100, period_ms);
 560 }
 561
 562 /*
 563  * hl_device_set_frequency - set the frequency of the device
 564  *
 565  * @hdev: pointer to habanalabs device structure
 566  * @freq: the new frequency value
 567  *
 568  * Change the frequency if needed. This function has no protection against
 569  * concurrency, therefore it is assumed that the calling function has protected
 570  * itself against the case of calling this function from multiple threads with
 571  * different values
 572  *
 573  * Returns 0 if no change was done, otherwise returns 1
 574  */
 575 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
 576 {
 577         if ((hdev->pm_mng_profile == PM_MANUAL) ||
 578                         (hdev->curr_pll_profile == freq))
 579                 return 0;
 580
 581         dev_dbg(hdev->dev, "Changing device frequency to %s\n",
 582                 freq == PLL_HIGH ? "high" : "low");
 583
 584         hdev->asic_funcs->set_pll_profile(hdev, freq);
 585
 586         hdev->curr_pll_profile = freq;
 587
 588         return 1;
 589 }
 590
 591 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
 592 {
 593         int rc = 0;
 594
 595         mutex_lock(&hdev->debug_lock);
 596
 597         if (!enable) {
 598                 if (!hdev->in_debug) {
 599                         dev_err(hdev->dev,
 600                                 "Failed to disable debug mode because device was not in debug mode\n");
 601                         rc = -EFAULT;
 602                         goto out;
 603                 }
 604
 605                 if (!hdev->hard_reset_pending)
 606                         hdev->asic_funcs->halt_coresight(hdev);
 607
 608                 hdev->in_debug = 0;
 609
 610                 if (!hdev->hard_reset_pending)
 611                         hdev->asic_funcs->enable_clock_gating(hdev);
 612
 613                 goto out;
 614         }
 615
 616         if (hdev->in_debug) {
 617                 dev_err(hdev->dev,
 618                         "Failed to enable debug mode because device is already in debug mode\n");
 619                 rc = -EFAULT;
 620                 goto out;
 621         }
 622
 623         hdev->asic_funcs->disable_clock_gating(hdev);
 624         hdev->in_debug = 1;
 625
 626 out:
 627         mutex_unlock(&hdev->debug_lock);
 628
 629         return rc;
 630 }
 631
 632 /*
 633  * hl_device_suspend - initiate device suspend
 634  *
 635  * @hdev: pointer to habanalabs device structure
 636  *
 637  * Puts the hw in the suspend state (all asics).
 638  * Returns 0 for success or an error on failure.
 639  * Called at driver suspend.
 640  */
 641 int hl_device_suspend(struct hl_device *hdev)
 642 {
 643         int rc;
 644
 645         pci_save_state(hdev->pdev);
 646
 647         /* Block future CS/VM/JOB completion operations */
 648         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 649         if (rc) {
 650                 dev_err(hdev->dev, "Can't suspend while in reset\n");
 651                 return -EIO;
 652         }
 653
 654         /* This blocks all other stuff that is not blocked by in_reset */
 655         hdev->disabled = true;
 656
 657         /*
 658          * Flush anyone that is inside the critical section of enqueue
 659          * jobs to the H/W
 660          */
 661         hdev->asic_funcs->hw_queues_lock(hdev);
 662         hdev->asic_funcs->hw_queues_unlock(hdev);
 663
 664         /* Flush processes that are sending message to CPU */
 665         mutex_lock(&hdev->send_cpu_message_lock);
 666         mutex_unlock(&hdev->send_cpu_message_lock);
 667
 668         rc = hdev->asic_funcs->suspend(hdev);
 669         if (rc)
 670                 dev_err(hdev->dev,
 671                         "Failed to disable PCI access of device CPU\n");
 672
 673         /* Shut down the device */
 674         pci_disable_device(hdev->pdev);
 675         pci_set_power_state(hdev->pdev, PCI_D3hot);
 676
 677         return 0;
 678 }
 679
 680 /*
 681  * hl_device_resume - initiate device resume
 682  *
 683  * @hdev: pointer to habanalabs device structure
 684  *
 685  * Bring the hw back to operating state (all asics).
 686  * Returns 0 for success or an error on failure.
 687  * Called at driver resume.
 688  */
 689 int hl_device_resume(struct hl_device *hdev)
 690 {
 691         int rc;
 692
 693         pci_set_power_state(hdev->pdev, PCI_D0);
 694         pci_restore_state(hdev->pdev);
 695         rc = pci_enable_device_mem(hdev->pdev);
 696         if (rc) {
 697                 dev_err(hdev->dev,
 698                         "Failed to enable PCI device in resume\n");
 699                 return rc;
 700         }
 701
 702         pci_set_master(hdev->pdev);
 703
 704         rc = hdev->asic_funcs->resume(hdev);
 705         if (rc) {
 706                 dev_err(hdev->dev, "Failed to resume device after suspend\n");
 707                 goto disable_device;
 708         }
 709
 710
 711         hdev->disabled = false;
 712         atomic_set(&hdev->in_reset, 0);
 713
 714         rc = hl_device_reset(hdev, true, false);
 715         if (rc) {
 716                 dev_err(hdev->dev, "Failed to reset device during resume\n");
 717                 goto disable_device;
 718         }
 719
 720         return 0;
 721
 722 disable_device:
 723         pci_clear_master(hdev->pdev);
 724         pci_disable_device(hdev->pdev);
 725
 726         return rc;
 727 }
 728
 729 static int device_kill_open_processes(struct hl_device *hdev)
 730 {
 731         u16 pending_total, pending_cnt;
 732         struct hl_fpriv *hpriv;
 733         struct task_struct *task = NULL;
 734
 735         if (hdev->pldm)
 736                 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
 737         else
 738                 pending_total = HL_PENDING_RESET_PER_SEC;
 739
 740         /* Giving time for user to close FD, and for processes that are inside
 741          * hl_device_open to finish
 742          */
 743         if (!list_empty(&hdev->fpriv_list))
 744                 ssleep(1);
 745
 746         mutex_lock(&hdev->fpriv_list_lock);
 747
 748         /* This section must be protected because we are dereferencing
 749          * pointers that are freed if the process exits
 750          */
 751         list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
 752                 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
 753                 if (task) {
 754                         dev_info(hdev->dev, "Killing user process pid=%d\n",
 755                                 task_pid_nr(task));
 756                         send_sig(SIGKILL, task, 1);
 757                         usleep_range(1000, 10000);
 758
 759                         put_task_struct(task);
 760                 }
 761         }
 762
 763         mutex_unlock(&hdev->fpriv_list_lock);
 764
 765         /* We killed the open users, but because the driver cleans up after the
 766          * user contexts are closed (e.g. mmu mappings), we need to wait again
 767          * to make sure the cleaning phase is finished before continuing with
 768          * the reset
 769          */
 770
 771         pending_cnt = pending_total;
 772
 773         while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
 774                 dev_info(hdev->dev,
 775                         "Waiting for all unmap operations to finish before hard reset\n");
 776
 777                 pending_cnt--;
 778
 779                 ssleep(1);
 780         }
 781
 782         return list_empty(&hdev->fpriv_list) ? 0 : -EBUSY;
 783 }
 784
 785 static void device_hard_reset_pending(struct work_struct *work)
 786 {
 787         struct hl_device_reset_work *device_reset_work =
 788                 container_of(work, struct hl_device_reset_work, reset_work);
 789         struct hl_device *hdev = device_reset_work->hdev;
 790
 791         hl_device_reset(hdev, true, true);
 792
 793         kfree(device_reset_work);
 794 }
 795
 796 /*
 797  * hl_device_reset - reset the device
 798  *
 799  * @hdev: pointer to habanalabs device structure
 800  * @hard_reset: should we do hard reset to all engines or just reset the
 801  *              compute/dma engines
 802  * @from_hard_reset_thread: is the caller the hard-reset thread
 803  *
 804  * Block future CS and wait for pending CS to be enqueued
 805  * Call ASIC H/W fini
 806  * Flush all completions
 807  * Re-initialize all internal data structures
 808  * Call ASIC H/W init, late_init
 809  * Test queues
 810  * Enable device
 811  *
 812  * Returns 0 for success or an error on failure.
 813  */
 814 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
 815                         bool from_hard_reset_thread)
 816 {
 817         int i, rc;
 818
 819         if (!hdev->init_done) {
 820                 dev_err(hdev->dev,
 821                         "Can't reset before initialization is done\n");
 822                 return 0;
 823         }
 824
 825         if ((!hard_reset) && (!hdev->supports_soft_reset)) {
 826                 dev_dbg(hdev->dev, "Doing hard-reset instead of soft-reset\n");
 827                 hard_reset = true;
 828         }
 829
 830         /*
 831          * Prevent concurrency in this function - only one reset should be
 832          * done at any given time. Only need to perform this if we didn't
 833          * get from the dedicated hard reset thread
 834          */
 835         if (!from_hard_reset_thread) {
 836                 /* Block future CS/VM/JOB completion operations */
 837                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 838                 if (rc)
 839                         return 0;
 840
 841                 /* This also blocks future CS/VM/JOB completion operations */
 842                 hdev->disabled = true;
 843
 844                 /* Flush anyone that is inside the critical section of enqueue
 845                  * jobs to the H/W
 846                  */
 847                 hdev->asic_funcs->hw_queues_lock(hdev);
 848                 hdev->asic_funcs->hw_queues_unlock(hdev);
 849
 850                 /* Flush anyone that is inside device open */
 851                 mutex_lock(&hdev->fpriv_list_lock);
 852                 mutex_unlock(&hdev->fpriv_list_lock);
 853
 854                 dev_err(hdev->dev, "Going to RESET device!\n");
 855         }
 856
 857 again:
 858         if ((hard_reset) && (!from_hard_reset_thread)) {
 859                 struct hl_device_reset_work *device_reset_work;
 860
 861                 hdev->hard_reset_pending = true;
 862
 863                 device_reset_work = kzalloc(sizeof(*device_reset_work),
 864                                                 GFP_ATOMIC);
 865                 if (!device_reset_work) {
 866                         rc = -ENOMEM;
 867                         goto out_err;
 868                 }
 869
 870                 /*
 871                  * Because the reset function can't run from interrupt or
 872                  * from heartbeat work, we need to call the reset function
 873                  * from a dedicated work
 874                  */
 875                 INIT_WORK(&device_reset_work->reset_work,
 876                                 device_hard_reset_pending);
 877                 device_reset_work->hdev = hdev;
 878                 schedule_work(&device_reset_work->reset_work);
 879
 880                 return 0;
 881         }
 882
 883         if (hard_reset) {
 884                 device_late_fini(hdev);
 885
 886                 /*
 887                  * Now that the heartbeat thread is closed, flush processes
 888                  * which are sending messages to CPU
 889                  */
 890                 mutex_lock(&hdev->send_cpu_message_lock);
 891                 mutex_unlock(&hdev->send_cpu_message_lock);
 892         }
 893
 894         /*
 895          * Halt the engines and disable interrupts so we won't get any more
 896          * completions from H/W and we won't have any accesses from the
 897          * H/W to the host machine
 898          */
 899         hdev->asic_funcs->halt_engines(hdev, hard_reset);
 900
 901         /* Go over all the queues, release all CS and their jobs */
 902         hl_cs_rollback_all(hdev);
 903
 904         if (hard_reset) {
 905                 /* Kill processes here after CS rollback. This is because the
 906                  * process can't really exit until all its CSs are done, which
 907                  * is what we do in cs rollback
 908                  */
 909                 rc = device_kill_open_processes(hdev);
 910                 if (rc) {
 911                         dev_crit(hdev->dev,
 912                                 "Failed to kill all open processes, stopping hard reset\n");
 913                         goto out_err;
 914                 }
 915
 916                 /* Flush the Event queue workers to make sure no other thread is
 917                  * reading or writing to registers during the reset
 918                  */
 919                 flush_workqueue(hdev->eq_wq);
 920         }
 921
 922         /* Release kernel context */
 923         if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
 924                 hdev->kernel_ctx = NULL;
 925
 926         /* Reset the H/W. It will be in idle state after this returns */
 927         hdev->asic_funcs->hw_fini(hdev, hard_reset);
 928
 929         if (hard_reset) {
 930                 hl_vm_fini(hdev);
 931                 hl_mmu_fini(hdev);
 932                 hl_eq_reset(hdev, &hdev->event_queue);
 933         }
 934
 935         /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
 936         hl_hw_queue_reset(hdev, hard_reset);
 937         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 938                 hl_cq_reset(hdev, &hdev->completion_queue[i]);
 939
 940         hdev->idle_busy_ts_idx = 0;
 941         hdev->idle_busy_ts_arr[0].busy_to_idle_ts = ktime_set(0, 0);
 942         hdev->idle_busy_ts_arr[0].idle_to_busy_ts = ktime_set(0, 0);
 943
 944         if (hdev->cs_active_cnt)
 945                 dev_crit(hdev->dev, "CS active cnt %d is not 0 during reset\n",
 946                         hdev->cs_active_cnt);
 947
 948         mutex_lock(&hdev->fpriv_list_lock);
 949
 950         /* Make sure the context switch phase will run again */
 951         if (hdev->compute_ctx) {
 952                 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
 953                 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
 954         }
 955
 956         mutex_unlock(&hdev->fpriv_list_lock);
 957
 958         /* Finished tear-down, starting to re-initialize */
 959
 960         if (hard_reset) {
 961                 hdev->device_cpu_disabled = false;
 962                 hdev->hard_reset_pending = false;
 963
 964                 if (hdev->kernel_ctx) {
 965                         dev_crit(hdev->dev,
 966                                 "kernel ctx was alive during hard reset, something is terribly wrong\n");
 967                         rc = -EBUSY;
 968                         goto out_err;
 969                 }
 970
 971                 rc = hl_mmu_init(hdev);
 972                 if (rc) {
 973                         dev_err(hdev->dev,
 974                                 "Failed to initialize MMU S/W after hard reset\n");
 975                         goto out_err;
 976                 }
 977
 978                 /* Allocate the kernel context */
 979                 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
 980                                                 GFP_KERNEL);
 981                 if (!hdev->kernel_ctx) {
 982                         rc = -ENOMEM;
 983                         goto out_err;
 984                 }
 985
 986                 hdev->compute_ctx = NULL;
 987
 988                 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
 989                 if (rc) {
 990                         dev_err(hdev->dev,
 991                                 "failed to init kernel ctx in hard reset\n");
 992                         kfree(hdev->kernel_ctx);
 993                         hdev->kernel_ctx = NULL;
 994                         goto out_err;
 995                 }
 996         }
 997
 998         rc = hdev->asic_funcs->hw_init(hdev);
 999         if (rc) {
1000                 dev_err(hdev->dev,
1001                         "failed to initialize the H/W after reset\n");
1002                 goto out_err;
1003         }
1004
1005         hdev->disabled = false;
1006
1007         /* Check that the communication with the device is working */
1008         rc = hdev->asic_funcs->test_queues(hdev);
1009         if (rc) {
1010                 dev_err(hdev->dev,
1011                         "Failed to detect if device is alive after reset\n");
1012                 goto out_err;
1013         }
1014
1015         if (hard_reset) {
1016                 rc = device_late_init(hdev);
1017                 if (rc) {
1018                         dev_err(hdev->dev,
1019                                 "Failed late init after hard reset\n");
1020                         goto out_err;
1021                 }
1022
1023                 rc = hl_vm_init(hdev);
1024                 if (rc) {
1025                         dev_err(hdev->dev,
1026                                 "Failed to init memory module after hard reset\n");
1027                         goto out_err;
1028                 }
1029
1030                 hl_set_max_power(hdev, hdev->max_power);
1031         } else {
1032                 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
1033                 if (rc) {
1034                         dev_err(hdev->dev,
1035                                 "Failed late init after soft reset\n");
1036                         goto out_err;
1037                 }
1038         }
1039
1040         atomic_set(&hdev->in_reset, 0);
1041
1042         if (hard_reset)
1043                 hdev->hard_reset_cnt++;
1044         else
1045                 hdev->soft_reset_cnt++;
1046
1047         dev_warn(hdev->dev, "Successfully finished resetting the device\n");
1048
1049         return 0;
1050
1051 out_err:
1052         hdev->disabled = true;
1053
1054         if (hard_reset) {
1055                 dev_err(hdev->dev,
1056                         "Failed to reset! Device is NOT usable\n");
1057                 hdev->hard_reset_cnt++;
1058         } else {
1059                 dev_err(hdev->dev,
1060                         "Failed to do soft-reset, trying hard reset\n");
1061                 hdev->soft_reset_cnt++;
1062                 hard_reset = true;
1063                 goto again;
1064         }
1065
1066         atomic_set(&hdev->in_reset, 0);
1067
1068         return rc;
1069 }
1070
1071 /*
1072  * hl_device_init - main initialization function for habanalabs device
1073  *
1074  * @hdev: pointer to habanalabs device structure
1075  *
1076  * Allocate an id for the device, do early initialization and then call the
1077  * ASIC specific initialization functions. Finally, create the cdev and the
1078  * Linux device to expose it to the user
1079  */
1080 int hl_device_init(struct hl_device *hdev, struct class *hclass)
1081 {
1082         int i, rc, cq_cnt, cq_ready_cnt;
1083         char *name;
1084         bool add_cdev_sysfs_on_err = false;
1085
1086         name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
1087         if (!name) {
1088                 rc = -ENOMEM;
1089                 goto out_disabled;
1090         }
1091
1092         /* Initialize cdev and device structures */
1093         rc = device_init_cdev(hdev, hclass, hdev->id, &hl_ops, name,
1094                                 &hdev->cdev, &hdev->dev);
1095
1096         kfree(name);
1097
1098         if (rc)
1099                 goto out_disabled;
1100
1101         name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
1102         if (!name) {
1103                 rc = -ENOMEM;
1104                 goto free_dev;
1105         }
1106
1107         /* Initialize cdev and device structures for control device */
1108         rc = device_init_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
1109                                 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
1110
1111         kfree(name);
1112
1113         if (rc)
1114                 goto free_dev;
1115
1116         /* Initialize ASIC function pointers and perform early init */
1117         rc = device_early_init(hdev);
1118         if (rc)
1119                 goto free_dev_ctrl;
1120
1121         /*
1122          * Start calling ASIC initialization. First S/W then H/W and finally
1123          * late init
1124          */
1125         rc = hdev->asic_funcs->sw_init(hdev);
1126         if (rc)
1127                 goto early_fini;
1128
1129         /*
1130          * Initialize the H/W queues. Must be done before hw_init, because
1131          * there the addresses of the kernel queue are being written to the
1132          * registers of the device
1133          */
1134         rc = hl_hw_queues_create(hdev);
1135         if (rc) {
1136                 dev_err(hdev->dev, "failed to initialize kernel queues\n");
1137                 goto sw_fini;
1138         }
1139
1140         cq_cnt = hdev->asic_prop.completion_queues_count;
1141
1142         /*
1143          * Initialize the completion queues. Must be done before hw_init,
1144          * because there the addresses of the completion queues are being
1145          * passed as arguments to request_irq
1146          */
1147         hdev->completion_queue = kcalloc(cq_cnt,
1148                                                 sizeof(*hdev->completion_queue),
1149                                                 GFP_KERNEL);
1150
1151         if (!hdev->completion_queue) {
1152                 dev_err(hdev->dev, "failed to allocate completion queues\n");
1153                 rc = -ENOMEM;
1154                 goto hw_queues_destroy;
1155         }
1156
1157         for (i = 0, cq_ready_cnt = 0 ; i < cq_cnt ; i++, cq_ready_cnt++) {
1158                 rc = hl_cq_init(hdev, &hdev->completion_queue[i],
1159                                 hdev->asic_funcs->get_queue_id_for_cq(hdev, i));
1160                 if (rc) {
1161                         dev_err(hdev->dev,
1162                                 "failed to initialize completion queue\n");
1163                         goto cq_fini;
1164                 }
1165         }
1166
1167         /*
1168          * Initialize the event queue. Must be done before hw_init,
1169          * because there the address of the event queue is being
1170          * passed as argument to request_irq
1171          */
1172         rc = hl_eq_init(hdev, &hdev->event_queue);
1173         if (rc) {
1174                 dev_err(hdev->dev, "failed to initialize event queue\n");
1175                 goto cq_fini;
1176         }
1177
1178         /* MMU S/W must be initialized before kernel context is created */
1179         rc = hl_mmu_init(hdev);
1180         if (rc) {
1181                 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
1182                 goto eq_fini;
1183         }
1184
1185         /* Allocate the kernel context */
1186         hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1187         if (!hdev->kernel_ctx) {
1188                 rc = -ENOMEM;
1189                 goto mmu_fini;
1190         }
1191
1192         hdev->compute_ctx = NULL;
1193
1194         rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1195         if (rc) {
1196                 dev_err(hdev->dev, "failed to initialize kernel context\n");
1197                 kfree(hdev->kernel_ctx);
1198                 goto mmu_fini;
1199         }
1200
1201         rc = hl_cb_pool_init(hdev);
1202         if (rc) {
1203                 dev_err(hdev->dev, "failed to initialize CB pool\n");
1204                 goto release_ctx;
1205         }
1206
1207         hl_debugfs_add_device(hdev);
1208
1209         if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
1210                 dev_info(hdev->dev,
1211                         "H/W state is dirty, must reset before initializing\n");
1212                 hdev->asic_funcs->halt_engines(hdev, true);
1213                 hdev->asic_funcs->hw_fini(hdev, true);
1214         }
1215
1216         /*
1217          * From this point, in case of an error, add char devices and create
1218          * sysfs nodes as part of the error flow, to allow debugging.
1219          */
1220         add_cdev_sysfs_on_err = true;
1221
1222         rc = hdev->asic_funcs->hw_init(hdev);
1223         if (rc) {
1224                 dev_err(hdev->dev, "failed to initialize the H/W\n");
1225                 rc = 0;
1226                 goto out_disabled;
1227         }
1228
1229         hdev->disabled = false;
1230
1231         /* Check that the communication with the device is working */
1232         rc = hdev->asic_funcs->test_queues(hdev);
1233         if (rc) {
1234                 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1235                 rc = 0;
1236                 goto out_disabled;
1237         }
1238
1239         rc = device_late_init(hdev);
1240         if (rc) {
1241                 dev_err(hdev->dev, "Failed late initialization\n");
1242                 rc = 0;
1243                 goto out_disabled;
1244         }
1245
1246         dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1247                 hdev->asic_name,
1248                 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1249
1250         rc = hl_vm_init(hdev);
1251         if (rc) {
1252                 dev_err(hdev->dev, "Failed to initialize memory module\n");
1253                 rc = 0;
1254                 goto out_disabled;
1255         }
1256
1257         /*
1258          * Expose devices and sysfs nodes to user.
1259          * From here there is no need to add char devices and create sysfs nodes
1260          * in case of an error.
1261          */
1262         add_cdev_sysfs_on_err = false;
1263         rc = device_cdev_sysfs_add(hdev);
1264         if (rc) {
1265                 dev_err(hdev->dev,
1266                         "Failed to add char devices and sysfs nodes\n");
1267                 rc = 0;
1268                 goto out_disabled;
1269         }
1270
1271         /*
1272          * hl_hwmon_init() must be called after device_late_init(), because only
1273          * there we get the information from the device about which
1274          * hwmon-related sensors the device supports.
1275          * Furthermore, it must be done after adding the device to the system.
1276          */
1277         rc = hl_hwmon_init(hdev);
1278         if (rc) {
1279                 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1280                 rc = 0;
1281                 goto out_disabled;
1282         }
1283
1284         dev_notice(hdev->dev,
1285                 "Successfully added device to habanalabs driver\n");
1286
1287         hdev->init_done = true;
1288
1289         return 0;
1290
1291 release_ctx:
1292         if (hl_ctx_put(hdev->kernel_ctx) != 1)
1293                 dev_err(hdev->dev,
1294                         "kernel ctx is still alive on initialization failure\n");
1295 mmu_fini:
1296         hl_mmu_fini(hdev);
1297 eq_fini:
1298         hl_eq_fini(hdev, &hdev->event_queue);
1299 cq_fini:
1300         for (i = 0 ; i < cq_ready_cnt ; i++)
1301                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1302         kfree(hdev->completion_queue);
1303 hw_queues_destroy:
1304         hl_hw_queues_destroy(hdev);
1305 sw_fini:
1306         hdev->asic_funcs->sw_fini(hdev);
1307 early_fini:
1308         device_early_fini(hdev);
1309 free_dev_ctrl:
1310         kfree(hdev->dev_ctrl);
1311 free_dev:
1312         kfree(hdev->dev);
1313 out_disabled:
1314         hdev->disabled = true;
1315         if (add_cdev_sysfs_on_err)
1316                 device_cdev_sysfs_add(hdev);
1317         if (hdev->pdev)
1318                 dev_err(&hdev->pdev->dev,
1319                         "Failed to initialize hl%d. Device is NOT usable !\n",
1320                         hdev->id / 2);
1321         else
1322                 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1323                         hdev->id / 2);
1324
1325         return rc;
1326 }
1327
1328 /*
1329  * hl_device_fini - main tear-down function for habanalabs device
1330  *
1331  * @hdev: pointer to habanalabs device structure
1332  *
1333  * Destroy the device, call ASIC fini functions and release the id
1334  */
1335 void hl_device_fini(struct hl_device *hdev)
1336 {
1337         int i, rc;
1338         ktime_t timeout;
1339
1340         dev_info(hdev->dev, "Removing device\n");
1341
1342         /*
1343          * This function is competing with the reset function, so try to
1344          * take the reset atomic and if we are already in middle of reset,
1345          * wait until reset function is finished. Reset function is designed
1346          * to always finish. However, in Gaudi, because of all the network
1347          * ports, the hard reset could take between 10-30 seconds
1348          */
1349
1350         timeout = ktime_add_us(ktime_get(),
1351                                 HL_HARD_RESET_MAX_TIMEOUT * 1000 * 1000);
1352         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1353         while (rc) {
1354                 usleep_range(50, 200);
1355                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1356                 if (ktime_compare(ktime_get(), timeout) > 0) {
1357                         WARN(1, "Failed to remove device because reset function did not finish\n");
1358                         return;
1359                 }
1360         }
1361
1362         /* Mark device as disabled */
1363         hdev->disabled = true;
1364
1365         /* Flush anyone that is inside the critical section of enqueue
1366          * jobs to the H/W
1367          */
1368         hdev->asic_funcs->hw_queues_lock(hdev);
1369         hdev->asic_funcs->hw_queues_unlock(hdev);
1370
1371         /* Flush anyone that is inside device open */
1372         mutex_lock(&hdev->fpriv_list_lock);
1373         mutex_unlock(&hdev->fpriv_list_lock);
1374
1375         hdev->hard_reset_pending = true;
1376
1377         hl_hwmon_fini(hdev);
1378
1379         device_late_fini(hdev);
1380
1381         hl_debugfs_remove_device(hdev);
1382
1383         /*
1384          * Halt the engines and disable interrupts so we won't get any more
1385          * completions from H/W and we won't have any accesses from the
1386          * H/W to the host machine
1387          */
1388         hdev->asic_funcs->halt_engines(hdev, true);
1389
1390         /* Go over all the queues, release all CS and their jobs */
1391         hl_cs_rollback_all(hdev);
1392
1393         /* Kill processes here after CS rollback. This is because the process
1394          * can't really exit until all its CSs are done, which is what we
1395          * do in cs rollback
1396          */
1397         rc = device_kill_open_processes(hdev);
1398         if (rc)
1399                 dev_crit(hdev->dev, "Failed to kill all open processes\n");
1400
1401         hl_cb_pool_fini(hdev);
1402
1403         /* Release kernel context */
1404         if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1405                 dev_err(hdev->dev, "kernel ctx is still alive\n");
1406
1407         /* Reset the H/W. It will be in idle state after this returns */
1408         hdev->asic_funcs->hw_fini(hdev, true);
1409
1410         hl_vm_fini(hdev);
1411
1412         hl_mmu_fini(hdev);
1413
1414         hl_eq_fini(hdev, &hdev->event_queue);
1415
1416         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1417                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1418         kfree(hdev->completion_queue);
1419
1420         hl_hw_queues_destroy(hdev);
1421
1422         /* Call ASIC S/W finalize function */
1423         hdev->asic_funcs->sw_fini(hdev);
1424
1425         device_early_fini(hdev);
1426
1427         /* Hide devices and sysfs nodes from user */
1428         device_cdev_sysfs_del(hdev);
1429
1430         pr_info("removed device successfully\n");
1431 }
1432
1433 /*
1434  * MMIO register access helper functions.
1435  */
1436
1437 /*
1438  * hl_rreg - Read an MMIO register
1439  *
1440  * @hdev: pointer to habanalabs device structure
1441  * @reg: MMIO register offset (in bytes)
1442  *
1443  * Returns the value of the MMIO register we are asked to read
1444  *
1445  */
1446 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1447 {
1448         return readl(hdev->rmmio + reg);
1449 }
1450
1451 /*
1452  * hl_wreg - Write to an MMIO register
1453  *
1454  * @hdev: pointer to habanalabs device structure
1455  * @reg: MMIO register offset (in bytes)
1456  * @val: 32-bit value
1457  *
1458  * Writes the 32-bit value into the MMIO register
1459  *
1460  */
1461 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1462 {
1463         writel(val, hdev->rmmio + reg);
1464 }