drivers/misc/habanalabs/device.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 /*
   4  * Copyright 2016-2019 HabanaLabs, Ltd.
   5  * All Rights Reserved.
   6  */
   7
   8 #define pr_fmt(fmt)                     "habanalabs: " fmt
   9
  10 #include "habanalabs.h"
  11
  12 #include <linux/pci.h>
  13 #include <linux/sched/signal.h>
  14 #include <linux/hwmon.h>
  15 #include <uapi/misc/habanalabs.h>
  16
  17 #define HL_PLDM_PENDING_RESET_PER_SEC   (HL_PENDING_RESET_PER_SEC * 10)
  18
  19 bool hl_device_disabled_or_in_reset(struct hl_device *hdev)
  20 {
  21         if ((hdev->disabled) || (atomic_read(&hdev->in_reset)))
  22                 return true;
  23         else
  24                 return false;
  25 }
  26
  27 enum hl_device_status hl_device_status(struct hl_device *hdev)
  28 {
  29         enum hl_device_status status;
  30
  31         if (hdev->disabled)
  32                 status = HL_DEVICE_STATUS_MALFUNCTION;
  33         else if (atomic_read(&hdev->in_reset))
  34                 status = HL_DEVICE_STATUS_IN_RESET;
  35         else
  36                 status = HL_DEVICE_STATUS_OPERATIONAL;
  37
  38         return status;
  39 };
  40
  41 static void hpriv_release(struct kref *ref)
  42 {
  43         struct hl_fpriv *hpriv;
  44         struct hl_device *hdev;
  45         struct hl_ctx *ctx;
  46
  47         hpriv = container_of(ref, struct hl_fpriv, refcount);
  48
  49         hdev = hpriv->hdev;
  50         ctx = hpriv->ctx;
  51
  52         put_pid(hpriv->taskpid);
  53
  54         hl_debugfs_remove_file(hpriv);
  55
  56         mutex_destroy(&hpriv->restore_phase_mutex);
  57
  58         mutex_lock(&hdev->fpriv_list_lock);
  59         list_del(&hpriv->dev_node);
  60         hdev->compute_ctx = NULL;
  61         mutex_unlock(&hdev->fpriv_list_lock);
  62
  63         kfree(hpriv);
  64 }
  65
  66 void hl_hpriv_get(struct hl_fpriv *hpriv)
  67 {
  68         kref_get(&hpriv->refcount);
  69 }
  70
  71 void hl_hpriv_put(struct hl_fpriv *hpriv)
  72 {
  73         kref_put(&hpriv->refcount, hpriv_release);
  74 }
  75
  76 /*
  77  * hl_device_release - release function for habanalabs device
  78  *
  79  * @inode: pointer to inode structure
  80  * @filp: pointer to file structure
  81  *
  82  * Called when process closes an habanalabs device
  83  */
  84 static int hl_device_release(struct inode *inode, struct file *filp)
  85 {
  86         struct hl_fpriv *hpriv = filp->private_data;
  87
  88         hl_cb_mgr_fini(hpriv->hdev, &hpriv->cb_mgr);
  89         hl_ctx_mgr_fini(hpriv->hdev, &hpriv->ctx_mgr);
  90
  91         filp->private_data = NULL;
  92
  93         hl_hpriv_put(hpriv);
  94
  95         return 0;
  96 }
  97
  98 static int hl_device_release_ctrl(struct inode *inode, struct file *filp)
  99 {
 100         struct hl_fpriv *hpriv = filp->private_data;
 101         struct hl_device *hdev;
 102
 103         filp->private_data = NULL;
 104
 105         hdev = hpriv->hdev;
 106
 107         mutex_lock(&hdev->fpriv_list_lock);
 108         list_del(&hpriv->dev_node);
 109         mutex_unlock(&hdev->fpriv_list_lock);
 110
 111         kfree(hpriv);
 112
 113         return 0;
 114 }
 115
 116 /*
 117  * hl_mmap - mmap function for habanalabs device
 118  *
 119  * @*filp: pointer to file structure
 120  * @*vma: pointer to vm_area_struct of the process
 121  *
 122  * Called when process does an mmap on habanalabs device. Call the device's mmap
 123  * function at the end of the common code.
 124  */
 125 static int hl_mmap(struct file *filp, struct vm_area_struct *vma)
 126 {
 127         struct hl_fpriv *hpriv = filp->private_data;
 128
 129         if ((vma->vm_pgoff & HL_MMAP_CB_MASK) == HL_MMAP_CB_MASK) {
 130                 vma->vm_pgoff ^= HL_MMAP_CB_MASK;
 131                 return hl_cb_mmap(hpriv, vma);
 132         }
 133
 134         return -EINVAL;
 135 }
 136
 137 static const struct file_operations hl_ops = {
 138         .owner = THIS_MODULE,
 139         .open = hl_device_open,
 140         .release = hl_device_release,
 141         .mmap = hl_mmap,
 142         .unlocked_ioctl = hl_ioctl,
 143         .compat_ioctl = hl_ioctl
 144 };
 145
 146 static const struct file_operations hl_ctrl_ops = {
 147         .owner = THIS_MODULE,
 148         .open = hl_device_open_ctrl,
 149         .release = hl_device_release_ctrl,
 150         .unlocked_ioctl = hl_ioctl_control,
 151         .compat_ioctl = hl_ioctl_control
 152 };
 153
 154 /*
 155  * device_setup_cdev - setup cdev and device for habanalabs device
 156  *
 157  * @hdev: pointer to habanalabs device structure
 158  * @hclass: pointer to the class object of the device
 159  * @minor: minor number of the specific device
 160  * @fpos: file operations to install for this device
 161  * @name: name of the device as it will appear in the filesystem
 162  * @cdev: pointer to the char device object that will be created
 163  * @dev: pointer to the device object that will be created
 164  *
 165  * Create a cdev and a Linux device for habanalabs's device. Need to be
 166  * called at the end of the habanalabs device initialization process,
 167  * because this function exposes the device to the user
 168  */
 169 static int device_setup_cdev(struct hl_device *hdev, struct class *hclass,
 170                                 int minor, const struct file_operations *fops,
 171                                 char *name, struct cdev *cdev,
 172                                 struct device **dev)
 173 {
 174         int err, devno = MKDEV(hdev->major, minor);
 175
 176         cdev_init(cdev, fops);
 177         cdev->owner = THIS_MODULE;
 178         err = cdev_add(cdev, devno, 1);
 179         if (err) {
 180                 pr_err("Failed to add char device %s\n", name);
 181                 return err;
 182         }
 183
 184         *dev = device_create(hclass, NULL, devno, NULL, "%s", name);
 185         if (IS_ERR(*dev)) {
 186                 pr_err("Failed to create device %s\n", name);
 187                 err = PTR_ERR(*dev);
 188                 goto err_device_create;
 189         }
 190
 191         dev_set_drvdata(*dev, hdev);
 192
 193         return 0;
 194
 195 err_device_create:
 196         cdev_del(cdev);
 197         return err;
 198 }
 199
 200 /*
 201  * device_early_init - do some early initialization for the habanalabs device
 202  *
 203  * @hdev: pointer to habanalabs device structure
 204  *
 205  * Install the relevant function pointers and call the early_init function,
 206  * if such a function exists
 207  */
 208 static int device_early_init(struct hl_device *hdev)
 209 {
 210         int rc;
 211
 212         switch (hdev->asic_type) {
 213         case ASIC_GOYA:
 214                 goya_set_asic_funcs(hdev);
 215                 strlcpy(hdev->asic_name, "GOYA", sizeof(hdev->asic_name));
 216                 break;
 217         default:
 218                 dev_err(hdev->dev, "Unrecognized ASIC type %d\n",
 219                         hdev->asic_type);
 220                 return -EINVAL;
 221         }
 222
 223         rc = hdev->asic_funcs->early_init(hdev);
 224         if (rc)
 225                 return rc;
 226
 227         rc = hl_asid_init(hdev);
 228         if (rc)
 229                 goto early_fini;
 230
 231         hdev->cq_wq = alloc_workqueue("hl-free-jobs", WQ_UNBOUND, 0);
 232         if (hdev->cq_wq == NULL) {
 233                 dev_err(hdev->dev, "Failed to allocate CQ workqueue\n");
 234                 rc = -ENOMEM;
 235                 goto asid_fini;
 236         }
 237
 238         hdev->eq_wq = alloc_workqueue("hl-events", WQ_UNBOUND, 0);
 239         if (hdev->eq_wq == NULL) {
 240                 dev_err(hdev->dev, "Failed to allocate EQ workqueue\n");
 241                 rc = -ENOMEM;
 242                 goto free_cq_wq;
 243         }
 244
 245         hdev->hl_chip_info = kzalloc(sizeof(struct hwmon_chip_info),
 246                                         GFP_KERNEL);
 247         if (!hdev->hl_chip_info) {
 248                 rc = -ENOMEM;
 249                 goto free_eq_wq;
 250         }
 251
 252         hl_cb_mgr_init(&hdev->kernel_cb_mgr);
 253
 254         mutex_init(&hdev->send_cpu_message_lock);
 255         mutex_init(&hdev->debug_lock);
 256         mutex_init(&hdev->mmu_cache_lock);
 257         INIT_LIST_HEAD(&hdev->hw_queues_mirror_list);
 258         spin_lock_init(&hdev->hw_queues_mirror_lock);
 259         INIT_LIST_HEAD(&hdev->fpriv_list);
 260         mutex_init(&hdev->fpriv_list_lock);
 261         atomic_set(&hdev->in_reset, 0);
 262         atomic_set(&hdev->cs_active_cnt, 0);
 263
 264         return 0;
 265
 266 free_eq_wq:
 267         destroy_workqueue(hdev->eq_wq);
 268 free_cq_wq:
 269         destroy_workqueue(hdev->cq_wq);
 270 asid_fini:
 271         hl_asid_fini(hdev);
 272 early_fini:
 273         if (hdev->asic_funcs->early_fini)
 274                 hdev->asic_funcs->early_fini(hdev);
 275
 276         return rc;
 277 }
 278
 279 /*
 280  * device_early_fini - finalize all that was done in device_early_init
 281  *
 282  * @hdev: pointer to habanalabs device structure
 283  *
 284  */
 285 static void device_early_fini(struct hl_device *hdev)
 286 {
 287         mutex_destroy(&hdev->mmu_cache_lock);
 288         mutex_destroy(&hdev->debug_lock);
 289         mutex_destroy(&hdev->send_cpu_message_lock);
 290
 291         mutex_destroy(&hdev->fpriv_list_lock);
 292
 293         hl_cb_mgr_fini(hdev, &hdev->kernel_cb_mgr);
 294
 295         kfree(hdev->hl_chip_info);
 296
 297         destroy_workqueue(hdev->eq_wq);
 298         destroy_workqueue(hdev->cq_wq);
 299
 300         hl_asid_fini(hdev);
 301
 302         if (hdev->asic_funcs->early_fini)
 303                 hdev->asic_funcs->early_fini(hdev);
 304 }
 305
 306 static void set_freq_to_low_job(struct work_struct *work)
 307 {
 308         struct hl_device *hdev = container_of(work, struct hl_device,
 309                                                 work_freq.work);
 310
 311         mutex_lock(&hdev->fpriv_list_lock);
 312
 313         if (!hdev->compute_ctx)
 314                 hl_device_set_frequency(hdev, PLL_LOW);
 315
 316         mutex_unlock(&hdev->fpriv_list_lock);
 317
 318         schedule_delayed_work(&hdev->work_freq,
 319                         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 320 }
 321
 322 static void hl_device_heartbeat(struct work_struct *work)
 323 {
 324         struct hl_device *hdev = container_of(work, struct hl_device,
 325                                                 work_heartbeat.work);
 326
 327         if (hl_device_disabled_or_in_reset(hdev))
 328                 goto reschedule;
 329
 330         if (!hdev->asic_funcs->send_heartbeat(hdev))
 331                 goto reschedule;
 332
 333         dev_err(hdev->dev, "Device heartbeat failed!\n");
 334         hl_device_reset(hdev, true, false);
 335
 336         return;
 337
 338 reschedule:
 339         schedule_delayed_work(&hdev->work_heartbeat,
 340                         usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 341 }
 342
 343 /*
 344  * device_late_init - do late stuff initialization for the habanalabs device
 345  *
 346  * @hdev: pointer to habanalabs device structure
 347  *
 348  * Do stuff that either needs the device H/W queues to be active or needs
 349  * to happen after all the rest of the initialization is finished
 350  */
 351 static int device_late_init(struct hl_device *hdev)
 352 {
 353         int rc;
 354
 355         if (hdev->asic_funcs->late_init) {
 356                 rc = hdev->asic_funcs->late_init(hdev);
 357                 if (rc) {
 358                         dev_err(hdev->dev,
 359                                 "failed late initialization for the H/W\n");
 360                         return rc;
 361                 }
 362         }
 363
 364         hdev->high_pll = hdev->asic_prop.high_pll;
 365
 366         /* force setting to low frequency */
 367         hdev->curr_pll_profile = PLL_LOW;
 368
 369         if (hdev->pm_mng_profile == PM_AUTO)
 370                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LOW);
 371         else
 372                 hdev->asic_funcs->set_pll_profile(hdev, PLL_LAST);
 373
 374         INIT_DELAYED_WORK(&hdev->work_freq, set_freq_to_low_job);
 375         schedule_delayed_work(&hdev->work_freq,
 376         usecs_to_jiffies(HL_PLL_LOW_JOB_FREQ_USEC));
 377
 378         if (hdev->heartbeat) {
 379                 INIT_DELAYED_WORK(&hdev->work_heartbeat, hl_device_heartbeat);
 380                 schedule_delayed_work(&hdev->work_heartbeat,
 381                                 usecs_to_jiffies(HL_HEARTBEAT_PER_USEC));
 382         }
 383
 384         hdev->late_init_done = true;
 385
 386         return 0;
 387 }
 388
 389 /*
 390  * device_late_fini - finalize all that was done in device_late_init
 391  *
 392  * @hdev: pointer to habanalabs device structure
 393  *
 394  */
 395 static void device_late_fini(struct hl_device *hdev)
 396 {
 397         if (!hdev->late_init_done)
 398                 return;
 399
 400         cancel_delayed_work_sync(&hdev->work_freq);
 401         if (hdev->heartbeat)
 402                 cancel_delayed_work_sync(&hdev->work_heartbeat);
 403
 404         if (hdev->asic_funcs->late_fini)
 405                 hdev->asic_funcs->late_fini(hdev);
 406
 407         hdev->late_init_done = false;
 408 }
 409
 410 /*
 411  * hl_device_set_frequency - set the frequency of the device
 412  *
 413  * @hdev: pointer to habanalabs device structure
 414  * @freq: the new frequency value
 415  *
 416  * Change the frequency if needed. This function has no protection against
 417  * concurrency, therefore it is assumed that the calling function has protected
 418  * itself against the case of calling this function from multiple threads with
 419  * different values
 420  *
 421  * Returns 0 if no change was done, otherwise returns 1
 422  */
 423 int hl_device_set_frequency(struct hl_device *hdev, enum hl_pll_frequency freq)
 424 {
 425         if ((hdev->pm_mng_profile == PM_MANUAL) ||
 426                         (hdev->curr_pll_profile == freq))
 427                 return 0;
 428
 429         dev_dbg(hdev->dev, "Changing device frequency to %s\n",
 430                 freq == PLL_HIGH ? "high" : "low");
 431
 432         hdev->asic_funcs->set_pll_profile(hdev, freq);
 433
 434         hdev->curr_pll_profile = freq;
 435
 436         return 1;
 437 }
 438
 439 int hl_device_set_debug_mode(struct hl_device *hdev, bool enable)
 440 {
 441         int rc = 0;
 442
 443         mutex_lock(&hdev->debug_lock);
 444
 445         if (!enable) {
 446                 if (!hdev->in_debug) {
 447                         dev_err(hdev->dev,
 448                                 "Failed to disable debug mode because device was not in debug mode\n");
 449                         rc = -EFAULT;
 450                         goto out;
 451                 }
 452
 453                 hdev->asic_funcs->halt_coresight(hdev);
 454                 hdev->in_debug = 0;
 455
 456                 goto out;
 457         }
 458
 459         if (hdev->in_debug) {
 460                 dev_err(hdev->dev,
 461                         "Failed to enable debug mode because device is already in debug mode\n");
 462                 rc = -EFAULT;
 463                 goto out;
 464         }
 465
 466         hdev->in_debug = 1;
 467
 468 out:
 469         mutex_unlock(&hdev->debug_lock);
 470
 471         return rc;
 472 }
 473
 474 /*
 475  * hl_device_suspend - initiate device suspend
 476  *
 477  * @hdev: pointer to habanalabs device structure
 478  *
 479  * Puts the hw in the suspend state (all asics).
 480  * Returns 0 for success or an error on failure.
 481  * Called at driver suspend.
 482  */
 483 int hl_device_suspend(struct hl_device *hdev)
 484 {
 485         int rc;
 486
 487         pci_save_state(hdev->pdev);
 488
 489         /* Block future CS/VM/JOB completion operations */
 490         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 491         if (rc) {
 492                 dev_err(hdev->dev, "Can't suspend while in reset\n");
 493                 return -EIO;
 494         }
 495
 496         /* This blocks all other stuff that is not blocked by in_reset */
 497         hdev->disabled = true;
 498
 499         /*
 500          * Flush anyone that is inside the critical section of enqueue
 501          * jobs to the H/W
 502          */
 503         hdev->asic_funcs->hw_queues_lock(hdev);
 504         hdev->asic_funcs->hw_queues_unlock(hdev);
 505
 506         /* Flush processes that are sending message to CPU */
 507         mutex_lock(&hdev->send_cpu_message_lock);
 508         mutex_unlock(&hdev->send_cpu_message_lock);
 509
 510         rc = hdev->asic_funcs->suspend(hdev);
 511         if (rc)
 512                 dev_err(hdev->dev,
 513                         "Failed to disable PCI access of device CPU\n");
 514
 515         /* Shut down the device */
 516         pci_disable_device(hdev->pdev);
 517         pci_set_power_state(hdev->pdev, PCI_D3hot);
 518
 519         return 0;
 520 }
 521
 522 /*
 523  * hl_device_resume - initiate device resume
 524  *
 525  * @hdev: pointer to habanalabs device structure
 526  *
 527  * Bring the hw back to operating state (all asics).
 528  * Returns 0 for success or an error on failure.
 529  * Called at driver resume.
 530  */
 531 int hl_device_resume(struct hl_device *hdev)
 532 {
 533         int rc;
 534
 535         pci_set_power_state(hdev->pdev, PCI_D0);
 536         pci_restore_state(hdev->pdev);
 537         rc = pci_enable_device_mem(hdev->pdev);
 538         if (rc) {
 539                 dev_err(hdev->dev,
 540                         "Failed to enable PCI device in resume\n");
 541                 return rc;
 542         }
 543
 544         pci_set_master(hdev->pdev);
 545
 546         rc = hdev->asic_funcs->resume(hdev);
 547         if (rc) {
 548                 dev_err(hdev->dev, "Failed to resume device after suspend\n");
 549                 goto disable_device;
 550         }
 551
 552
 553         hdev->disabled = false;
 554         atomic_set(&hdev->in_reset, 0);
 555
 556         rc = hl_device_reset(hdev, true, false);
 557         if (rc) {
 558                 dev_err(hdev->dev, "Failed to reset device during resume\n");
 559                 goto disable_device;
 560         }
 561
 562         return 0;
 563
 564 disable_device:
 565         pci_clear_master(hdev->pdev);
 566         pci_disable_device(hdev->pdev);
 567
 568         return rc;
 569 }
 570
 571 static void device_kill_open_processes(struct hl_device *hdev)
 572 {
 573         u16 pending_total, pending_cnt;
 574         struct hl_fpriv *hpriv;
 575         struct task_struct *task = NULL;
 576
 577         if (hdev->pldm)
 578                 pending_total = HL_PLDM_PENDING_RESET_PER_SEC;
 579         else
 580                 pending_total = HL_PENDING_RESET_PER_SEC;
 581
 582         /* Giving time for user to close FD, and for processes that are inside
 583          * hl_device_open to finish
 584          */
 585         if (!list_empty(&hdev->fpriv_list))
 586                 ssleep(1);
 587
 588         mutex_lock(&hdev->fpriv_list_lock);
 589
 590         /* This section must be protected because we are dereferencing
 591          * pointers that are freed if the process exits
 592          */
 593         list_for_each_entry(hpriv, &hdev->fpriv_list, dev_node) {
 594                 task = get_pid_task(hpriv->taskpid, PIDTYPE_PID);
 595                 if (task) {
 596                         dev_info(hdev->dev, "Killing user process pid=%d\n",
 597                                 task_pid_nr(task));
 598                         send_sig(SIGKILL, task, 1);
 599                         usleep_range(1000, 10000);
 600
 601                         put_task_struct(task);
 602                 }
 603         }
 604
 605         mutex_unlock(&hdev->fpriv_list_lock);
 606
 607         /* We killed the open users, but because the driver cleans up after the
 608          * user contexts are closed (e.g. mmu mappings), we need to wait again
 609          * to make sure the cleaning phase is finished before continuing with
 610          * the reset
 611          */
 612
 613         pending_cnt = pending_total;
 614
 615         while ((!list_empty(&hdev->fpriv_list)) && (pending_cnt)) {
 616                 dev_info(hdev->dev,
 617                         "Waiting for all unmap operations to finish before hard reset\n");
 618
 619                 pending_cnt--;
 620
 621                 ssleep(1);
 622         }
 623
 624         if (!list_empty(&hdev->fpriv_list))
 625                 dev_crit(hdev->dev,
 626                         "Going to hard reset with open user contexts\n");
 627 }
 628
 629 static void device_hard_reset_pending(struct work_struct *work)
 630 {
 631         struct hl_device_reset_work *device_reset_work =
 632                 container_of(work, struct hl_device_reset_work, reset_work);
 633         struct hl_device *hdev = device_reset_work->hdev;
 634
 635         hl_device_reset(hdev, true, true);
 636
 637         kfree(device_reset_work);
 638 }
 639
 640 /*
 641  * hl_device_reset - reset the device
 642  *
 643  * @hdev: pointer to habanalabs device structure
 644  * @hard_reset: should we do hard reset to all engines or just reset the
 645  *              compute/dma engines
 646  *
 647  * Block future CS and wait for pending CS to be enqueued
 648  * Call ASIC H/W fini
 649  * Flush all completions
 650  * Re-initialize all internal data structures
 651  * Call ASIC H/W init, late_init
 652  * Test queues
 653  * Enable device
 654  *
 655  * Returns 0 for success or an error on failure.
 656  */
 657 int hl_device_reset(struct hl_device *hdev, bool hard_reset,
 658                         bool from_hard_reset_thread)
 659 {
 660         int i, rc;
 661
 662         if (!hdev->init_done) {
 663                 dev_err(hdev->dev,
 664                         "Can't reset before initialization is done\n");
 665                 return 0;
 666         }
 667
 668         /*
 669          * Prevent concurrency in this function - only one reset should be
 670          * done at any given time. Only need to perform this if we didn't
 671          * get from the dedicated hard reset thread
 672          */
 673         if (!from_hard_reset_thread) {
 674                 /* Block future CS/VM/JOB completion operations */
 675                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
 676                 if (rc)
 677                         return 0;
 678
 679                 /* This also blocks future CS/VM/JOB completion operations */
 680                 hdev->disabled = true;
 681
 682                 /* Flush anyone that is inside the critical section of enqueue
 683                  * jobs to the H/W
 684                  */
 685                 hdev->asic_funcs->hw_queues_lock(hdev);
 686                 hdev->asic_funcs->hw_queues_unlock(hdev);
 687
 688                 /* Flush anyone that is inside device open */
 689                 mutex_lock(&hdev->fpriv_list_lock);
 690                 mutex_unlock(&hdev->fpriv_list_lock);
 691
 692                 dev_err(hdev->dev, "Going to RESET device!\n");
 693         }
 694
 695 again:
 696         if ((hard_reset) && (!from_hard_reset_thread)) {
 697                 struct hl_device_reset_work *device_reset_work;
 698
 699                 hdev->hard_reset_pending = true;
 700
 701                 device_reset_work = kzalloc(sizeof(*device_reset_work),
 702                                                 GFP_ATOMIC);
 703                 if (!device_reset_work) {
 704                         rc = -ENOMEM;
 705                         goto out_err;
 706                 }
 707
 708                 /*
 709                  * Because the reset function can't run from interrupt or
 710                  * from heartbeat work, we need to call the reset function
 711                  * from a dedicated work
 712                  */
 713                 INIT_WORK(&device_reset_work->reset_work,
 714                                 device_hard_reset_pending);
 715                 device_reset_work->hdev = hdev;
 716                 schedule_work(&device_reset_work->reset_work);
 717
 718                 return 0;
 719         }
 720
 721         if (hard_reset) {
 722                 device_late_fini(hdev);
 723
 724                 /*
 725                  * Now that the heartbeat thread is closed, flush processes
 726                  * which are sending messages to CPU
 727                  */
 728                 mutex_lock(&hdev->send_cpu_message_lock);
 729                 mutex_unlock(&hdev->send_cpu_message_lock);
 730         }
 731
 732         /*
 733          * Halt the engines and disable interrupts so we won't get any more
 734          * completions from H/W and we won't have any accesses from the
 735          * H/W to the host machine
 736          */
 737         hdev->asic_funcs->halt_engines(hdev, hard_reset);
 738
 739         /* Go over all the queues, release all CS and their jobs */
 740         hl_cs_rollback_all(hdev);
 741
 742         /* Kill processes here after CS rollback. This is because the process
 743          * can't really exit until all its CSs are done, which is what we
 744          * do in cs rollback
 745          */
 746         if (from_hard_reset_thread)
 747                 device_kill_open_processes(hdev);
 748
 749         /* Release kernel context */
 750         if ((hard_reset) && (hl_ctx_put(hdev->kernel_ctx) == 1))
 751                 hdev->kernel_ctx = NULL;
 752
 753         /* Reset the H/W. It will be in idle state after this returns */
 754         hdev->asic_funcs->hw_fini(hdev, hard_reset);
 755
 756         if (hard_reset) {
 757                 hl_vm_fini(hdev);
 758                 hl_mmu_fini(hdev);
 759                 hl_eq_reset(hdev, &hdev->event_queue);
 760         }
 761
 762         /* Re-initialize PI,CI to 0 in all queues (hw queue, cq) */
 763         hl_hw_queue_reset(hdev, hard_reset);
 764         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
 765                 hl_cq_reset(hdev, &hdev->completion_queue[i]);
 766
 767         mutex_lock(&hdev->fpriv_list_lock);
 768
 769         /* Make sure the context switch phase will run again */
 770         if (hdev->compute_ctx) {
 771                 atomic_set(&hdev->compute_ctx->thread_ctx_switch_token, 1);
 772                 hdev->compute_ctx->thread_ctx_switch_wait_token = 0;
 773         }
 774
 775         mutex_unlock(&hdev->fpriv_list_lock);
 776
 777         /* Finished tear-down, starting to re-initialize */
 778
 779         if (hard_reset) {
 780                 hdev->device_cpu_disabled = false;
 781                 hdev->hard_reset_pending = false;
 782
 783                 if (hdev->kernel_ctx) {
 784                         dev_crit(hdev->dev,
 785                                 "kernel ctx was alive during hard reset, something is terribly wrong\n");
 786                         rc = -EBUSY;
 787                         goto out_err;
 788                 }
 789
 790                 rc = hl_mmu_init(hdev);
 791                 if (rc) {
 792                         dev_err(hdev->dev,
 793                                 "Failed to initialize MMU S/W after hard reset\n");
 794                         goto out_err;
 795                 }
 796
 797                 /* Allocate the kernel context */
 798                 hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx),
 799                                                 GFP_KERNEL);
 800                 if (!hdev->kernel_ctx) {
 801                         rc = -ENOMEM;
 802                         goto out_err;
 803                 }
 804
 805                 hdev->compute_ctx = NULL;
 806
 807                 rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
 808                 if (rc) {
 809                         dev_err(hdev->dev,
 810                                 "failed to init kernel ctx in hard reset\n");
 811                         kfree(hdev->kernel_ctx);
 812                         hdev->kernel_ctx = NULL;
 813                         goto out_err;
 814                 }
 815         }
 816
 817         rc = hdev->asic_funcs->hw_init(hdev);
 818         if (rc) {
 819                 dev_err(hdev->dev,
 820                         "failed to initialize the H/W after reset\n");
 821                 goto out_err;
 822         }
 823
 824         hdev->disabled = false;
 825
 826         /* Check that the communication with the device is working */
 827         rc = hdev->asic_funcs->test_queues(hdev);
 828         if (rc) {
 829                 dev_err(hdev->dev,
 830                         "Failed to detect if device is alive after reset\n");
 831                 goto out_err;
 832         }
 833
 834         if (hard_reset) {
 835                 rc = device_late_init(hdev);
 836                 if (rc) {
 837                         dev_err(hdev->dev,
 838                                 "Failed late init after hard reset\n");
 839                         goto out_err;
 840                 }
 841
 842                 rc = hl_vm_init(hdev);
 843                 if (rc) {
 844                         dev_err(hdev->dev,
 845                                 "Failed to init memory module after hard reset\n");
 846                         goto out_err;
 847                 }
 848
 849                 hl_set_max_power(hdev, hdev->max_power);
 850         } else {
 851                 rc = hdev->asic_funcs->soft_reset_late_init(hdev);
 852                 if (rc) {
 853                         dev_err(hdev->dev,
 854                                 "Failed late init after soft reset\n");
 855                         goto out_err;
 856                 }
 857         }
 858
 859         atomic_set(&hdev->in_reset, 0);
 860
 861         if (hard_reset)
 862                 hdev->hard_reset_cnt++;
 863         else
 864                 hdev->soft_reset_cnt++;
 865
 866         return 0;
 867
 868 out_err:
 869         hdev->disabled = true;
 870
 871         if (hard_reset) {
 872                 dev_err(hdev->dev,
 873                         "Failed to reset! Device is NOT usable\n");
 874                 hdev->hard_reset_cnt++;
 875         } else {
 876                 dev_err(hdev->dev,
 877                         "Failed to do soft-reset, trying hard reset\n");
 878                 hdev->soft_reset_cnt++;
 879                 hard_reset = true;
 880                 goto again;
 881         }
 882
 883         atomic_set(&hdev->in_reset, 0);
 884
 885         return rc;
 886 }
 887
 888 /*
 889  * hl_device_init - main initialization function for habanalabs device
 890  *
 891  * @hdev: pointer to habanalabs device structure
 892  *
 893  * Allocate an id for the device, do early initialization and then call the
 894  * ASIC specific initialization functions. Finally, create the cdev and the
 895  * Linux device to expose it to the user
 896  */
 897 int hl_device_init(struct hl_device *hdev, struct class *hclass)
 898 {
 899         int i, rc, cq_ready_cnt;
 900         char *name;
 901
 902         name = kasprintf(GFP_KERNEL, "hl%d", hdev->id / 2);
 903         if (!name)
 904                 return -ENOMEM;
 905
 906         /* Create device */
 907         rc = device_setup_cdev(hdev, hclass, hdev->id, &hl_ops, name,
 908                                 &hdev->cdev, &hdev->dev);
 909
 910         kfree(name);
 911
 912         if (rc)
 913                 goto out_disabled;
 914
 915         name = kasprintf(GFP_KERNEL, "hl_controlD%d", hdev->id / 2);
 916         if (!name) {
 917                 rc = -ENOMEM;
 918                 goto release_device;
 919         }
 920
 921         /* Create control device */
 922         rc = device_setup_cdev(hdev, hclass, hdev->id_control, &hl_ctrl_ops,
 923                                 name, &hdev->cdev_ctrl, &hdev->dev_ctrl);
 924
 925         kfree(name);
 926
 927         if (rc)
 928                 goto release_device;
 929
 930         /* Initialize ASIC function pointers and perform early init */
 931         rc = device_early_init(hdev);
 932         if (rc)
 933                 goto release_control_device;
 934
 935         /*
 936          * Start calling ASIC initialization. First S/W then H/W and finally
 937          * late init
 938          */
 939         rc = hdev->asic_funcs->sw_init(hdev);
 940         if (rc)
 941                 goto early_fini;
 942
 943         /*
 944          * Initialize the H/W queues. Must be done before hw_init, because
 945          * there the addresses of the kernel queue are being written to the
 946          * registers of the device
 947          */
 948         rc = hl_hw_queues_create(hdev);
 949         if (rc) {
 950                 dev_err(hdev->dev, "failed to initialize kernel queues\n");
 951                 goto sw_fini;
 952         }
 953
 954         /*
 955          * Initialize the completion queues. Must be done before hw_init,
 956          * because there the addresses of the completion queues are being
 957          * passed as arguments to request_irq
 958          */
 959         hdev->completion_queue =
 960                         kcalloc(hdev->asic_prop.completion_queues_count,
 961                                 sizeof(*hdev->completion_queue), GFP_KERNEL);
 962
 963         if (!hdev->completion_queue) {
 964                 dev_err(hdev->dev, "failed to allocate completion queues\n");
 965                 rc = -ENOMEM;
 966                 goto hw_queues_destroy;
 967         }
 968
 969         for (i = 0, cq_ready_cnt = 0;
 970                         i < hdev->asic_prop.completion_queues_count;
 971                         i++, cq_ready_cnt++) {
 972                 rc = hl_cq_init(hdev, &hdev->completion_queue[i], i);
 973                 if (rc) {
 974                         dev_err(hdev->dev,
 975                                 "failed to initialize completion queue\n");
 976                         goto cq_fini;
 977                 }
 978         }
 979
 980         /*
 981          * Initialize the event queue. Must be done before hw_init,
 982          * because there the address of the event queue is being
 983          * passed as argument to request_irq
 984          */
 985         rc = hl_eq_init(hdev, &hdev->event_queue);
 986         if (rc) {
 987                 dev_err(hdev->dev, "failed to initialize event queue\n");
 988                 goto cq_fini;
 989         }
 990
 991         /* MMU S/W must be initialized before kernel context is created */
 992         rc = hl_mmu_init(hdev);
 993         if (rc) {
 994                 dev_err(hdev->dev, "Failed to initialize MMU S/W structures\n");
 995                 goto eq_fini;
 996         }
 997
 998         /* Allocate the kernel context */
 999         hdev->kernel_ctx = kzalloc(sizeof(*hdev->kernel_ctx), GFP_KERNEL);
1000         if (!hdev->kernel_ctx) {
1001                 rc = -ENOMEM;
1002                 goto mmu_fini;
1003         }
1004
1005         hdev->compute_ctx = NULL;
1006
1007         rc = hl_ctx_init(hdev, hdev->kernel_ctx, true);
1008         if (rc) {
1009                 dev_err(hdev->dev, "failed to initialize kernel context\n");
1010                 kfree(hdev->kernel_ctx);
1011                 goto mmu_fini;
1012         }
1013
1014         rc = hl_cb_pool_init(hdev);
1015         if (rc) {
1016                 dev_err(hdev->dev, "failed to initialize CB pool\n");
1017                 goto release_ctx;
1018         }
1019
1020         rc = hl_sysfs_init(hdev);
1021         if (rc) {
1022                 dev_err(hdev->dev, "failed to initialize sysfs\n");
1023                 goto free_cb_pool;
1024         }
1025
1026         hl_debugfs_add_device(hdev);
1027
1028         if (hdev->asic_funcs->get_hw_state(hdev) == HL_DEVICE_HW_STATE_DIRTY) {
1029                 dev_info(hdev->dev,
1030                         "H/W state is dirty, must reset before initializing\n");
1031                 hdev->asic_funcs->hw_fini(hdev, true);
1032         }
1033
1034         rc = hdev->asic_funcs->hw_init(hdev);
1035         if (rc) {
1036                 dev_err(hdev->dev, "failed to initialize the H/W\n");
1037                 rc = 0;
1038                 goto out_disabled;
1039         }
1040
1041         hdev->disabled = false;
1042
1043         /* Check that the communication with the device is working */
1044         rc = hdev->asic_funcs->test_queues(hdev);
1045         if (rc) {
1046                 dev_err(hdev->dev, "Failed to detect if device is alive\n");
1047                 rc = 0;
1048                 goto out_disabled;
1049         }
1050
1051         rc = device_late_init(hdev);
1052         if (rc) {
1053                 dev_err(hdev->dev, "Failed late initialization\n");
1054                 rc = 0;
1055                 goto out_disabled;
1056         }
1057
1058         dev_info(hdev->dev, "Found %s device with %lluGB DRAM\n",
1059                 hdev->asic_name,
1060                 hdev->asic_prop.dram_size / 1024 / 1024 / 1024);
1061
1062         rc = hl_vm_init(hdev);
1063         if (rc) {
1064                 dev_err(hdev->dev, "Failed to initialize memory module\n");
1065                 rc = 0;
1066                 goto out_disabled;
1067         }
1068
1069         /*
1070          * hl_hwmon_init must be called after device_late_init, because only
1071          * there we get the information from the device about which
1072          * hwmon-related sensors the device supports
1073          */
1074         rc = hl_hwmon_init(hdev);
1075         if (rc) {
1076                 dev_err(hdev->dev, "Failed to initialize hwmon\n");
1077                 rc = 0;
1078                 goto out_disabled;
1079         }
1080
1081         dev_notice(hdev->dev,
1082                 "Successfully added device to habanalabs driver\n");
1083
1084         hdev->init_done = true;
1085
1086         return 0;
1087
1088 free_cb_pool:
1089         hl_cb_pool_fini(hdev);
1090 release_ctx:
1091         if (hl_ctx_put(hdev->kernel_ctx) != 1)
1092                 dev_err(hdev->dev,
1093                         "kernel ctx is still alive on initialization failure\n");
1094 mmu_fini:
1095         hl_mmu_fini(hdev);
1096 eq_fini:
1097         hl_eq_fini(hdev, &hdev->event_queue);
1098 cq_fini:
1099         for (i = 0 ; i < cq_ready_cnt ; i++)
1100                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1101         kfree(hdev->completion_queue);
1102 hw_queues_destroy:
1103         hl_hw_queues_destroy(hdev);
1104 sw_fini:
1105         hdev->asic_funcs->sw_fini(hdev);
1106 early_fini:
1107         device_early_fini(hdev);
1108 release_control_device:
1109         device_destroy(hclass, hdev->dev_ctrl->devt);
1110         cdev_del(&hdev->cdev_ctrl);
1111 release_device:
1112         device_destroy(hclass, hdev->dev->devt);
1113         cdev_del(&hdev->cdev);
1114 out_disabled:
1115         hdev->disabled = true;
1116         if (hdev->pdev)
1117                 dev_err(&hdev->pdev->dev,
1118                         "Failed to initialize hl%d. Device is NOT usable !\n",
1119                         hdev->id);
1120         else
1121                 pr_err("Failed to initialize hl%d. Device is NOT usable !\n",
1122                         hdev->id);
1123
1124         return rc;
1125 }
1126
1127 /*
1128  * hl_device_fini - main tear-down function for habanalabs device
1129  *
1130  * @hdev: pointer to habanalabs device structure
1131  *
1132  * Destroy the device, call ASIC fini functions and release the id
1133  */
1134 void hl_device_fini(struct hl_device *hdev)
1135 {
1136         int i, rc;
1137         ktime_t timeout;
1138
1139         dev_info(hdev->dev, "Removing device\n");
1140
1141         /*
1142          * This function is competing with the reset function, so try to
1143          * take the reset atomic and if we are already in middle of reset,
1144          * wait until reset function is finished. Reset function is designed
1145          * to always finish (could take up to a few seconds in worst case).
1146          */
1147
1148         timeout = ktime_add_us(ktime_get(),
1149                                 HL_PENDING_RESET_PER_SEC * 1000 * 1000 * 4);
1150         rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1151         while (rc) {
1152                 usleep_range(50, 200);
1153                 rc = atomic_cmpxchg(&hdev->in_reset, 0, 1);
1154                 if (ktime_compare(ktime_get(), timeout) > 0) {
1155                         WARN(1, "Failed to remove device because reset function did not finish\n");
1156                         return;
1157                 }
1158         }
1159
1160         /* Mark device as disabled */
1161         hdev->disabled = true;
1162
1163         /* Flush anyone that is inside the critical section of enqueue
1164          * jobs to the H/W
1165          */
1166         hdev->asic_funcs->hw_queues_lock(hdev);
1167         hdev->asic_funcs->hw_queues_unlock(hdev);
1168
1169         /* Flush anyone that is inside device open */
1170         mutex_lock(&hdev->fpriv_list_lock);
1171         mutex_unlock(&hdev->fpriv_list_lock);
1172
1173         hdev->hard_reset_pending = true;
1174
1175         hl_hwmon_fini(hdev);
1176
1177         device_late_fini(hdev);
1178
1179         hl_debugfs_remove_device(hdev);
1180
1181         hl_sysfs_fini(hdev);
1182
1183         /*
1184          * Halt the engines and disable interrupts so we won't get any more
1185          * completions from H/W and we won't have any accesses from the
1186          * H/W to the host machine
1187          */
1188         hdev->asic_funcs->halt_engines(hdev, true);
1189
1190         /* Go over all the queues, release all CS and their jobs */
1191         hl_cs_rollback_all(hdev);
1192
1193         /* Kill processes here after CS rollback. This is because the process
1194          * can't really exit until all its CSs are done, which is what we
1195          * do in cs rollback
1196          */
1197         device_kill_open_processes(hdev);
1198
1199         hl_cb_pool_fini(hdev);
1200
1201         /* Release kernel context */
1202         if ((hdev->kernel_ctx) && (hl_ctx_put(hdev->kernel_ctx) != 1))
1203                 dev_err(hdev->dev, "kernel ctx is still alive\n");
1204
1205         /* Reset the H/W. It will be in idle state after this returns */
1206         hdev->asic_funcs->hw_fini(hdev, true);
1207
1208         hl_vm_fini(hdev);
1209
1210         hl_mmu_fini(hdev);
1211
1212         hl_eq_fini(hdev, &hdev->event_queue);
1213
1214         for (i = 0 ; i < hdev->asic_prop.completion_queues_count ; i++)
1215                 hl_cq_fini(hdev, &hdev->completion_queue[i]);
1216         kfree(hdev->completion_queue);
1217
1218         hl_hw_queues_destroy(hdev);
1219
1220         /* Call ASIC S/W finalize function */
1221         hdev->asic_funcs->sw_fini(hdev);
1222
1223         device_early_fini(hdev);
1224
1225         /* Hide device from user */
1226         device_destroy(hdev->dev_ctrl->class, hdev->dev_ctrl->devt);
1227         cdev_del(&hdev->cdev_ctrl);
1228         device_destroy(hdev->dev->class, hdev->dev->devt);
1229         cdev_del(&hdev->cdev);
1230
1231         pr_info("removed device successfully\n");
1232 }
1233
1234 /*
1235  * MMIO register access helper functions.
1236  */
1237
1238 /*
1239  * hl_rreg - Read an MMIO register
1240  *
1241  * @hdev: pointer to habanalabs device structure
1242  * @reg: MMIO register offset (in bytes)
1243  *
1244  * Returns the value of the MMIO register we are asked to read
1245  *
1246  */
1247 inline u32 hl_rreg(struct hl_device *hdev, u32 reg)
1248 {
1249         return readl(hdev->rmmio + reg);
1250 }
1251
1252 /*
1253  * hl_wreg - Write to an MMIO register
1254  *
1255  * @hdev: pointer to habanalabs device structure
1256  * @reg: MMIO register offset (in bytes)
1257  * @val: 32-bit value
1258  *
1259  * Writes the 32-bit value into the MMIO register
1260  *
1261  */
1262 inline void hl_wreg(struct hl_device *hdev, u32 reg, u32 val)
1263 {
1264         writel(val, hdev->rmmio + reg);
1265 }