drivers/gpu/drm/amd/amdkfd/kfd_process.c

   1 /*
   2  * Copyright 2014 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  */
  22
  23 #include <linux/mutex.h>
  24 #include <linux/log2.h>
  25 #include <linux/sched.h>
  26 #include <linux/sched/mm.h>
  27 #include <linux/sched/task.h>
  28 #include <linux/mmu_context.h>
  29 #include <linux/slab.h>
  30 #include <linux/amd-iommu.h>
  31 #include <linux/notifier.h>
  32 #include <linux/compat.h>
  33 #include <linux/mman.h>
  34 #include <linux/file.h>
  35 #include <linux/pm_runtime.h>
  36 #include "amdgpu_amdkfd.h"
  37 #include "amdgpu.h"
  38
  39 struct mm_struct;
  40
  41 #include "kfd_priv.h"
  42 #include "kfd_device_queue_manager.h"
  43 #include "kfd_dbgmgr.h"
  44 #include "kfd_iommu.h"
  45 #include "kfd_svm.h"
  46
  47 /*
  48  * List of struct kfd_process (field kfd_process).
  49  * Unique/indexed by mm_struct*
  50  */
  51 DEFINE_HASHTABLE(kfd_processes_table, KFD_PROCESS_TABLE_SIZE);
  52 static DEFINE_MUTEX(kfd_processes_mutex);
  53
  54 DEFINE_SRCU(kfd_processes_srcu);
  55
  56 /* For process termination handling */
  57 static struct workqueue_struct *kfd_process_wq;
  58
  59 /* Ordered, single-threaded workqueue for restoring evicted
  60  * processes. Restoring multiple processes concurrently under memory
  61  * pressure can lead to processes blocking each other from validating
  62  * their BOs and result in a live-lock situation where processes
  63  * remain evicted indefinitely.
  64  */
  65 static struct workqueue_struct *kfd_restore_wq;
  66
  67 static struct kfd_process *find_process(const struct task_struct *thread);
  68 static void kfd_process_ref_release(struct kref *ref);
  69 static struct kfd_process *create_process(const struct task_struct *thread);
  70 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep);
  71
  72 static void evict_process_worker(struct work_struct *work);
  73 static void restore_process_worker(struct work_struct *work);
  74
  75 struct kfd_procfs_tree {
  76         struct kobject *kobj;
  77 };
  78
  79 static struct kfd_procfs_tree procfs;
  80
  81 /*
  82  * Structure for SDMA activity tracking
  83  */
  84 struct kfd_sdma_activity_handler_workarea {
  85         struct work_struct sdma_activity_work;
  86         struct kfd_process_device *pdd;
  87         uint64_t sdma_activity_counter;
  88 };
  89
  90 struct temp_sdma_queue_list {
  91         uint64_t __user *rptr;
  92         uint64_t sdma_val;
  93         unsigned int queue_id;
  94         struct list_head list;
  95 };
  96
  97 static void kfd_sdma_activity_worker(struct work_struct *work)
  98 {
  99         struct kfd_sdma_activity_handler_workarea *workarea;
 100         struct kfd_process_device *pdd;
 101         uint64_t val;
 102         struct mm_struct *mm;
 103         struct queue *q;
 104         struct qcm_process_device *qpd;
 105         struct device_queue_manager *dqm;
 106         int ret = 0;
 107         struct temp_sdma_queue_list sdma_q_list;
 108         struct temp_sdma_queue_list *sdma_q, *next;
 109
 110         workarea = container_of(work, struct kfd_sdma_activity_handler_workarea,
 111                                 sdma_activity_work);
 112
 113         pdd = workarea->pdd;
 114         if (!pdd)
 115                 return;
 116         dqm = pdd->dev->dqm;
 117         qpd = &pdd->qpd;
 118         if (!dqm || !qpd)
 119                 return;
 120         /*
 121          * Total SDMA activity is current SDMA activity + past SDMA activity
 122          * Past SDMA count is stored in pdd.
 123          * To get the current activity counters for all active SDMA queues,
 124          * we loop over all SDMA queues and get their counts from user-space.
 125          *
 126          * We cannot call get_user() with dqm_lock held as it can cause
 127          * a circular lock dependency situation. To read the SDMA stats,
 128          * we need to do the following:
 129          *
 130          * 1. Create a temporary list of SDMA queue nodes from the qpd->queues_list,
 131          *    with dqm_lock/dqm_unlock().
 132          * 2. Call get_user() for each node in temporary list without dqm_lock.
 133          *    Save the SDMA count for each node and also add the count to the total
 134          *    SDMA count counter.
 135          *    Its possible, during this step, a few SDMA queue nodes got deleted
 136          *    from the qpd->queues_list.
 137          * 3. Do a second pass over qpd->queues_list to check if any nodes got deleted.
 138          *    If any node got deleted, its SDMA count would be captured in the sdma
 139          *    past activity counter. So subtract the SDMA counter stored in step 2
 140          *    for this node from the total SDMA count.
 141          */
 142         INIT_LIST_HEAD(&sdma_q_list.list);
 143
 144         /*
 145          * Create the temp list of all SDMA queues
 146          */
 147         dqm_lock(dqm);
 148
 149         list_for_each_entry(q, &qpd->queues_list, list) {
 150                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 151                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 152                         continue;
 153
 154                 sdma_q = kzalloc(sizeof(struct temp_sdma_queue_list), GFP_KERNEL);
 155                 if (!sdma_q) {
 156                         dqm_unlock(dqm);
 157                         goto cleanup;
 158                 }
 159
 160                 INIT_LIST_HEAD(&sdma_q->list);
 161                 sdma_q->rptr = (uint64_t __user *)q->properties.read_ptr;
 162                 sdma_q->queue_id = q->properties.queue_id;
 163                 list_add_tail(&sdma_q->list, &sdma_q_list.list);
 164         }
 165
 166         /*
 167          * If the temp list is empty, then no SDMA queues nodes were found in
 168          * qpd->queues_list. Return the past activity count as the total sdma
 169          * count
 170          */
 171         if (list_empty(&sdma_q_list.list)) {
 172                 workarea->sdma_activity_counter = pdd->sdma_past_activity_counter;
 173                 dqm_unlock(dqm);
 174                 return;
 175         }
 176
 177         dqm_unlock(dqm);
 178
 179         /*
 180          * Get the usage count for each SDMA queue in temp_list.
 181          */
 182         mm = get_task_mm(pdd->process->lead_thread);
 183         if (!mm)
 184                 goto cleanup;
 185
 186         kthread_use_mm(mm);
 187
 188         list_for_each_entry(sdma_q, &sdma_q_list.list, list) {
 189                 val = 0;
 190                 ret = read_sdma_queue_counter(sdma_q->rptr, &val);
 191                 if (ret) {
 192                         pr_debug("Failed to read SDMA queue active counter for queue id: %d",
 193                                  sdma_q->queue_id);
 194                 } else {
 195                         sdma_q->sdma_val = val;
 196                         workarea->sdma_activity_counter += val;
 197                 }
 198         }
 199
 200         kthread_unuse_mm(mm);
 201         mmput(mm);
 202
 203         /*
 204          * Do a second iteration over qpd_queues_list to check if any SDMA
 205          * nodes got deleted while fetching SDMA counter.
 206          */
 207         dqm_lock(dqm);
 208
 209         workarea->sdma_activity_counter += pdd->sdma_past_activity_counter;
 210
 211         list_for_each_entry(q, &qpd->queues_list, list) {
 212                 if (list_empty(&sdma_q_list.list))
 213                         break;
 214
 215                 if ((q->properties.type != KFD_QUEUE_TYPE_SDMA) &&
 216                     (q->properties.type != KFD_QUEUE_TYPE_SDMA_XGMI))
 217                         continue;
 218
 219                 list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 220                         if (((uint64_t __user *)q->properties.read_ptr == sdma_q->rptr) &&
 221                              (sdma_q->queue_id == q->properties.queue_id)) {
 222                                 list_del(&sdma_q->list);
 223                                 kfree(sdma_q);
 224                                 break;
 225                         }
 226                 }
 227         }
 228
 229         dqm_unlock(dqm);
 230
 231         /*
 232          * If temp list is not empty, it implies some queues got deleted
 233          * from qpd->queues_list during SDMA usage read. Subtract the SDMA
 234          * count for each node from the total SDMA count.
 235          */
 236         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 237                 workarea->sdma_activity_counter -= sdma_q->sdma_val;
 238                 list_del(&sdma_q->list);
 239                 kfree(sdma_q);
 240         }
 241
 242         return;
 243
 244 cleanup:
 245         list_for_each_entry_safe(sdma_q, next, &sdma_q_list.list, list) {
 246                 list_del(&sdma_q->list);
 247                 kfree(sdma_q);
 248         }
 249 }
 250
 251 /**
 252  * @kfd_get_cu_occupancy - Collect number of waves in-flight on this device
 253  * by current process. Translates acquired wave count into number of compute units
 254  * that are occupied.
 255  *
 256  * @atr: Handle of attribute that allows reporting of wave count. The attribute
 257  * handle encapsulates GPU device it is associated with, thereby allowing collection
 258  * of waves in flight, etc
 259  *
 260  * @buffer: Handle of user provided buffer updated with wave count
 261  *
 262  * Return: Number of bytes written to user buffer or an error value
 263  */
 264 static int kfd_get_cu_occupancy(struct attribute *attr, char *buffer)
 265 {
 266         int cu_cnt;
 267         int wave_cnt;
 268         int max_waves_per_cu;
 269         struct kfd_dev *dev = NULL;
 270         struct kfd_process *proc = NULL;
 271         struct kfd_process_device *pdd = NULL;
 272
 273         pdd = container_of(attr, struct kfd_process_device, attr_cu_occupancy);
 274         dev = pdd->dev;
 275         if (dev->kfd2kgd->get_cu_occupancy == NULL)
 276                 return -EINVAL;
 277
 278         cu_cnt = 0;
 279         proc = pdd->process;
 280         if (pdd->qpd.queue_count == 0) {
 281                 pr_debug("Gpu-Id: %d has no active queues for process %d\n",
 282                          dev->id, proc->pasid);
 283                 return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 284         }
 285
 286         /* Collect wave count from device if it supports */
 287         wave_cnt = 0;
 288         max_waves_per_cu = 0;
 289         dev->kfd2kgd->get_cu_occupancy(dev->kgd, proc->pasid, &wave_cnt,
 290                         &max_waves_per_cu);
 291
 292         /* Translate wave count to number of compute units */
 293         cu_cnt = (wave_cnt + (max_waves_per_cu - 1)) / max_waves_per_cu;
 294         return snprintf(buffer, PAGE_SIZE, "%d\n", cu_cnt);
 295 }
 296
 297 static ssize_t kfd_procfs_show(struct kobject *kobj, struct attribute *attr,
 298                                char *buffer)
 299 {
 300         if (strcmp(attr->name, "pasid") == 0) {
 301                 struct kfd_process *p = container_of(attr, struct kfd_process,
 302                                                      attr_pasid);
 303
 304                 return snprintf(buffer, PAGE_SIZE, "%d\n", p->pasid);
 305         } else if (strncmp(attr->name, "vram_", 5) == 0) {
 306                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 307                                                               attr_vram);
 308                 return snprintf(buffer, PAGE_SIZE, "%llu\n", READ_ONCE(pdd->vram_usage));
 309         } else if (strncmp(attr->name, "sdma_", 5) == 0) {
 310                 struct kfd_process_device *pdd = container_of(attr, struct kfd_process_device,
 311                                                               attr_sdma);
 312                 struct kfd_sdma_activity_handler_workarea sdma_activity_work_handler;
 313
 314                 INIT_WORK(&sdma_activity_work_handler.sdma_activity_work,
 315                                         kfd_sdma_activity_worker);
 316
 317                 sdma_activity_work_handler.pdd = pdd;
 318                 sdma_activity_work_handler.sdma_activity_counter = 0;
 319
 320                 schedule_work(&sdma_activity_work_handler.sdma_activity_work);
 321
 322                 flush_work(&sdma_activity_work_handler.sdma_activity_work);
 323
 324                 return snprintf(buffer, PAGE_SIZE, "%llu\n",
 325                                 (sdma_activity_work_handler.sdma_activity_counter)/
 326                                  SDMA_ACTIVITY_DIVISOR);
 327         } else {
 328                 pr_err("Invalid attribute");
 329                 return -EINVAL;
 330         }
 331
 332         return 0;
 333 }
 334
 335 static void kfd_procfs_kobj_release(struct kobject *kobj)
 336 {
 337         kfree(kobj);
 338 }
 339
 340 static const struct sysfs_ops kfd_procfs_ops = {
 341         .show = kfd_procfs_show,
 342 };
 343
 344 static struct kobj_type procfs_type = {
 345         .release = kfd_procfs_kobj_release,
 346         .sysfs_ops = &kfd_procfs_ops,
 347 };
 348
 349 void kfd_procfs_init(void)
 350 {
 351         int ret = 0;
 352
 353         procfs.kobj = kfd_alloc_struct(procfs.kobj);
 354         if (!procfs.kobj)
 355                 return;
 356
 357         ret = kobject_init_and_add(procfs.kobj, &procfs_type,
 358                                    &kfd_device->kobj, "proc");
 359         if (ret) {
 360                 pr_warn("Could not create procfs proc folder");
 361                 /* If we fail to create the procfs, clean up */
 362                 kfd_procfs_shutdown();
 363         }
 364 }
 365
 366 void kfd_procfs_shutdown(void)
 367 {
 368         if (procfs.kobj) {
 369                 kobject_del(procfs.kobj);
 370                 kobject_put(procfs.kobj);
 371                 procfs.kobj = NULL;
 372         }
 373 }
 374
 375 static ssize_t kfd_procfs_queue_show(struct kobject *kobj,
 376                                      struct attribute *attr, char *buffer)
 377 {
 378         struct queue *q = container_of(kobj, struct queue, kobj);
 379
 380         if (!strcmp(attr->name, "size"))
 381                 return snprintf(buffer, PAGE_SIZE, "%llu",
 382                                 q->properties.queue_size);
 383         else if (!strcmp(attr->name, "type"))
 384                 return snprintf(buffer, PAGE_SIZE, "%d", q->properties.type);
 385         else if (!strcmp(attr->name, "gpuid"))
 386                 return snprintf(buffer, PAGE_SIZE, "%u", q->device->id);
 387         else
 388                 pr_err("Invalid attribute");
 389
 390         return 0;
 391 }
 392
 393 static ssize_t kfd_procfs_stats_show(struct kobject *kobj,
 394                                      struct attribute *attr, char *buffer)
 395 {
 396         if (strcmp(attr->name, "evicted_ms") == 0) {
 397                 struct kfd_process_device *pdd = container_of(attr,
 398                                 struct kfd_process_device,
 399                                 attr_evict);
 400                 uint64_t evict_jiffies;
 401
 402                 evict_jiffies = atomic64_read(&pdd->evict_duration_counter);
 403
 404                 return snprintf(buffer,
 405                                 PAGE_SIZE,
 406                                 "%llu\n",
 407                                 jiffies64_to_msecs(evict_jiffies));
 408
 409         /* Sysfs handle that gets CU occupancy is per device */
 410         } else if (strcmp(attr->name, "cu_occupancy") == 0) {
 411                 return kfd_get_cu_occupancy(attr, buffer);
 412         } else {
 413                 pr_err("Invalid attribute");
 414         }
 415
 416         return 0;
 417 }
 418
 419 static ssize_t kfd_sysfs_counters_show(struct kobject *kobj,
 420                                        struct attribute *attr, char *buf)
 421 {
 422         struct kfd_process_device *pdd;
 423
 424         if (!strcmp(attr->name, "faults")) {
 425                 pdd = container_of(attr, struct kfd_process_device,
 426                                    attr_faults);
 427                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->faults));
 428         }
 429         if (!strcmp(attr->name, "page_in")) {
 430                 pdd = container_of(attr, struct kfd_process_device,
 431                                    attr_page_in);
 432                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_in));
 433         }
 434         if (!strcmp(attr->name, "page_out")) {
 435                 pdd = container_of(attr, struct kfd_process_device,
 436                                    attr_page_out);
 437                 return sysfs_emit(buf, "%llu\n", READ_ONCE(pdd->page_out));
 438         }
 439         return 0;
 440 }
 441
 442 static struct attribute attr_queue_size = {
 443         .name = "size",
 444         .mode = KFD_SYSFS_FILE_MODE
 445 };
 446
 447 static struct attribute attr_queue_type = {
 448         .name = "type",
 449         .mode = KFD_SYSFS_FILE_MODE
 450 };
 451
 452 static struct attribute attr_queue_gpuid = {
 453         .name = "gpuid",
 454         .mode = KFD_SYSFS_FILE_MODE
 455 };
 456
 457 static struct attribute *procfs_queue_attrs[] = {
 458         &attr_queue_size,
 459         &attr_queue_type,
 460         &attr_queue_gpuid,
 461         NULL
 462 };
 463
 464 static const struct sysfs_ops procfs_queue_ops = {
 465         .show = kfd_procfs_queue_show,
 466 };
 467
 468 static struct kobj_type procfs_queue_type = {
 469         .sysfs_ops = &procfs_queue_ops,
 470         .default_attrs = procfs_queue_attrs,
 471 };
 472
 473 static const struct sysfs_ops procfs_stats_ops = {
 474         .show = kfd_procfs_stats_show,
 475 };
 476
 477 static struct kobj_type procfs_stats_type = {
 478         .sysfs_ops = &procfs_stats_ops,
 479         .release = kfd_procfs_kobj_release,
 480 };
 481
 482 static const struct sysfs_ops sysfs_counters_ops = {
 483         .show = kfd_sysfs_counters_show,
 484 };
 485
 486 static struct kobj_type sysfs_counters_type = {
 487         .sysfs_ops = &sysfs_counters_ops,
 488         .release = kfd_procfs_kobj_release,
 489 };
 490
 491 int kfd_procfs_add_queue(struct queue *q)
 492 {
 493         struct kfd_process *proc;
 494         int ret;
 495
 496         if (!q || !q->process)
 497                 return -EINVAL;
 498         proc = q->process;
 499
 500         /* Create proc/<pid>/queues/<queue id> folder */
 501         if (!proc->kobj_queues)
 502                 return -EFAULT;
 503         ret = kobject_init_and_add(&q->kobj, &procfs_queue_type,
 504                         proc->kobj_queues, "%u", q->properties.queue_id);
 505         if (ret < 0) {
 506                 pr_warn("Creating proc/<pid>/queues/%u failed",
 507                         q->properties.queue_id);
 508                 kobject_put(&q->kobj);
 509                 return ret;
 510         }
 511
 512         return 0;
 513 }
 514
 515 static void kfd_sysfs_create_file(struct kobject *kobj, struct attribute *attr,
 516                                  char *name)
 517 {
 518         int ret;
 519
 520         if (!kobj || !attr || !name)
 521                 return;
 522
 523         attr->name = name;
 524         attr->mode = KFD_SYSFS_FILE_MODE;
 525         sysfs_attr_init(attr);
 526
 527         ret = sysfs_create_file(kobj, attr);
 528         if (ret)
 529                 pr_warn("Create sysfs %s/%s failed %d", kobj->name, name, ret);
 530 }
 531
 532 static void kfd_procfs_add_sysfs_stats(struct kfd_process *p)
 533 {
 534         int ret;
 535         int i;
 536         char stats_dir_filename[MAX_SYSFS_FILENAME_LEN];
 537
 538         if (!p || !p->kobj)
 539                 return;
 540
 541         /*
 542          * Create sysfs files for each GPU:
 543          * - proc/<pid>/stats_<gpuid>/
 544          * - proc/<pid>/stats_<gpuid>/evicted_ms
 545          * - proc/<pid>/stats_<gpuid>/cu_occupancy
 546          */
 547         for (i = 0; i < p->n_pdds; i++) {
 548                 struct kfd_process_device *pdd = p->pdds[i];
 549
 550                 snprintf(stats_dir_filename, MAX_SYSFS_FILENAME_LEN,
 551                                 "stats_%u", pdd->dev->id);
 552                 pdd->kobj_stats = kfd_alloc_struct(pdd->kobj_stats);
 553                 if (!pdd->kobj_stats)
 554                         return;
 555
 556                 ret = kobject_init_and_add(pdd->kobj_stats,
 557                                            &procfs_stats_type,
 558                                            p->kobj,
 559                                            stats_dir_filename);
 560
 561                 if (ret) {
 562                         pr_warn("Creating KFD proc/stats_%s folder failed",
 563                                 stats_dir_filename);
 564                         kobject_put(pdd->kobj_stats);
 565                         pdd->kobj_stats = NULL;
 566                         return;
 567                 }
 568
 569                 kfd_sysfs_create_file(pdd->kobj_stats, &pdd->attr_evict,
 570                                       "evicted_ms");
 571                 /* Add sysfs file to report compute unit occupancy */
 572                 if (pdd->dev->kfd2kgd->get_cu_occupancy)
 573                         kfd_sysfs_create_file(pdd->kobj_stats,
 574                                               &pdd->attr_cu_occupancy,
 575                                               "cu_occupancy");
 576         }
 577 }
 578
 579 static void kfd_procfs_add_sysfs_counters(struct kfd_process *p)
 580 {
 581         int ret = 0;
 582         int i;
 583         char counters_dir_filename[MAX_SYSFS_FILENAME_LEN];
 584
 585         if (!p || !p->kobj)
 586                 return;
 587
 588         /*
 589          * Create sysfs files for each GPU which supports SVM
 590          * - proc/<pid>/counters_<gpuid>/
 591          * - proc/<pid>/counters_<gpuid>/faults
 592          * - proc/<pid>/counters_<gpuid>/page_in
 593          * - proc/<pid>/counters_<gpuid>/page_out
 594          */
 595         for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
 596                 struct kfd_process_device *pdd = p->pdds[i];
 597                 struct kobject *kobj_counters;
 598
 599                 snprintf(counters_dir_filename, MAX_SYSFS_FILENAME_LEN,
 600                         "counters_%u", pdd->dev->id);
 601                 kobj_counters = kfd_alloc_struct(kobj_counters);
 602                 if (!kobj_counters)
 603                         return;
 604
 605                 ret = kobject_init_and_add(kobj_counters, &sysfs_counters_type,
 606                                            p->kobj, counters_dir_filename);
 607                 if (ret) {
 608                         pr_warn("Creating KFD proc/%s folder failed",
 609                                 counters_dir_filename);
 610                         kobject_put(kobj_counters);
 611                         return;
 612                 }
 613
 614                 pdd->kobj_counters = kobj_counters;
 615                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_faults,
 616                                       "faults");
 617                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_in,
 618                                       "page_in");
 619                 kfd_sysfs_create_file(kobj_counters, &pdd->attr_page_out,
 620                                       "page_out");
 621         }
 622 }
 623
 624 static void kfd_procfs_add_sysfs_files(struct kfd_process *p)
 625 {
 626         int i;
 627
 628         if (!p || !p->kobj)
 629                 return;
 630
 631         /*
 632          * Create sysfs files for each GPU:
 633          * - proc/<pid>/vram_<gpuid>
 634          * - proc/<pid>/sdma_<gpuid>
 635          */
 636         for (i = 0; i < p->n_pdds; i++) {
 637                 struct kfd_process_device *pdd = p->pdds[i];
 638
 639                 snprintf(pdd->vram_filename, MAX_SYSFS_FILENAME_LEN, "vram_%u",
 640                          pdd->dev->id);
 641                 kfd_sysfs_create_file(p->kobj, &pdd->attr_vram,
 642                                       pdd->vram_filename);
 643
 644                 snprintf(pdd->sdma_filename, MAX_SYSFS_FILENAME_LEN, "sdma_%u",
 645                          pdd->dev->id);
 646                 kfd_sysfs_create_file(p->kobj, &pdd->attr_sdma,
 647                                             pdd->sdma_filename);
 648         }
 649 }
 650
 651 void kfd_procfs_del_queue(struct queue *q)
 652 {
 653         if (!q)
 654                 return;
 655
 656         kobject_del(&q->kobj);
 657         kobject_put(&q->kobj);
 658 }
 659
 660 int kfd_process_create_wq(void)
 661 {
 662         if (!kfd_process_wq)
 663                 kfd_process_wq = alloc_workqueue("kfd_process_wq", 0, 0);
 664         if (!kfd_restore_wq)
 665                 kfd_restore_wq = alloc_ordered_workqueue("kfd_restore_wq", 0);
 666
 667         if (!kfd_process_wq || !kfd_restore_wq) {
 668                 kfd_process_destroy_wq();
 669                 return -ENOMEM;
 670         }
 671
 672         return 0;
 673 }
 674
 675 void kfd_process_destroy_wq(void)
 676 {
 677         if (kfd_process_wq) {
 678                 destroy_workqueue(kfd_process_wq);
 679                 kfd_process_wq = NULL;
 680         }
 681         if (kfd_restore_wq) {
 682                 destroy_workqueue(kfd_restore_wq);
 683                 kfd_restore_wq = NULL;
 684         }
 685 }
 686
 687 static void kfd_process_free_gpuvm(struct kgd_mem *mem,
 688                         struct kfd_process_device *pdd)
 689 {
 690         struct kfd_dev *dev = pdd->dev;
 691
 692         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(dev->kgd, mem, pdd->drm_priv);
 693         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(dev->kgd, mem, pdd->drm_priv,
 694                                                NULL);
 695 }
 696
 697 /* kfd_process_alloc_gpuvm - Allocate GPU VM for the KFD process
 698  *      This function should be only called right after the process
 699  *      is created and when kfd_processes_mutex is still being held
 700  *      to avoid concurrency. Because of that exclusiveness, we do
 701  *      not need to take p->mutex.
 702  */
 703 static int kfd_process_alloc_gpuvm(struct kfd_process_device *pdd,
 704                                    uint64_t gpu_va, uint32_t size,
 705                                    uint32_t flags, void **kptr)
 706 {
 707         struct kfd_dev *kdev = pdd->dev;
 708         struct kgd_mem *mem = NULL;
 709         int handle;
 710         int err;
 711
 712         err = amdgpu_amdkfd_gpuvm_alloc_memory_of_gpu(kdev->kgd, gpu_va, size,
 713                                                  pdd->drm_priv, &mem, NULL, flags);
 714         if (err)
 715                 goto err_alloc_mem;
 716
 717         err = amdgpu_amdkfd_gpuvm_map_memory_to_gpu(kdev->kgd, mem,
 718                         pdd->drm_priv, NULL);
 719         if (err)
 720                 goto err_map_mem;
 721
 722         err = amdgpu_amdkfd_gpuvm_sync_memory(kdev->kgd, mem, true);
 723         if (err) {
 724                 pr_debug("Sync memory failed, wait interrupted by user signal\n");
 725                 goto sync_memory_failed;
 726         }
 727
 728         /* Create an obj handle so kfd_process_device_remove_obj_handle
 729          * will take care of the bo removal when the process finishes.
 730          * We do not need to take p->mutex, because the process is just
 731          * created and the ioctls have not had the chance to run.
 732          */
 733         handle = kfd_process_device_create_obj_handle(pdd, mem);
 734
 735         if (handle < 0) {
 736                 err = handle;
 737                 goto free_gpuvm;
 738         }
 739
 740         if (kptr) {
 741                 err = amdgpu_amdkfd_gpuvm_map_gtt_bo_to_kernel(kdev->kgd,
 742                                 (struct kgd_mem *)mem, kptr, NULL);
 743                 if (err) {
 744                         pr_debug("Map GTT BO to kernel failed\n");
 745                         goto free_obj_handle;
 746                 }
 747         }
 748
 749         return err;
 750
 751 free_obj_handle:
 752         kfd_process_device_remove_obj_handle(pdd, handle);
 753 free_gpuvm:
 754 sync_memory_failed:
 755         kfd_process_free_gpuvm(mem, pdd);
 756         return err;
 757
 758 err_map_mem:
 759         amdgpu_amdkfd_gpuvm_free_memory_of_gpu(kdev->kgd, mem, pdd->drm_priv,
 760                                                NULL);
 761 err_alloc_mem:
 762         *kptr = NULL;
 763         return err;
 764 }
 765
 766 /* kfd_process_device_reserve_ib_mem - Reserve memory inside the
 767  *      process for IB usage The memory reserved is for KFD to submit
 768  *      IB to AMDGPU from kernel.  If the memory is reserved
 769  *      successfully, ib_kaddr will have the CPU/kernel
 770  *      address. Check ib_kaddr before accessing the memory.
 771  */
 772 static int kfd_process_device_reserve_ib_mem(struct kfd_process_device *pdd)
 773 {
 774         struct qcm_process_device *qpd = &pdd->qpd;
 775         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT |
 776                         KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE |
 777                         KFD_IOC_ALLOC_MEM_FLAGS_WRITABLE |
 778                         KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
 779         void *kaddr;
 780         int ret;
 781
 782         if (qpd->ib_kaddr || !qpd->ib_base)
 783                 return 0;
 784
 785         /* ib_base is only set for dGPU */
 786         ret = kfd_process_alloc_gpuvm(pdd, qpd->ib_base, PAGE_SIZE, flags,
 787                                       &kaddr);
 788         if (ret)
 789                 return ret;
 790
 791         qpd->ib_kaddr = kaddr;
 792
 793         return 0;
 794 }
 795
 796 struct kfd_process *kfd_create_process(struct file *filep)
 797 {
 798         struct kfd_process *process;
 799         struct task_struct *thread = current;
 800         int ret;
 801
 802         if (!thread->mm)
 803                 return ERR_PTR(-EINVAL);
 804
 805         /* Only the pthreads threading model is supported. */
 806         if (thread->group_leader->mm != thread->mm)
 807                 return ERR_PTR(-EINVAL);
 808
 809         /*
 810          * take kfd processes mutex before starting of process creation
 811          * so there won't be a case where two threads of the same process
 812          * create two kfd_process structures
 813          */
 814         mutex_lock(&kfd_processes_mutex);
 815
 816         /* A prior open of /dev/kfd could have already created the process. */
 817         process = find_process(thread);
 818         if (process) {
 819                 pr_debug("Process already found\n");
 820         } else {
 821                 process = create_process(thread);
 822                 if (IS_ERR(process))
 823                         goto out;
 824
 825                 ret = kfd_process_init_cwsr_apu(process, filep);
 826                 if (ret)
 827                         goto out_destroy;
 828
 829                 if (!procfs.kobj)
 830                         goto out;
 831
 832                 process->kobj = kfd_alloc_struct(process->kobj);
 833                 if (!process->kobj) {
 834                         pr_warn("Creating procfs kobject failed");
 835                         goto out;
 836                 }
 837                 ret = kobject_init_and_add(process->kobj, &procfs_type,
 838                                            procfs.kobj, "%d",
 839                                            (int)process->lead_thread->pid);
 840                 if (ret) {
 841                         pr_warn("Creating procfs pid directory failed");
 842                         kobject_put(process->kobj);
 843                         goto out;
 844                 }
 845
 846                 kfd_sysfs_create_file(process->kobj, &process->attr_pasid,
 847                                       "pasid");
 848
 849                 process->kobj_queues = kobject_create_and_add("queues",
 850                                                         process->kobj);
 851                 if (!process->kobj_queues)
 852                         pr_warn("Creating KFD proc/queues folder failed");
 853
 854                 kfd_procfs_add_sysfs_stats(process);
 855                 kfd_procfs_add_sysfs_files(process);
 856                 kfd_procfs_add_sysfs_counters(process);
 857         }
 858 out:
 859         if (!IS_ERR(process))
 860                 kref_get(&process->ref);
 861         mutex_unlock(&kfd_processes_mutex);
 862
 863         return process;
 864
 865 out_destroy:
 866         hash_del_rcu(&process->kfd_processes);
 867         mutex_unlock(&kfd_processes_mutex);
 868         synchronize_srcu(&kfd_processes_srcu);
 869         /* kfd_process_free_notifier will trigger the cleanup */
 870         mmu_notifier_put(&process->mmu_notifier);
 871         return ERR_PTR(ret);
 872 }
 873
 874 struct kfd_process *kfd_get_process(const struct task_struct *thread)
 875 {
 876         struct kfd_process *process;
 877
 878         if (!thread->mm)
 879                 return ERR_PTR(-EINVAL);
 880
 881         /* Only the pthreads threading model is supported. */
 882         if (thread->group_leader->mm != thread->mm)
 883                 return ERR_PTR(-EINVAL);
 884
 885         process = find_process(thread);
 886         if (!process)
 887                 return ERR_PTR(-EINVAL);
 888
 889         return process;
 890 }
 891
 892 static struct kfd_process *find_process_by_mm(const struct mm_struct *mm)
 893 {
 894         struct kfd_process *process;
 895
 896         hash_for_each_possible_rcu(kfd_processes_table, process,
 897                                         kfd_processes, (uintptr_t)mm)
 898                 if (process->mm == mm)
 899                         return process;
 900
 901         return NULL;
 902 }
 903
 904 static struct kfd_process *find_process(const struct task_struct *thread)
 905 {
 906         struct kfd_process *p;
 907         int idx;
 908
 909         idx = srcu_read_lock(&kfd_processes_srcu);
 910         p = find_process_by_mm(thread->mm);
 911         srcu_read_unlock(&kfd_processes_srcu, idx);
 912
 913         return p;
 914 }
 915
 916 void kfd_unref_process(struct kfd_process *p)
 917 {
 918         kref_put(&p->ref, kfd_process_ref_release);
 919 }
 920
 921
 922 static void kfd_process_device_free_bos(struct kfd_process_device *pdd)
 923 {
 924         struct kfd_process *p = pdd->process;
 925         void *mem;
 926         int id;
 927         int i;
 928
 929         /*
 930          * Remove all handles from idr and release appropriate
 931          * local memory object
 932          */
 933         idr_for_each_entry(&pdd->alloc_idr, mem, id) {
 934
 935                 for (i = 0; i < p->n_pdds; i++) {
 936                         struct kfd_process_device *peer_pdd = p->pdds[i];
 937
 938                         if (!peer_pdd->drm_priv)
 939                                 continue;
 940                         amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
 941                                 peer_pdd->dev->kgd, mem, peer_pdd->drm_priv);
 942                 }
 943
 944                 amdgpu_amdkfd_gpuvm_free_memory_of_gpu(pdd->dev->kgd, mem,
 945                                                        pdd->drm_priv, NULL);
 946                 kfd_process_device_remove_obj_handle(pdd, id);
 947         }
 948 }
 949
 950 static void kfd_process_free_outstanding_kfd_bos(struct kfd_process *p)
 951 {
 952         int i;
 953
 954         for (i = 0; i < p->n_pdds; i++)
 955                 kfd_process_device_free_bos(p->pdds[i]);
 956 }
 957
 958 static void kfd_process_destroy_pdds(struct kfd_process *p)
 959 {
 960         int i;
 961
 962         for (i = 0; i < p->n_pdds; i++) {
 963                 struct kfd_process_device *pdd = p->pdds[i];
 964
 965                 pr_debug("Releasing pdd (topology id %d) for process (pasid 0x%x)\n",
 966                                 pdd->dev->id, p->pasid);
 967
 968                 if (pdd->drm_file) {
 969                         amdgpu_amdkfd_gpuvm_release_process_vm(
 970                                         pdd->dev->kgd, pdd->drm_priv);
 971                         fput(pdd->drm_file);
 972                 }
 973
 974                 if (pdd->qpd.cwsr_kaddr && !pdd->qpd.cwsr_base)
 975                         free_pages((unsigned long)pdd->qpd.cwsr_kaddr,
 976                                 get_order(KFD_CWSR_TBA_TMA_SIZE));
 977
 978                 kfree(pdd->qpd.doorbell_bitmap);
 979                 idr_destroy(&pdd->alloc_idr);
 980
 981                 kfd_free_process_doorbells(pdd->dev, pdd->doorbell_index);
 982
 983                 /*
 984                  * before destroying pdd, make sure to report availability
 985                  * for auto suspend
 986                  */
 987                 if (pdd->runtime_inuse) {
 988                         pm_runtime_mark_last_busy(pdd->dev->ddev->dev);
 989                         pm_runtime_put_autosuspend(pdd->dev->ddev->dev);
 990                         pdd->runtime_inuse = false;
 991                 }
 992
 993                 kfree(pdd);
 994                 p->pdds[i] = NULL;
 995         }
 996         p->n_pdds = 0;
 997 }
 998
 999 static void kfd_process_remove_sysfs(struct kfd_process *p)
1000 {
1001         struct kfd_process_device *pdd;
1002         int i;
1003
1004         if (!p->kobj)
1005                 return;
1006
1007         sysfs_remove_file(p->kobj, &p->attr_pasid);
1008         kobject_del(p->kobj_queues);
1009         kobject_put(p->kobj_queues);
1010         p->kobj_queues = NULL;
1011
1012         for (i = 0; i < p->n_pdds; i++) {
1013                 pdd = p->pdds[i];
1014
1015                 sysfs_remove_file(p->kobj, &pdd->attr_vram);
1016                 sysfs_remove_file(p->kobj, &pdd->attr_sdma);
1017
1018                 sysfs_remove_file(pdd->kobj_stats, &pdd->attr_evict);
1019                 if (pdd->dev->kfd2kgd->get_cu_occupancy)
1020                         sysfs_remove_file(pdd->kobj_stats,
1021                                           &pdd->attr_cu_occupancy);
1022                 kobject_del(pdd->kobj_stats);
1023                 kobject_put(pdd->kobj_stats);
1024                 pdd->kobj_stats = NULL;
1025         }
1026
1027         for_each_set_bit(i, p->svms.bitmap_supported, p->n_pdds) {
1028                 pdd = p->pdds[i];
1029
1030                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_faults);
1031                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_in);
1032                 sysfs_remove_file(pdd->kobj_counters, &pdd->attr_page_out);
1033                 kobject_del(pdd->kobj_counters);
1034                 kobject_put(pdd->kobj_counters);
1035                 pdd->kobj_counters = NULL;
1036         }
1037
1038         kobject_del(p->kobj);
1039         kobject_put(p->kobj);
1040         p->kobj = NULL;
1041 }
1042
1043 /* No process locking is needed in this function, because the process
1044  * is not findable any more. We must assume that no other thread is
1045  * using it any more, otherwise we couldn't safely free the process
1046  * structure in the end.
1047  */
1048 static void kfd_process_wq_release(struct work_struct *work)
1049 {
1050         struct kfd_process *p = container_of(work, struct kfd_process,
1051                                              release_work);
1052         kfd_process_remove_sysfs(p);
1053         kfd_iommu_unbind_process(p);
1054
1055         kfd_process_free_outstanding_kfd_bos(p);
1056         svm_range_list_fini(p);
1057
1058         kfd_process_destroy_pdds(p);
1059         dma_fence_put(p->ef);
1060
1061         kfd_event_free_process(p);
1062
1063         kfd_pasid_free(p->pasid);
1064         mutex_destroy(&p->mutex);
1065
1066         put_task_struct(p->lead_thread);
1067
1068         kfree(p);
1069 }
1070
1071 static void kfd_process_ref_release(struct kref *ref)
1072 {
1073         struct kfd_process *p = container_of(ref, struct kfd_process, ref);
1074
1075         INIT_WORK(&p->release_work, kfd_process_wq_release);
1076         queue_work(kfd_process_wq, &p->release_work);
1077 }
1078
1079 static struct mmu_notifier *kfd_process_alloc_notifier(struct mm_struct *mm)
1080 {
1081         int idx = srcu_read_lock(&kfd_processes_srcu);
1082         struct kfd_process *p = find_process_by_mm(mm);
1083
1084         srcu_read_unlock(&kfd_processes_srcu, idx);
1085
1086         return p ? &p->mmu_notifier : ERR_PTR(-ESRCH);
1087 }
1088
1089 static void kfd_process_free_notifier(struct mmu_notifier *mn)
1090 {
1091         kfd_unref_process(container_of(mn, struct kfd_process, mmu_notifier));
1092 }
1093
1094 static void kfd_process_notifier_release(struct mmu_notifier *mn,
1095                                         struct mm_struct *mm)
1096 {
1097         struct kfd_process *p;
1098         int i;
1099
1100         /*
1101          * The kfd_process structure can not be free because the
1102          * mmu_notifier srcu is read locked
1103          */
1104         p = container_of(mn, struct kfd_process, mmu_notifier);
1105         if (WARN_ON(p->mm != mm))
1106                 return;
1107
1108         mutex_lock(&kfd_processes_mutex);
1109         hash_del_rcu(&p->kfd_processes);
1110         mutex_unlock(&kfd_processes_mutex);
1111         synchronize_srcu(&kfd_processes_srcu);
1112
1113         cancel_delayed_work_sync(&p->eviction_work);
1114         cancel_delayed_work_sync(&p->restore_work);
1115         cancel_delayed_work_sync(&p->svms.restore_work);
1116
1117         mutex_lock(&p->mutex);
1118
1119         /* Iterate over all process device data structures and if the
1120          * pdd is in debug mode, we should first force unregistration,
1121          * then we will be able to destroy the queues
1122          */
1123         for (i = 0; i < p->n_pdds; i++) {
1124                 struct kfd_dev *dev = p->pdds[i]->dev;
1125
1126                 mutex_lock(kfd_get_dbgmgr_mutex());
1127                 if (dev && dev->dbgmgr && dev->dbgmgr->pasid == p->pasid) {
1128                         if (!kfd_dbgmgr_unregister(dev->dbgmgr, p)) {
1129                                 kfd_dbgmgr_destroy(dev->dbgmgr);
1130                                 dev->dbgmgr = NULL;
1131                         }
1132                 }
1133                 mutex_unlock(kfd_get_dbgmgr_mutex());
1134         }
1135
1136         kfd_process_dequeue_from_all_devices(p);
1137         pqm_uninit(&p->pqm);
1138
1139         /* Indicate to other users that MM is no longer valid */
1140         p->mm = NULL;
1141         /* Signal the eviction fence after user mode queues are
1142          * destroyed. This allows any BOs to be freed without
1143          * triggering pointless evictions or waiting for fences.
1144          */
1145         dma_fence_signal(p->ef);
1146
1147         mutex_unlock(&p->mutex);
1148
1149         mmu_notifier_put(&p->mmu_notifier);
1150 }
1151
1152 static const struct mmu_notifier_ops kfd_process_mmu_notifier_ops = {
1153         .release = kfd_process_notifier_release,
1154         .alloc_notifier = kfd_process_alloc_notifier,
1155         .free_notifier = kfd_process_free_notifier,
1156 };
1157
1158 static int kfd_process_init_cwsr_apu(struct kfd_process *p, struct file *filep)
1159 {
1160         unsigned long  offset;
1161         int i;
1162
1163         for (i = 0; i < p->n_pdds; i++) {
1164                 struct kfd_dev *dev = p->pdds[i]->dev;
1165                 struct qcm_process_device *qpd = &p->pdds[i]->qpd;
1166
1167                 if (!dev->cwsr_enabled || qpd->cwsr_kaddr || qpd->cwsr_base)
1168                         continue;
1169
1170                 offset = KFD_MMAP_TYPE_RESERVED_MEM | KFD_MMAP_GPU_ID(dev->id);
1171                 qpd->tba_addr = (int64_t)vm_mmap(filep, 0,
1172                         KFD_CWSR_TBA_TMA_SIZE, PROT_READ | PROT_EXEC,
1173                         MAP_SHARED, offset);
1174
1175                 if (IS_ERR_VALUE(qpd->tba_addr)) {
1176                         int err = qpd->tba_addr;
1177
1178                         pr_err("Failure to set tba address. error %d.\n", err);
1179                         qpd->tba_addr = 0;
1180                         qpd->cwsr_kaddr = NULL;
1181                         return err;
1182                 }
1183
1184                 memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1185
1186                 qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1187                 pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1188                         qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1189         }
1190
1191         return 0;
1192 }
1193
1194 static int kfd_process_device_init_cwsr_dgpu(struct kfd_process_device *pdd)
1195 {
1196         struct kfd_dev *dev = pdd->dev;
1197         struct qcm_process_device *qpd = &pdd->qpd;
1198         uint32_t flags = KFD_IOC_ALLOC_MEM_FLAGS_GTT
1199                         | KFD_IOC_ALLOC_MEM_FLAGS_NO_SUBSTITUTE
1200                         | KFD_IOC_ALLOC_MEM_FLAGS_EXECUTABLE;
1201         void *kaddr;
1202         int ret;
1203
1204         if (!dev->cwsr_enabled || qpd->cwsr_kaddr || !qpd->cwsr_base)
1205                 return 0;
1206
1207         /* cwsr_base is only set for dGPU */
1208         ret = kfd_process_alloc_gpuvm(pdd, qpd->cwsr_base,
1209                                       KFD_CWSR_TBA_TMA_SIZE, flags, &kaddr);
1210         if (ret)
1211                 return ret;
1212
1213         qpd->cwsr_kaddr = kaddr;
1214         qpd->tba_addr = qpd->cwsr_base;
1215
1216         memcpy(qpd->cwsr_kaddr, dev->cwsr_isa, dev->cwsr_isa_size);
1217
1218         qpd->tma_addr = qpd->tba_addr + KFD_CWSR_TMA_OFFSET;
1219         pr_debug("set tba :0x%llx, tma:0x%llx, cwsr_kaddr:%p for pqm.\n",
1220                  qpd->tba_addr, qpd->tma_addr, qpd->cwsr_kaddr);
1221
1222         return 0;
1223 }
1224
1225 void kfd_process_set_trap_handler(struct qcm_process_device *qpd,
1226                                   uint64_t tba_addr,
1227                                   uint64_t tma_addr)
1228 {
1229         if (qpd->cwsr_kaddr) {
1230                 /* KFD trap handler is bound, record as second-level TBA/TMA
1231                  * in first-level TMA. First-level trap will jump to second.
1232                  */
1233                 uint64_t *tma =
1234                         (uint64_t *)(qpd->cwsr_kaddr + KFD_CWSR_TMA_OFFSET);
1235                 tma[0] = tba_addr;
1236                 tma[1] = tma_addr;
1237         } else {
1238                 /* No trap handler bound, bind as first-level TBA/TMA. */
1239                 qpd->tba_addr = tba_addr;
1240                 qpd->tma_addr = tma_addr;
1241         }
1242 }
1243
1244 bool kfd_process_xnack_mode(struct kfd_process *p, bool supported)
1245 {
1246         int i;
1247
1248         /* On most GFXv9 GPUs, the retry mode in the SQ must match the
1249          * boot time retry setting. Mixing processes with different
1250          * XNACK/retry settings can hang the GPU.
1251          *
1252          * Different GPUs can have different noretry settings depending
1253          * on HW bugs or limitations. We need to find at least one
1254          * XNACK mode for this process that's compatible with all GPUs.
1255          * Fortunately GPUs with retry enabled (noretry=0) can run code
1256          * built for XNACK-off. On GFXv9 it may perform slower.
1257          *
1258          * Therefore applications built for XNACK-off can always be
1259          * supported and will be our fallback if any GPU does not
1260          * support retry.
1261          */
1262         for (i = 0; i < p->n_pdds; i++) {
1263                 struct kfd_dev *dev = p->pdds[i]->dev;
1264
1265                 /* Only consider GFXv9 and higher GPUs. Older GPUs don't
1266                  * support the SVM APIs and don't need to be considered
1267                  * for the XNACK mode selection.
1268                  */
1269                 if (dev->device_info->asic_family < CHIP_VEGA10)
1270                         continue;
1271                 /* Aldebaran can always support XNACK because it can support
1272                  * per-process XNACK mode selection. But let the dev->noretry
1273                  * setting still influence the default XNACK mode.
1274                  */
1275                 if (supported &&
1276                     dev->device_info->asic_family == CHIP_ALDEBARAN)
1277                         continue;
1278
1279                 /* GFXv10 and later GPUs do not support shader preemption
1280                  * during page faults. This can lead to poor QoS for queue
1281                  * management and memory-manager-related preemptions or
1282                  * even deadlocks.
1283                  */
1284                 if (dev->device_info->asic_family >= CHIP_NAVI10)
1285                         return false;
1286
1287                 if (dev->noretry)
1288                         return false;
1289         }
1290
1291         return true;
1292 }
1293
1294 /*
1295  * On return the kfd_process is fully operational and will be freed when the
1296  * mm is released
1297  */
1298 static struct kfd_process *create_process(const struct task_struct *thread)
1299 {
1300         struct kfd_process *process;
1301         struct mmu_notifier *mn;
1302         int err = -ENOMEM;
1303
1304         process = kzalloc(sizeof(*process), GFP_KERNEL);
1305         if (!process)
1306                 goto err_alloc_process;
1307
1308         kref_init(&process->ref);
1309         mutex_init(&process->mutex);
1310         process->mm = thread->mm;
1311         process->lead_thread = thread->group_leader;
1312         process->n_pdds = 0;
1313         INIT_DELAYED_WORK(&process->eviction_work, evict_process_worker);
1314         INIT_DELAYED_WORK(&process->restore_work, restore_process_worker);
1315         process->last_restore_timestamp = get_jiffies_64();
1316         kfd_event_init_process(process);
1317         process->is_32bit_user_mode = in_compat_syscall();
1318
1319         process->pasid = kfd_pasid_alloc();
1320         if (process->pasid == 0)
1321                 goto err_alloc_pasid;
1322
1323         err = pqm_init(&process->pqm, process);
1324         if (err != 0)
1325                 goto err_process_pqm_init;
1326
1327         /* init process apertures*/
1328         err = kfd_init_apertures(process);
1329         if (err != 0)
1330                 goto err_init_apertures;
1331
1332         /* Check XNACK support after PDDs are created in kfd_init_apertures */
1333         process->xnack_enabled = kfd_process_xnack_mode(process, false);
1334
1335         err = svm_range_list_init(process);
1336         if (err)
1337                 goto err_init_svm_range_list;
1338
1339         /* alloc_notifier needs to find the process in the hash table */
1340         hash_add_rcu(kfd_processes_table, &process->kfd_processes,
1341                         (uintptr_t)process->mm);
1342
1343         /* MMU notifier registration must be the last call that can fail
1344          * because after this point we cannot unwind the process creation.
1345          * After this point, mmu_notifier_put will trigger the cleanup by
1346          * dropping the last process reference in the free_notifier.
1347          */
1348         mn = mmu_notifier_get(&kfd_process_mmu_notifier_ops, process->mm);
1349         if (IS_ERR(mn)) {
1350                 err = PTR_ERR(mn);
1351                 goto err_register_notifier;
1352         }
1353         BUG_ON(mn != &process->mmu_notifier);
1354
1355         get_task_struct(process->lead_thread);
1356
1357         return process;
1358
1359 err_register_notifier:
1360         hash_del_rcu(&process->kfd_processes);
1361         svm_range_list_fini(process);
1362 err_init_svm_range_list:
1363         kfd_process_free_outstanding_kfd_bos(process);
1364         kfd_process_destroy_pdds(process);
1365 err_init_apertures:
1366         pqm_uninit(&process->pqm);
1367 err_process_pqm_init:
1368         kfd_pasid_free(process->pasid);
1369 err_alloc_pasid:
1370         mutex_destroy(&process->mutex);
1371         kfree(process);
1372 err_alloc_process:
1373         return ERR_PTR(err);
1374 }
1375
1376 static int init_doorbell_bitmap(struct qcm_process_device *qpd,
1377                         struct kfd_dev *dev)
1378 {
1379         unsigned int i;
1380         int range_start = dev->shared_resources.non_cp_doorbells_start;
1381         int range_end = dev->shared_resources.non_cp_doorbells_end;
1382
1383         if (!KFD_IS_SOC15(dev->device_info->asic_family))
1384                 return 0;
1385
1386         qpd->doorbell_bitmap =
1387                 kzalloc(DIV_ROUND_UP(KFD_MAX_NUM_OF_QUEUES_PER_PROCESS,
1388                                      BITS_PER_BYTE), GFP_KERNEL);
1389         if (!qpd->doorbell_bitmap)
1390                 return -ENOMEM;
1391
1392         /* Mask out doorbells reserved for SDMA, IH, and VCN on SOC15. */
1393         pr_debug("reserved doorbell 0x%03x - 0x%03x\n", range_start, range_end);
1394         pr_debug("reserved doorbell 0x%03x - 0x%03x\n",
1395                         range_start + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1396                         range_end + KFD_QUEUE_DOORBELL_MIRROR_OFFSET);
1397
1398         for (i = 0; i < KFD_MAX_NUM_OF_QUEUES_PER_PROCESS / 2; i++) {
1399                 if (i >= range_start && i <= range_end) {
1400                         set_bit(i, qpd->doorbell_bitmap);
1401                         set_bit(i + KFD_QUEUE_DOORBELL_MIRROR_OFFSET,
1402                                 qpd->doorbell_bitmap);
1403                 }
1404         }
1405
1406         return 0;
1407 }
1408
1409 struct kfd_process_device *kfd_get_process_device_data(struct kfd_dev *dev,
1410                                                         struct kfd_process *p)
1411 {
1412         int i;
1413
1414         for (i = 0; i < p->n_pdds; i++)
1415                 if (p->pdds[i]->dev == dev)
1416                         return p->pdds[i];
1417
1418         return NULL;
1419 }
1420
1421 struct kfd_process_device *kfd_create_process_device_data(struct kfd_dev *dev,
1422                                                         struct kfd_process *p)
1423 {
1424         struct kfd_process_device *pdd = NULL;
1425
1426         if (WARN_ON_ONCE(p->n_pdds >= MAX_GPU_INSTANCE))
1427                 return NULL;
1428         pdd = kzalloc(sizeof(*pdd), GFP_KERNEL);
1429         if (!pdd)
1430                 return NULL;
1431
1432         if (kfd_alloc_process_doorbells(dev, &pdd->doorbell_index) < 0) {
1433                 pr_err("Failed to alloc doorbell for pdd\n");
1434                 goto err_free_pdd;
1435         }
1436
1437         if (init_doorbell_bitmap(&pdd->qpd, dev)) {
1438                 pr_err("Failed to init doorbell for process\n");
1439                 goto err_free_pdd;
1440         }
1441
1442         pdd->dev = dev;
1443         INIT_LIST_HEAD(&pdd->qpd.queues_list);
1444         INIT_LIST_HEAD(&pdd->qpd.priv_queue_list);
1445         pdd->qpd.dqm = dev->dqm;
1446         pdd->qpd.pqm = &p->pqm;
1447         pdd->qpd.evicted = 0;
1448         pdd->qpd.mapped_gws_queue = false;
1449         pdd->process = p;
1450         pdd->bound = PDD_UNBOUND;
1451         pdd->already_dequeued = false;
1452         pdd->runtime_inuse = false;
1453         pdd->vram_usage = 0;
1454         pdd->sdma_past_activity_counter = 0;
1455         atomic64_set(&pdd->evict_duration_counter, 0);
1456         p->pdds[p->n_pdds++] = pdd;
1457
1458         /* Init idr used for memory handle translation */
1459         idr_init(&pdd->alloc_idr);
1460
1461         return pdd;
1462
1463 err_free_pdd:
1464         kfree(pdd);
1465         return NULL;
1466 }
1467
1468 /**
1469  * kfd_process_device_init_vm - Initialize a VM for a process-device
1470  *
1471  * @pdd: The process-device
1472  * @drm_file: Optional pointer to a DRM file descriptor
1473  *
1474  * If @drm_file is specified, it will be used to acquire the VM from
1475  * that file descriptor. If successful, the @pdd takes ownership of
1476  * the file descriptor.
1477  *
1478  * If @drm_file is NULL, a new VM is created.
1479  *
1480  * Returns 0 on success, -errno on failure.
1481  */
1482 int kfd_process_device_init_vm(struct kfd_process_device *pdd,
1483                                struct file *drm_file)
1484 {
1485         struct kfd_process *p;
1486         struct kfd_dev *dev;
1487         int ret;
1488
1489         if (!drm_file)
1490                 return -EINVAL;
1491
1492         if (pdd->drm_priv)
1493                 return -EBUSY;
1494
1495         p = pdd->process;
1496         dev = pdd->dev;
1497
1498         ret = amdgpu_amdkfd_gpuvm_acquire_process_vm(
1499                 dev->kgd, drm_file, p->pasid,
1500                 &p->kgd_process_info, &p->ef);
1501         if (ret) {
1502                 pr_err("Failed to create process VM object\n");
1503                 return ret;
1504         }
1505         pdd->drm_priv = drm_file->private_data;
1506
1507         ret = kfd_process_device_reserve_ib_mem(pdd);
1508         if (ret)
1509                 goto err_reserve_ib_mem;
1510         ret = kfd_process_device_init_cwsr_dgpu(pdd);
1511         if (ret)
1512                 goto err_init_cwsr;
1513
1514         pdd->drm_file = drm_file;
1515
1516         return 0;
1517
1518 err_init_cwsr:
1519 err_reserve_ib_mem:
1520         kfd_process_device_free_bos(pdd);
1521         pdd->drm_priv = NULL;
1522
1523         return ret;
1524 }
1525
1526 /*
1527  * Direct the IOMMU to bind the process (specifically the pasid->mm)
1528  * to the device.
1529  * Unbinding occurs when the process dies or the device is removed.
1530  *
1531  * Assumes that the process lock is held.
1532  */
1533 struct kfd_process_device *kfd_bind_process_to_device(struct kfd_dev *dev,
1534                                                         struct kfd_process *p)
1535 {
1536         struct kfd_process_device *pdd;
1537         int err;
1538
1539         pdd = kfd_get_process_device_data(dev, p);
1540         if (!pdd) {
1541                 pr_err("Process device data doesn't exist\n");
1542                 return ERR_PTR(-ENOMEM);
1543         }
1544
1545         if (!pdd->drm_priv)
1546                 return ERR_PTR(-ENODEV);
1547
1548         /*
1549          * signal runtime-pm system to auto resume and prevent
1550          * further runtime suspend once device pdd is created until
1551          * pdd is destroyed.
1552          */
1553         if (!pdd->runtime_inuse) {
1554                 err = pm_runtime_get_sync(dev->ddev->dev);
1555                 if (err < 0) {
1556                         pm_runtime_put_autosuspend(dev->ddev->dev);
1557                         return ERR_PTR(err);
1558                 }
1559         }
1560
1561         err = kfd_iommu_bind_process_to_device(pdd);
1562         if (err)
1563                 goto out;
1564
1565         /*
1566          * make sure that runtime_usage counter is incremented just once
1567          * per pdd
1568          */
1569         pdd->runtime_inuse = true;
1570
1571         return pdd;
1572
1573 out:
1574         /* balance runpm reference count and exit with error */
1575         if (!pdd->runtime_inuse) {
1576                 pm_runtime_mark_last_busy(dev->ddev->dev);
1577                 pm_runtime_put_autosuspend(dev->ddev->dev);
1578         }
1579
1580         return ERR_PTR(err);
1581 }
1582
1583 /* Create specific handle mapped to mem from process local memory idr
1584  * Assumes that the process lock is held.
1585  */
1586 int kfd_process_device_create_obj_handle(struct kfd_process_device *pdd,
1587                                         void *mem)
1588 {
1589         return idr_alloc(&pdd->alloc_idr, mem, 0, 0, GFP_KERNEL);
1590 }
1591
1592 /* Translate specific handle from process local memory idr
1593  * Assumes that the process lock is held.
1594  */
1595 void *kfd_process_device_translate_handle(struct kfd_process_device *pdd,
1596                                         int handle)
1597 {
1598         if (handle < 0)
1599                 return NULL;
1600
1601         return idr_find(&pdd->alloc_idr, handle);
1602 }
1603
1604 /* Remove specific handle from process local memory idr
1605  * Assumes that the process lock is held.
1606  */
1607 void kfd_process_device_remove_obj_handle(struct kfd_process_device *pdd,
1608                                         int handle)
1609 {
1610         if (handle >= 0)
1611                 idr_remove(&pdd->alloc_idr, handle);
1612 }
1613
1614 /* This increments the process->ref counter. */
1615 struct kfd_process *kfd_lookup_process_by_pasid(u32 pasid)
1616 {
1617         struct kfd_process *p, *ret_p = NULL;
1618         unsigned int temp;
1619
1620         int idx = srcu_read_lock(&kfd_processes_srcu);
1621
1622         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1623                 if (p->pasid == pasid) {
1624                         kref_get(&p->ref);
1625                         ret_p = p;
1626                         break;
1627                 }
1628         }
1629
1630         srcu_read_unlock(&kfd_processes_srcu, idx);
1631
1632         return ret_p;
1633 }
1634
1635 /* This increments the process->ref counter. */
1636 struct kfd_process *kfd_lookup_process_by_mm(const struct mm_struct *mm)
1637 {
1638         struct kfd_process *p;
1639
1640         int idx = srcu_read_lock(&kfd_processes_srcu);
1641
1642         p = find_process_by_mm(mm);
1643         if (p)
1644                 kref_get(&p->ref);
1645
1646         srcu_read_unlock(&kfd_processes_srcu, idx);
1647
1648         return p;
1649 }
1650
1651 /* kfd_process_evict_queues - Evict all user queues of a process
1652  *
1653  * Eviction is reference-counted per process-device. This means multiple
1654  * evictions from different sources can be nested safely.
1655  */
1656 int kfd_process_evict_queues(struct kfd_process *p)
1657 {
1658         int r = 0;
1659         int i;
1660         unsigned int n_evicted = 0;
1661
1662         for (i = 0; i < p->n_pdds; i++) {
1663                 struct kfd_process_device *pdd = p->pdds[i];
1664
1665                 r = pdd->dev->dqm->ops.evict_process_queues(pdd->dev->dqm,
1666                                                             &pdd->qpd);
1667                 if (r) {
1668                         pr_err("Failed to evict process queues\n");
1669                         goto fail;
1670                 }
1671                 n_evicted++;
1672         }
1673
1674         return r;
1675
1676 fail:
1677         /* To keep state consistent, roll back partial eviction by
1678          * restoring queues
1679          */
1680         for (i = 0; i < p->n_pdds; i++) {
1681                 struct kfd_process_device *pdd = p->pdds[i];
1682
1683                 if (n_evicted == 0)
1684                         break;
1685                 if (pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1686                                                               &pdd->qpd))
1687                         pr_err("Failed to restore queues\n");
1688
1689                 n_evicted--;
1690         }
1691
1692         return r;
1693 }
1694
1695 /* kfd_process_restore_queues - Restore all user queues of a process */
1696 int kfd_process_restore_queues(struct kfd_process *p)
1697 {
1698         int r, ret = 0;
1699         int i;
1700
1701         for (i = 0; i < p->n_pdds; i++) {
1702                 struct kfd_process_device *pdd = p->pdds[i];
1703
1704                 r = pdd->dev->dqm->ops.restore_process_queues(pdd->dev->dqm,
1705                                                               &pdd->qpd);
1706                 if (r) {
1707                         pr_err("Failed to restore process queues\n");
1708                         if (!ret)
1709                                 ret = r;
1710                 }
1711         }
1712
1713         return ret;
1714 }
1715
1716 int kfd_process_gpuidx_from_gpuid(struct kfd_process *p, uint32_t gpu_id)
1717 {
1718         int i;
1719
1720         for (i = 0; i < p->n_pdds; i++)
1721                 if (p->pdds[i] && gpu_id == p->pdds[i]->dev->id)
1722                         return i;
1723         return -EINVAL;
1724 }
1725
1726 int
1727 kfd_process_gpuid_from_kgd(struct kfd_process *p, struct amdgpu_device *adev,
1728                            uint32_t *gpuid, uint32_t *gpuidx)
1729 {
1730         struct kgd_dev *kgd = (struct kgd_dev *)adev;
1731         int i;
1732
1733         for (i = 0; i < p->n_pdds; i++)
1734                 if (p->pdds[i] && p->pdds[i]->dev->kgd == kgd) {
1735                         *gpuid = p->pdds[i]->dev->id;
1736                         *gpuidx = i;
1737                         return 0;
1738                 }
1739         return -EINVAL;
1740 }
1741
1742 static void evict_process_worker(struct work_struct *work)
1743 {
1744         int ret;
1745         struct kfd_process *p;
1746         struct delayed_work *dwork;
1747
1748         dwork = to_delayed_work(work);
1749
1750         /* Process termination destroys this worker thread. So during the
1751          * lifetime of this thread, kfd_process p will be valid
1752          */
1753         p = container_of(dwork, struct kfd_process, eviction_work);
1754         WARN_ONCE(p->last_eviction_seqno != p->ef->seqno,
1755                   "Eviction fence mismatch\n");
1756
1757         /* Narrow window of overlap between restore and evict work
1758          * item is possible. Once amdgpu_amdkfd_gpuvm_restore_process_bos
1759          * unreserves KFD BOs, it is possible to evicted again. But
1760          * restore has few more steps of finish. So lets wait for any
1761          * previous restore work to complete
1762          */
1763         flush_delayed_work(&p->restore_work);
1764
1765         pr_debug("Started evicting pasid 0x%x\n", p->pasid);
1766         ret = kfd_process_evict_queues(p);
1767         if (!ret) {
1768                 dma_fence_signal(p->ef);
1769                 dma_fence_put(p->ef);
1770                 p->ef = NULL;
1771                 queue_delayed_work(kfd_restore_wq, &p->restore_work,
1772                                 msecs_to_jiffies(PROCESS_RESTORE_TIME_MS));
1773
1774                 pr_debug("Finished evicting pasid 0x%x\n", p->pasid);
1775         } else
1776                 pr_err("Failed to evict queues of pasid 0x%x\n", p->pasid);
1777 }
1778
1779 static void restore_process_worker(struct work_struct *work)
1780 {
1781         struct delayed_work *dwork;
1782         struct kfd_process *p;
1783         int ret = 0;
1784
1785         dwork = to_delayed_work(work);
1786
1787         /* Process termination destroys this worker thread. So during the
1788          * lifetime of this thread, kfd_process p will be valid
1789          */
1790         p = container_of(dwork, struct kfd_process, restore_work);
1791         pr_debug("Started restoring pasid 0x%x\n", p->pasid);
1792
1793         /* Setting last_restore_timestamp before successful restoration.
1794          * Otherwise this would have to be set by KGD (restore_process_bos)
1795          * before KFD BOs are unreserved. If not, the process can be evicted
1796          * again before the timestamp is set.
1797          * If restore fails, the timestamp will be set again in the next
1798          * attempt. This would mean that the minimum GPU quanta would be
1799          * PROCESS_ACTIVE_TIME_MS - (time to execute the following two
1800          * functions)
1801          */
1802
1803         p->last_restore_timestamp = get_jiffies_64();
1804         ret = amdgpu_amdkfd_gpuvm_restore_process_bos(p->kgd_process_info,
1805                                                      &p->ef);
1806         if (ret) {
1807                 pr_debug("Failed to restore BOs of pasid 0x%x, retry after %d ms\n",
1808                          p->pasid, PROCESS_BACK_OFF_TIME_MS);
1809                 ret = queue_delayed_work(kfd_restore_wq, &p->restore_work,
1810                                 msecs_to_jiffies(PROCESS_BACK_OFF_TIME_MS));
1811                 WARN(!ret, "reschedule restore work failed\n");
1812                 return;
1813         }
1814
1815         ret = kfd_process_restore_queues(p);
1816         if (!ret)
1817                 pr_debug("Finished restoring pasid 0x%x\n", p->pasid);
1818         else
1819                 pr_err("Failed to restore queues of pasid 0x%x\n", p->pasid);
1820 }
1821
1822 void kfd_suspend_all_processes(void)
1823 {
1824         struct kfd_process *p;
1825         unsigned int temp;
1826         int idx = srcu_read_lock(&kfd_processes_srcu);
1827
1828         WARN(debug_evictions, "Evicting all processes");
1829         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1830                 cancel_delayed_work_sync(&p->eviction_work);
1831                 cancel_delayed_work_sync(&p->restore_work);
1832
1833                 if (kfd_process_evict_queues(p))
1834                         pr_err("Failed to suspend process 0x%x\n", p->pasid);
1835                 dma_fence_signal(p->ef);
1836                 dma_fence_put(p->ef);
1837                 p->ef = NULL;
1838         }
1839         srcu_read_unlock(&kfd_processes_srcu, idx);
1840 }
1841
1842 int kfd_resume_all_processes(void)
1843 {
1844         struct kfd_process *p;
1845         unsigned int temp;
1846         int ret = 0, idx = srcu_read_lock(&kfd_processes_srcu);
1847
1848         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1849                 if (!queue_delayed_work(kfd_restore_wq, &p->restore_work, 0)) {
1850                         pr_err("Restore process %d failed during resume\n",
1851                                p->pasid);
1852                         ret = -EFAULT;
1853                 }
1854         }
1855         srcu_read_unlock(&kfd_processes_srcu, idx);
1856         return ret;
1857 }
1858
1859 int kfd_reserved_mem_mmap(struct kfd_dev *dev, struct kfd_process *process,
1860                           struct vm_area_struct *vma)
1861 {
1862         struct kfd_process_device *pdd;
1863         struct qcm_process_device *qpd;
1864
1865         if ((vma->vm_end - vma->vm_start) != KFD_CWSR_TBA_TMA_SIZE) {
1866                 pr_err("Incorrect CWSR mapping size.\n");
1867                 return -EINVAL;
1868         }
1869
1870         pdd = kfd_get_process_device_data(dev, process);
1871         if (!pdd)
1872                 return -EINVAL;
1873         qpd = &pdd->qpd;
1874
1875         qpd->cwsr_kaddr = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
1876                                         get_order(KFD_CWSR_TBA_TMA_SIZE));
1877         if (!qpd->cwsr_kaddr) {
1878                 pr_err("Error allocating per process CWSR buffer.\n");
1879                 return -ENOMEM;
1880         }
1881
1882         vma->vm_flags |= VM_IO | VM_DONTCOPY | VM_DONTEXPAND
1883                 | VM_NORESERVE | VM_DONTDUMP | VM_PFNMAP;
1884         /* Mapping pages to user process */
1885         return remap_pfn_range(vma, vma->vm_start,
1886                                PFN_DOWN(__pa(qpd->cwsr_kaddr)),
1887                                KFD_CWSR_TBA_TMA_SIZE, vma->vm_page_prot);
1888 }
1889
1890 void kfd_flush_tlb(struct kfd_process_device *pdd, enum TLB_FLUSH_TYPE type)
1891 {
1892         struct kfd_dev *dev = pdd->dev;
1893
1894         if (dev->dqm->sched_policy == KFD_SCHED_POLICY_NO_HWS) {
1895                 /* Nothing to flush until a VMID is assigned, which
1896                  * only happens when the first queue is created.
1897                  */
1898                 if (pdd->qpd.vmid)
1899                         amdgpu_amdkfd_flush_gpu_tlb_vmid(dev->kgd,
1900                                                         pdd->qpd.vmid);
1901         } else {
1902                 amdgpu_amdkfd_flush_gpu_tlb_pasid(dev->kgd,
1903                                         pdd->process->pasid, type);
1904         }
1905 }
1906
1907 #if defined(CONFIG_DEBUG_FS)
1908
1909 int kfd_debugfs_mqds_by_process(struct seq_file *m, void *data)
1910 {
1911         struct kfd_process *p;
1912         unsigned int temp;
1913         int r = 0;
1914
1915         int idx = srcu_read_lock(&kfd_processes_srcu);
1916
1917         hash_for_each_rcu(kfd_processes_table, temp, p, kfd_processes) {
1918                 seq_printf(m, "Process %d PASID 0x%x:\n",
1919                            p->lead_thread->tgid, p->pasid);
1920
1921                 mutex_lock(&p->mutex);
1922                 r = pqm_debugfs_mqds(m, &p->pqm);
1923                 mutex_unlock(&p->mutex);
1924
1925                 if (r)
1926                         break;
1927         }
1928
1929         srcu_read_unlock(&kfd_processes_srcu, idx);
1930
1931         return r;
1932 }
1933
1934 #endif
1935