drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/poll.h>
  31 #include <drm/drm_debugfs.h>
  32
  33 #include "amdgpu.h"
  34 #include "amdgpu_pm.h"
  35 #include "amdgpu_dm_debugfs.h"
  36 #include "amdgpu_ras.h"
  37
  38 /**
  39  * amdgpu_debugfs_add_files - Add simple debugfs entries
  40  *
  41  * @adev:  Device to attach debugfs entries to
  42  * @files:  Array of function callbacks that respond to reads
  43  * @nfiles: Number of callbacks to register
  44  *
  45  */
  46 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
  47                              const struct drm_info_list *files,
  48                              unsigned nfiles)
  49 {
  50         unsigned i;
  51
  52         for (i = 0; i < adev->debugfs_count; i++) {
  53                 if (adev->debugfs[i].files == files) {
  54                         /* Already registered */
  55                         return 0;
  56                 }
  57         }
  58
  59         i = adev->debugfs_count + 1;
  60         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
  61                 DRM_ERROR("Reached maximum number of debugfs components.\n");
  62                 DRM_ERROR("Report so we increase "
  63                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
  64                 return -EINVAL;
  65         }
  66         adev->debugfs[adev->debugfs_count].files = files;
  67         adev->debugfs[adev->debugfs_count].num_files = nfiles;
  68         adev->debugfs_count = i;
  69 #if defined(CONFIG_DEBUG_FS)
  70         drm_debugfs_create_files(files, nfiles,
  71                                  adev->ddev->primary->debugfs_root,
  72                                  adev->ddev->primary);
  73 #endif
  74         return 0;
  75 }
  76
  77 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
  78 {
  79 #if defined(CONFIG_DEBUG_FS)
  80         unsigned long timeout = 600 * HZ;
  81         int ret;
  82
  83         wake_up_interruptible(&adev->autodump.gpu_hang);
  84
  85         ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
  86         if (ret == 0) {
  87                 pr_err("autodump: timeout, move on to gpu recovery\n");
  88                 return -ETIMEDOUT;
  89         }
  90 #endif
  91         return 0;
  92 }
  93
  94 #if defined(CONFIG_DEBUG_FS)
  95
  96 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
  97 {
  98         struct amdgpu_device *adev = inode->i_private;
  99         int ret;
 100
 101         file->private_data = adev;
 102
 103         mutex_lock(&adev->lock_reset);
 104         if (adev->autodump.dumping.done) {
 105                 reinit_completion(&adev->autodump.dumping);
 106                 ret = 0;
 107         } else {
 108                 ret = -EBUSY;
 109         }
 110         mutex_unlock(&adev->lock_reset);
 111
 112         return ret;
 113 }
 114
 115 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
 116 {
 117         struct amdgpu_device *adev = file->private_data;
 118
 119         complete_all(&adev->autodump.dumping);
 120         return 0;
 121 }
 122
 123 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
 124 {
 125         struct amdgpu_device *adev = file->private_data;
 126
 127         poll_wait(file, &adev->autodump.gpu_hang, poll_table);
 128
 129         if (adev->in_gpu_reset)
 130                 return POLLIN | POLLRDNORM | POLLWRNORM;
 131
 132         return 0;
 133 }
 134
 135 static const struct file_operations autodump_debug_fops = {
 136         .owner = THIS_MODULE,
 137         .open = amdgpu_debugfs_autodump_open,
 138         .poll = amdgpu_debugfs_autodump_poll,
 139         .release = amdgpu_debugfs_autodump_release,
 140 };
 141
 142 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
 143 {
 144         init_completion(&adev->autodump.dumping);
 145         complete_all(&adev->autodump.dumping);
 146         init_waitqueue_head(&adev->autodump.gpu_hang);
 147
 148         debugfs_create_file("amdgpu_autodump", 0600,
 149                 adev->ddev->primary->debugfs_root,
 150                 adev, &autodump_debug_fops);
 151 }
 152
 153 /**
 154  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
 155  *
 156  * @read: True if reading
 157  * @f: open file handle
 158  * @buf: User buffer to write/read to
 159  * @size: Number of bytes to write/read
 160  * @pos:  Offset to seek to
 161  *
 162  * This debugfs entry has special meaning on the offset being sought.
 163  * Various bits have different meanings:
 164  *
 165  * Bit 62:  Indicates a GRBM bank switch is needed
 166  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
 167  *                      zero)
 168  * Bits 24..33: The SE or ME selector if needed
 169  * Bits 34..43: The SH (or SA) or PIPE selector if needed
 170  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
 171  *
 172  * Bit 23:  Indicates that the PM power gating lock should be held
 173  *                      This is necessary to read registers that might be
 174  *                      unreliable during a power gating transistion.
 175  *
 176  * The lower bits are the BYTE offset of the register to read.  This
 177  * allows reading multiple registers in a single call and having
 178  * the returned size reflect that.
 179  */
 180 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 181                 char __user *buf, size_t size, loff_t *pos)
 182 {
 183         struct amdgpu_device *adev = file_inode(f)->i_private;
 184         ssize_t result = 0;
 185         int r;
 186         bool pm_pg_lock, use_bank, use_ring;
 187         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
 188
 189         pm_pg_lock = use_bank = use_ring = false;
 190         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
 191
 192         if (size & 0x3 || *pos & 0x3 ||
 193                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
 194                 return -EINVAL;
 195
 196         /* are we reading registers for which a PG lock is necessary? */
 197         pm_pg_lock = (*pos >> 23) & 1;
 198
 199         if (*pos & (1ULL << 62)) {
 200                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
 201                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
 202                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 203
 204                 if (se_bank == 0x3FF)
 205                         se_bank = 0xFFFFFFFF;
 206                 if (sh_bank == 0x3FF)
 207                         sh_bank = 0xFFFFFFFF;
 208                 if (instance_bank == 0x3FF)
 209                         instance_bank = 0xFFFFFFFF;
 210                 use_bank = true;
 211         } else if (*pos & (1ULL << 61)) {
 212
 213                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 214                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 215                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 216                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 217
 218                 use_ring = true;
 219         } else {
 220                 use_bank = use_ring = false;
 221         }
 222
 223         *pos &= (1UL << 22) - 1;
 224
 225         r = pm_runtime_get_sync(adev->ddev->dev);
 226         if (r < 0) {
 227                 pm_runtime_put_autosuspend(adev->ddev->dev);
 228                 return r;
 229         }
 230
 231         r = amdgpu_virt_enable_access_debugfs(adev);
 232         if (r < 0) {
 233                 pm_runtime_put_autosuspend(adev->ddev->dev);
 234                 return r;
 235         }
 236
 237         if (use_bank) {
 238                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 239                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 240                         pm_runtime_mark_last_busy(adev->ddev->dev);
 241                         pm_runtime_put_autosuspend(adev->ddev->dev);
 242                         amdgpu_virt_disable_access_debugfs(adev);
 243                         return -EINVAL;
 244                 }
 245                 mutex_lock(&adev->grbm_idx_mutex);
 246                 amdgpu_gfx_select_se_sh(adev, se_bank,
 247                                         sh_bank, instance_bank);
 248         } else if (use_ring) {
 249                 mutex_lock(&adev->srbm_mutex);
 250                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 251         }
 252
 253         if (pm_pg_lock)
 254                 mutex_lock(&adev->pm.mutex);
 255
 256         while (size) {
 257                 uint32_t value;
 258
 259                 if (read) {
 260                         value = RREG32(*pos >> 2);
 261                         r = put_user(value, (uint32_t *)buf);
 262                 } else {
 263                         r = get_user(value, (uint32_t *)buf);
 264                         if (!r)
 265                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
 266                 }
 267                 if (r) {
 268                         result = r;
 269                         goto end;
 270                 }
 271
 272                 result += 4;
 273                 buf += 4;
 274                 *pos += 4;
 275                 size -= 4;
 276         }
 277
 278 end:
 279         if (use_bank) {
 280                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 281                 mutex_unlock(&adev->grbm_idx_mutex);
 282         } else if (use_ring) {
 283                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 284                 mutex_unlock(&adev->srbm_mutex);
 285         }
 286
 287         if (pm_pg_lock)
 288                 mutex_unlock(&adev->pm.mutex);
 289
 290         pm_runtime_mark_last_busy(adev->ddev->dev);
 291         pm_runtime_put_autosuspend(adev->ddev->dev);
 292
 293         amdgpu_virt_disable_access_debugfs(adev);
 294         return result;
 295 }
 296
 297 /**
 298  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 299  */
 300 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 301                                         size_t size, loff_t *pos)
 302 {
 303         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 304 }
 305
 306 /**
 307  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 308  */
 309 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 310                                          size_t size, loff_t *pos)
 311 {
 312         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 313 }
 314
 315
 316 /**
 317  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 318  *
 319  * @f: open file handle
 320  * @buf: User buffer to store read data in
 321  * @size: Number of bytes to read
 322  * @pos:  Offset to seek to
 323  *
 324  * The lower bits are the BYTE offset of the register to read.  This
 325  * allows reading multiple registers in a single call and having
 326  * the returned size reflect that.
 327  */
 328 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 329                                         size_t size, loff_t *pos)
 330 {
 331         struct amdgpu_device *adev = file_inode(f)->i_private;
 332         ssize_t result = 0;
 333         int r;
 334
 335         if (size & 0x3 || *pos & 0x3)
 336                 return -EINVAL;
 337
 338         r = pm_runtime_get_sync(adev->ddev->dev);
 339         if (r < 0) {
 340                 pm_runtime_put_autosuspend(adev->ddev->dev);
 341                 return r;
 342         }
 343
 344         r = amdgpu_virt_enable_access_debugfs(adev);
 345         if (r < 0) {
 346                 pm_runtime_put_autosuspend(adev->ddev->dev);
 347                 return r;
 348         }
 349
 350         while (size) {
 351                 uint32_t value;
 352
 353                 value = RREG32_PCIE(*pos >> 2);
 354                 r = put_user(value, (uint32_t *)buf);
 355                 if (r) {
 356                         pm_runtime_mark_last_busy(adev->ddev->dev);
 357                         pm_runtime_put_autosuspend(adev->ddev->dev);
 358                         amdgpu_virt_disable_access_debugfs(adev);
 359                         return r;
 360                 }
 361
 362                 result += 4;
 363                 buf += 4;
 364                 *pos += 4;
 365                 size -= 4;
 366         }
 367
 368         pm_runtime_mark_last_busy(adev->ddev->dev);
 369         pm_runtime_put_autosuspend(adev->ddev->dev);
 370
 371         amdgpu_virt_disable_access_debugfs(adev);
 372         return result;
 373 }
 374
 375 /**
 376  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 377  *
 378  * @f: open file handle
 379  * @buf: User buffer to write data from
 380  * @size: Number of bytes to write
 381  * @pos:  Offset to seek to
 382  *
 383  * The lower bits are the BYTE offset of the register to write.  This
 384  * allows writing multiple registers in a single call and having
 385  * the returned size reflect that.
 386  */
 387 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 388                                          size_t size, loff_t *pos)
 389 {
 390         struct amdgpu_device *adev = file_inode(f)->i_private;
 391         ssize_t result = 0;
 392         int r;
 393
 394         if (size & 0x3 || *pos & 0x3)
 395                 return -EINVAL;
 396
 397         r = pm_runtime_get_sync(adev->ddev->dev);
 398         if (r < 0) {
 399                 pm_runtime_put_autosuspend(adev->ddev->dev);
 400                 return r;
 401         }
 402
 403         r = amdgpu_virt_enable_access_debugfs(adev);
 404         if (r < 0) {
 405                 pm_runtime_put_autosuspend(adev->ddev->dev);
 406                 return r;
 407         }
 408
 409         while (size) {
 410                 uint32_t value;
 411
 412                 r = get_user(value, (uint32_t *)buf);
 413                 if (r) {
 414                         pm_runtime_mark_last_busy(adev->ddev->dev);
 415                         pm_runtime_put_autosuspend(adev->ddev->dev);
 416                         amdgpu_virt_disable_access_debugfs(adev);
 417                         return r;
 418                 }
 419
 420                 WREG32_PCIE(*pos >> 2, value);
 421
 422                 result += 4;
 423                 buf += 4;
 424                 *pos += 4;
 425                 size -= 4;
 426         }
 427
 428         pm_runtime_mark_last_busy(adev->ddev->dev);
 429         pm_runtime_put_autosuspend(adev->ddev->dev);
 430
 431         amdgpu_virt_disable_access_debugfs(adev);
 432         return result;
 433 }
 434
 435 /**
 436  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 437  *
 438  * @f: open file handle
 439  * @buf: User buffer to store read data in
 440  * @size: Number of bytes to read
 441  * @pos:  Offset to seek to
 442  *
 443  * The lower bits are the BYTE offset of the register to read.  This
 444  * allows reading multiple registers in a single call and having
 445  * the returned size reflect that.
 446  */
 447 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 448                                         size_t size, loff_t *pos)
 449 {
 450         struct amdgpu_device *adev = file_inode(f)->i_private;
 451         ssize_t result = 0;
 452         int r;
 453
 454         if (size & 0x3 || *pos & 0x3)
 455                 return -EINVAL;
 456
 457         r = pm_runtime_get_sync(adev->ddev->dev);
 458         if (r < 0) {
 459                 pm_runtime_put_autosuspend(adev->ddev->dev);
 460                 return r;
 461         }
 462
 463         r = amdgpu_virt_enable_access_debugfs(adev);
 464         if (r < 0) {
 465                 pm_runtime_put_autosuspend(adev->ddev->dev);
 466                 return r;
 467         }
 468
 469         while (size) {
 470                 uint32_t value;
 471
 472                 value = RREG32_DIDT(*pos >> 2);
 473                 r = put_user(value, (uint32_t *)buf);
 474                 if (r) {
 475                         pm_runtime_mark_last_busy(adev->ddev->dev);
 476                         pm_runtime_put_autosuspend(adev->ddev->dev);
 477                         amdgpu_virt_disable_access_debugfs(adev);
 478                         return r;
 479                 }
 480
 481                 result += 4;
 482                 buf += 4;
 483                 *pos += 4;
 484                 size -= 4;
 485         }
 486
 487         pm_runtime_mark_last_busy(adev->ddev->dev);
 488         pm_runtime_put_autosuspend(adev->ddev->dev);
 489
 490         amdgpu_virt_disable_access_debugfs(adev);
 491         return result;
 492 }
 493
 494 /**
 495  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 496  *
 497  * @f: open file handle
 498  * @buf: User buffer to write data from
 499  * @size: Number of bytes to write
 500  * @pos:  Offset to seek to
 501  *
 502  * The lower bits are the BYTE offset of the register to write.  This
 503  * allows writing multiple registers in a single call and having
 504  * the returned size reflect that.
 505  */
 506 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 507                                          size_t size, loff_t *pos)
 508 {
 509         struct amdgpu_device *adev = file_inode(f)->i_private;
 510         ssize_t result = 0;
 511         int r;
 512
 513         if (size & 0x3 || *pos & 0x3)
 514                 return -EINVAL;
 515
 516         r = pm_runtime_get_sync(adev->ddev->dev);
 517         if (r < 0) {
 518                 pm_runtime_put_autosuspend(adev->ddev->dev);
 519                 return r;
 520         }
 521
 522         r = amdgpu_virt_enable_access_debugfs(adev);
 523         if (r < 0) {
 524                 pm_runtime_put_autosuspend(adev->ddev->dev);
 525                 return r;
 526         }
 527
 528         while (size) {
 529                 uint32_t value;
 530
 531                 r = get_user(value, (uint32_t *)buf);
 532                 if (r) {
 533                         pm_runtime_mark_last_busy(adev->ddev->dev);
 534                         pm_runtime_put_autosuspend(adev->ddev->dev);
 535                         amdgpu_virt_disable_access_debugfs(adev);
 536                         return r;
 537                 }
 538
 539                 WREG32_DIDT(*pos >> 2, value);
 540
 541                 result += 4;
 542                 buf += 4;
 543                 *pos += 4;
 544                 size -= 4;
 545         }
 546
 547         pm_runtime_mark_last_busy(adev->ddev->dev);
 548         pm_runtime_put_autosuspend(adev->ddev->dev);
 549
 550         amdgpu_virt_disable_access_debugfs(adev);
 551         return result;
 552 }
 553
 554 /**
 555  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 556  *
 557  * @f: open file handle
 558  * @buf: User buffer to store read data in
 559  * @size: Number of bytes to read
 560  * @pos:  Offset to seek to
 561  *
 562  * The lower bits are the BYTE offset of the register to read.  This
 563  * allows reading multiple registers in a single call and having
 564  * the returned size reflect that.
 565  */
 566 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 567                                         size_t size, loff_t *pos)
 568 {
 569         struct amdgpu_device *adev = file_inode(f)->i_private;
 570         ssize_t result = 0;
 571         int r;
 572
 573         if (size & 0x3 || *pos & 0x3)
 574                 return -EINVAL;
 575
 576         r = pm_runtime_get_sync(adev->ddev->dev);
 577         if (r < 0) {
 578                 pm_runtime_put_autosuspend(adev->ddev->dev);
 579                 return r;
 580         }
 581
 582         r = amdgpu_virt_enable_access_debugfs(adev);
 583         if (r < 0) {
 584                 pm_runtime_put_autosuspend(adev->ddev->dev);
 585                 return r;
 586         }
 587
 588         while (size) {
 589                 uint32_t value;
 590
 591                 value = RREG32_SMC(*pos);
 592                 r = put_user(value, (uint32_t *)buf);
 593                 if (r) {
 594                         pm_runtime_mark_last_busy(adev->ddev->dev);
 595                         pm_runtime_put_autosuspend(adev->ddev->dev);
 596                         amdgpu_virt_disable_access_debugfs(adev);
 597                         return r;
 598                 }
 599
 600                 result += 4;
 601                 buf += 4;
 602                 *pos += 4;
 603                 size -= 4;
 604         }
 605
 606         pm_runtime_mark_last_busy(adev->ddev->dev);
 607         pm_runtime_put_autosuspend(adev->ddev->dev);
 608
 609         amdgpu_virt_disable_access_debugfs(adev);
 610         return result;
 611 }
 612
 613 /**
 614  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 615  *
 616  * @f: open file handle
 617  * @buf: User buffer to write data from
 618  * @size: Number of bytes to write
 619  * @pos:  Offset to seek to
 620  *
 621  * The lower bits are the BYTE offset of the register to write.  This
 622  * allows writing multiple registers in a single call and having
 623  * the returned size reflect that.
 624  */
 625 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 626                                          size_t size, loff_t *pos)
 627 {
 628         struct amdgpu_device *adev = file_inode(f)->i_private;
 629         ssize_t result = 0;
 630         int r;
 631
 632         if (size & 0x3 || *pos & 0x3)
 633                 return -EINVAL;
 634
 635         r = pm_runtime_get_sync(adev->ddev->dev);
 636         if (r < 0) {
 637                 pm_runtime_put_autosuspend(adev->ddev->dev);
 638                 return r;
 639         }
 640
 641         r = amdgpu_virt_enable_access_debugfs(adev);
 642         if (r < 0) {
 643                 pm_runtime_put_autosuspend(adev->ddev->dev);
 644                 return r;
 645         }
 646
 647         while (size) {
 648                 uint32_t value;
 649
 650                 r = get_user(value, (uint32_t *)buf);
 651                 if (r) {
 652                         pm_runtime_mark_last_busy(adev->ddev->dev);
 653                         pm_runtime_put_autosuspend(adev->ddev->dev);
 654                         amdgpu_virt_disable_access_debugfs(adev);
 655                         return r;
 656                 }
 657
 658                 WREG32_SMC(*pos, value);
 659
 660                 result += 4;
 661                 buf += 4;
 662                 *pos += 4;
 663                 size -= 4;
 664         }
 665
 666         pm_runtime_mark_last_busy(adev->ddev->dev);
 667         pm_runtime_put_autosuspend(adev->ddev->dev);
 668
 669         amdgpu_virt_disable_access_debugfs(adev);
 670         return result;
 671 }
 672
 673 /**
 674  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 675  *
 676  * @f: open file handle
 677  * @buf: User buffer to store read data in
 678  * @size: Number of bytes to read
 679  * @pos:  Offset to seek to
 680  *
 681  * This file is used to access configuration data in a somewhat
 682  * stable fashion.  The format is a series of DWORDs with the first
 683  * indicating which revision it is.  New content is appended to the
 684  * end so that older software can still read the data.
 685  */
 686
 687 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 688                                         size_t size, loff_t *pos)
 689 {
 690         struct amdgpu_device *adev = file_inode(f)->i_private;
 691         ssize_t result = 0;
 692         int r;
 693         uint32_t *config, no_regs = 0;
 694
 695         if (size & 0x3 || *pos & 0x3)
 696                 return -EINVAL;
 697
 698         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 699         if (!config)
 700                 return -ENOMEM;
 701
 702         /* version, increment each time something is added */
 703         config[no_regs++] = 3;
 704         config[no_regs++] = adev->gfx.config.max_shader_engines;
 705         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 706         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 707         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 708         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 709         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 710         config[no_regs++] = adev->gfx.config.max_gprs;
 711         config[no_regs++] = adev->gfx.config.max_gs_threads;
 712         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 713         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 714         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 715         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 716         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 717         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 718         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 719         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 720         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 721         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 722         config[no_regs++] = adev->gfx.config.num_gpus;
 723         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 724         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 725         config[no_regs++] = adev->gfx.config.gb_addr_config;
 726         config[no_regs++] = adev->gfx.config.num_rbs;
 727
 728         /* rev==1 */
 729         config[no_regs++] = adev->rev_id;
 730         config[no_regs++] = adev->pg_flags;
 731         config[no_regs++] = adev->cg_flags;
 732
 733         /* rev==2 */
 734         config[no_regs++] = adev->family;
 735         config[no_regs++] = adev->external_rev_id;
 736
 737         /* rev==3 */
 738         config[no_regs++] = adev->pdev->device;
 739         config[no_regs++] = adev->pdev->revision;
 740         config[no_regs++] = adev->pdev->subsystem_device;
 741         config[no_regs++] = adev->pdev->subsystem_vendor;
 742
 743         while (size && (*pos < no_regs * 4)) {
 744                 uint32_t value;
 745
 746                 value = config[*pos >> 2];
 747                 r = put_user(value, (uint32_t *)buf);
 748                 if (r) {
 749                         kfree(config);
 750                         return r;
 751                 }
 752
 753                 result += 4;
 754                 buf += 4;
 755                 *pos += 4;
 756                 size -= 4;
 757         }
 758
 759         kfree(config);
 760         return result;
 761 }
 762
 763 /**
 764  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 765  *
 766  * @f: open file handle
 767  * @buf: User buffer to store read data in
 768  * @size: Number of bytes to read
 769  * @pos:  Offset to seek to
 770  *
 771  * The offset is treated as the BYTE address of one of the sensors
 772  * enumerated in amd/include/kgd_pp_interface.h under the
 773  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 774  * you would use the offset 3 * 4 = 12.
 775  */
 776 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 777                                         size_t size, loff_t *pos)
 778 {
 779         struct amdgpu_device *adev = file_inode(f)->i_private;
 780         int idx, x, outsize, r, valuesize;
 781         uint32_t values[16];
 782
 783         if (size & 3 || *pos & 0x3)
 784                 return -EINVAL;
 785
 786         if (!adev->pm.dpm_enabled)
 787                 return -EINVAL;
 788
 789         /* convert offset to sensor number */
 790         idx = *pos >> 2;
 791
 792         valuesize = sizeof(values);
 793
 794         r = pm_runtime_get_sync(adev->ddev->dev);
 795         if (r < 0) {
 796                 pm_runtime_put_autosuspend(adev->ddev->dev);
 797                 return r;
 798         }
 799
 800         r = amdgpu_virt_enable_access_debugfs(adev);
 801         if (r < 0) {
 802                 pm_runtime_put_autosuspend(adev->ddev->dev);
 803                 return r;
 804         }
 805
 806         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 807
 808         pm_runtime_mark_last_busy(adev->ddev->dev);
 809         pm_runtime_put_autosuspend(adev->ddev->dev);
 810
 811         if (r) {
 812                 amdgpu_virt_disable_access_debugfs(adev);
 813                 return r;
 814         }
 815
 816         if (size > valuesize) {
 817                 amdgpu_virt_disable_access_debugfs(adev);
 818                 return -EINVAL;
 819         }
 820
 821         outsize = 0;
 822         x = 0;
 823         if (!r) {
 824                 while (size) {
 825                         r = put_user(values[x++], (int32_t *)buf);
 826                         buf += 4;
 827                         size -= 4;
 828                         outsize += 4;
 829                 }
 830         }
 831
 832         amdgpu_virt_disable_access_debugfs(adev);
 833         return !r ? outsize : r;
 834 }
 835
 836 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 837  *
 838  * @f: open file handle
 839  * @buf: User buffer to store read data in
 840  * @size: Number of bytes to read
 841  * @pos:  Offset to seek to
 842  *
 843  * The offset being sought changes which wave that the status data
 844  * will be returned for.  The bits are used as follows:
 845  *
 846  * Bits 0..6:   Byte offset into data
 847  * Bits 7..14:  SE selector
 848  * Bits 15..22: SH/SA selector
 849  * Bits 23..30: CU/{WGP+SIMD} selector
 850  * Bits 31..36: WAVE ID selector
 851  * Bits 37..44: SIMD ID selector
 852  *
 853  * The returned data begins with one DWORD of version information
 854  * Followed by WAVE STATUS registers relevant to the GFX IP version
 855  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 856  */
 857 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 858                                         size_t size, loff_t *pos)
 859 {
 860         struct amdgpu_device *adev = f->f_inode->i_private;
 861         int r, x;
 862         ssize_t result=0;
 863         uint32_t offset, se, sh, cu, wave, simd, data[32];
 864
 865         if (size & 3 || *pos & 3)
 866                 return -EINVAL;
 867
 868         /* decode offset */
 869         offset = (*pos & GENMASK_ULL(6, 0));
 870         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 871         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 872         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 873         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 874         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 875
 876         r = pm_runtime_get_sync(adev->ddev->dev);
 877         if (r < 0) {
 878                 pm_runtime_put_autosuspend(adev->ddev->dev);
 879                 return r;
 880         }
 881
 882         r = amdgpu_virt_enable_access_debugfs(adev);
 883         if (r < 0) {
 884                 pm_runtime_put_autosuspend(adev->ddev->dev);
 885                 return r;
 886         }
 887
 888         /* switch to the specific se/sh/cu */
 889         mutex_lock(&adev->grbm_idx_mutex);
 890         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 891
 892         x = 0;
 893         if (adev->gfx.funcs->read_wave_data)
 894                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 895
 896         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 897         mutex_unlock(&adev->grbm_idx_mutex);
 898
 899         pm_runtime_mark_last_busy(adev->ddev->dev);
 900         pm_runtime_put_autosuspend(adev->ddev->dev);
 901
 902         if (!x) {
 903                 amdgpu_virt_disable_access_debugfs(adev);
 904                 return -EINVAL;
 905         }
 906
 907         while (size && (offset < x * 4)) {
 908                 uint32_t value;
 909
 910                 value = data[offset >> 2];
 911                 r = put_user(value, (uint32_t *)buf);
 912                 if (r) {
 913                         amdgpu_virt_disable_access_debugfs(adev);
 914                         return r;
 915                 }
 916
 917                 result += 4;
 918                 buf += 4;
 919                 offset += 4;
 920                 size -= 4;
 921         }
 922
 923         amdgpu_virt_disable_access_debugfs(adev);
 924         return result;
 925 }
 926
 927 /** amdgpu_debugfs_gpr_read - Read wave gprs
 928  *
 929  * @f: open file handle
 930  * @buf: User buffer to store read data in
 931  * @size: Number of bytes to read
 932  * @pos:  Offset to seek to
 933  *
 934  * The offset being sought changes which wave that the status data
 935  * will be returned for.  The bits are used as follows:
 936  *
 937  * Bits 0..11:  Byte offset into data
 938  * Bits 12..19: SE selector
 939  * Bits 20..27: SH/SA selector
 940  * Bits 28..35: CU/{WGP+SIMD} selector
 941  * Bits 36..43: WAVE ID selector
 942  * Bits 37..44: SIMD ID selector
 943  * Bits 52..59: Thread selector
 944  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 945  *
 946  * The return data comes from the SGPR or VGPR register bank for
 947  * the selected operational unit.
 948  */
 949 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 950                                         size_t size, loff_t *pos)
 951 {
 952         struct amdgpu_device *adev = f->f_inode->i_private;
 953         int r;
 954         ssize_t result = 0;
 955         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 956
 957         if (size > 4096 || size & 3 || *pos & 3)
 958                 return -EINVAL;
 959
 960         /* decode offset */
 961         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 962         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 963         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 964         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 965         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 966         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
 967         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
 968         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 969
 970         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 971         if (!data)
 972                 return -ENOMEM;
 973
 974         r = pm_runtime_get_sync(adev->ddev->dev);
 975         if (r < 0)
 976                 goto err;
 977
 978         r = amdgpu_virt_enable_access_debugfs(adev);
 979         if (r < 0)
 980                 goto err;
 981
 982         /* switch to the specific se/sh/cu */
 983         mutex_lock(&adev->grbm_idx_mutex);
 984         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 985
 986         if (bank == 0) {
 987                 if (adev->gfx.funcs->read_wave_vgprs)
 988                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
 989         } else {
 990                 if (adev->gfx.funcs->read_wave_sgprs)
 991                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
 992         }
 993
 994         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 995         mutex_unlock(&adev->grbm_idx_mutex);
 996
 997         pm_runtime_mark_last_busy(adev->ddev->dev);
 998         pm_runtime_put_autosuspend(adev->ddev->dev);
 999
1000         while (size) {
1001                 uint32_t value;
1002
1003                 value = data[result >> 2];
1004                 r = put_user(value, (uint32_t *)buf);
1005                 if (r) {
1006                         amdgpu_virt_disable_access_debugfs(adev);
1007                         goto err;
1008                 }
1009
1010                 result += 4;
1011                 buf += 4;
1012                 size -= 4;
1013         }
1014
1015         kfree(data);
1016         amdgpu_virt_disable_access_debugfs(adev);
1017         return result;
1018
1019 err:
1020         pm_runtime_put_autosuspend(adev->ddev->dev);
1021         kfree(data);
1022         return r;
1023 }
1024
1025 /**
1026  * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
1027  *
1028  * @f: open file handle
1029  * @buf: User buffer to write data from
1030  * @size: Number of bytes to write
1031  * @pos:  Offset to seek to
1032  *
1033  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1034  */
1035 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1036                                          size_t size, loff_t *pos)
1037 {
1038         struct amdgpu_device *adev = file_inode(f)->i_private;
1039         ssize_t result = 0;
1040         int r;
1041
1042         if (size & 0x3 || *pos & 0x3)
1043                 return -EINVAL;
1044
1045         r = pm_runtime_get_sync(adev->ddev->dev);
1046         if (r < 0) {
1047                 pm_runtime_put_autosuspend(adev->ddev->dev);
1048                 return r;
1049         }
1050
1051         while (size) {
1052                 uint32_t value;
1053
1054                 r = get_user(value, (uint32_t *)buf);
1055                 if (r) {
1056                         pm_runtime_mark_last_busy(adev->ddev->dev);
1057                         pm_runtime_put_autosuspend(adev->ddev->dev);
1058                         return r;
1059                 }
1060
1061                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1062
1063                 result += 4;
1064                 buf += 4;
1065                 *pos += 4;
1066                 size -= 4;
1067         }
1068
1069         pm_runtime_mark_last_busy(adev->ddev->dev);
1070         pm_runtime_put_autosuspend(adev->ddev->dev);
1071
1072         return result;
1073 }
1074
1075
1076 static const struct file_operations amdgpu_debugfs_regs_fops = {
1077         .owner = THIS_MODULE,
1078         .read = amdgpu_debugfs_regs_read,
1079         .write = amdgpu_debugfs_regs_write,
1080         .llseek = default_llseek
1081 };
1082 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1083         .owner = THIS_MODULE,
1084         .read = amdgpu_debugfs_regs_didt_read,
1085         .write = amdgpu_debugfs_regs_didt_write,
1086         .llseek = default_llseek
1087 };
1088 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1089         .owner = THIS_MODULE,
1090         .read = amdgpu_debugfs_regs_pcie_read,
1091         .write = amdgpu_debugfs_regs_pcie_write,
1092         .llseek = default_llseek
1093 };
1094 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1095         .owner = THIS_MODULE,
1096         .read = amdgpu_debugfs_regs_smc_read,
1097         .write = amdgpu_debugfs_regs_smc_write,
1098         .llseek = default_llseek
1099 };
1100
1101 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1102         .owner = THIS_MODULE,
1103         .read = amdgpu_debugfs_gca_config_read,
1104         .llseek = default_llseek
1105 };
1106
1107 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1108         .owner = THIS_MODULE,
1109         .read = amdgpu_debugfs_sensor_read,
1110         .llseek = default_llseek
1111 };
1112
1113 static const struct file_operations amdgpu_debugfs_wave_fops = {
1114         .owner = THIS_MODULE,
1115         .read = amdgpu_debugfs_wave_read,
1116         .llseek = default_llseek
1117 };
1118 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1119         .owner = THIS_MODULE,
1120         .read = amdgpu_debugfs_gpr_read,
1121         .llseek = default_llseek
1122 };
1123
1124 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1125         .owner = THIS_MODULE,
1126         .write = amdgpu_debugfs_gfxoff_write,
1127 };
1128
1129 static const struct file_operations *debugfs_regs[] = {
1130         &amdgpu_debugfs_regs_fops,
1131         &amdgpu_debugfs_regs_didt_fops,
1132         &amdgpu_debugfs_regs_pcie_fops,
1133         &amdgpu_debugfs_regs_smc_fops,
1134         &amdgpu_debugfs_gca_config_fops,
1135         &amdgpu_debugfs_sensors_fops,
1136         &amdgpu_debugfs_wave_fops,
1137         &amdgpu_debugfs_gpr_fops,
1138         &amdgpu_debugfs_gfxoff_fops,
1139 };
1140
1141 static const char *debugfs_regs_names[] = {
1142         "amdgpu_regs",
1143         "amdgpu_regs_didt",
1144         "amdgpu_regs_pcie",
1145         "amdgpu_regs_smc",
1146         "amdgpu_gca_config",
1147         "amdgpu_sensors",
1148         "amdgpu_wave",
1149         "amdgpu_gpr",
1150         "amdgpu_gfxoff",
1151 };
1152
1153 /**
1154  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1155  *                                                              register access.
1156  *
1157  * @adev: The device to attach the debugfs entries to
1158  */
1159 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1160 {
1161         struct drm_minor *minor = adev->ddev->primary;
1162         struct dentry *ent, *root = minor->debugfs_root;
1163         unsigned int i;
1164
1165         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1166                 ent = debugfs_create_file(debugfs_regs_names[i],
1167                                           S_IFREG | S_IRUGO, root,
1168                                           adev, debugfs_regs[i]);
1169                 if (!i && !IS_ERR_OR_NULL(ent))
1170                         i_size_write(ent->d_inode, adev->rmmio_size);
1171                 adev->debugfs_regs[i] = ent;
1172         }
1173
1174         return 0;
1175 }
1176
1177 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
1178 {
1179         struct drm_info_node *node = (struct drm_info_node *) m->private;
1180         struct drm_device *dev = node->minor->dev;
1181         struct amdgpu_device *adev = dev->dev_private;
1182         int r = 0, i;
1183
1184         r = pm_runtime_get_sync(dev->dev);
1185         if (r < 0) {
1186                 pm_runtime_put_autosuspend(adev->ddev->dev);
1187                 return r;
1188         }
1189
1190         /* Avoid accidently unparking the sched thread during GPU reset */
1191         mutex_lock(&adev->lock_reset);
1192
1193         /* hold on the scheduler */
1194         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1195                 struct amdgpu_ring *ring = adev->rings[i];
1196
1197                 if (!ring || !ring->sched.thread)
1198                         continue;
1199                 kthread_park(ring->sched.thread);
1200         }
1201
1202         seq_printf(m, "run ib test:\n");
1203         r = amdgpu_ib_ring_tests(adev);
1204         if (r)
1205                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1206         else
1207                 seq_printf(m, "ib ring tests passed.\n");
1208
1209         /* go on the scheduler */
1210         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1211                 struct amdgpu_ring *ring = adev->rings[i];
1212
1213                 if (!ring || !ring->sched.thread)
1214                         continue;
1215                 kthread_unpark(ring->sched.thread);
1216         }
1217
1218         mutex_unlock(&adev->lock_reset);
1219
1220         pm_runtime_mark_last_busy(dev->dev);
1221         pm_runtime_put_autosuspend(dev->dev);
1222
1223         return 0;
1224 }
1225
1226 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1227 {
1228         struct drm_info_node *node = (struct drm_info_node *) m->private;
1229         struct drm_device *dev = node->minor->dev;
1230         struct amdgpu_device *adev = dev->dev_private;
1231
1232         seq_write(m, adev->bios, adev->bios_size);
1233         return 0;
1234 }
1235
1236 static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1237 {
1238         struct drm_info_node *node = (struct drm_info_node *)m->private;
1239         struct drm_device *dev = node->minor->dev;
1240         struct amdgpu_device *adev = dev->dev_private;
1241         int r;
1242
1243         r = pm_runtime_get_sync(dev->dev);
1244         if (r < 0) {
1245                 pm_runtime_put_autosuspend(adev->ddev->dev);
1246                 return r;
1247         }
1248
1249         seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1250
1251         pm_runtime_mark_last_busy(dev->dev);
1252         pm_runtime_put_autosuspend(dev->dev);
1253
1254         return 0;
1255 }
1256
1257 static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1258 {
1259         struct drm_info_node *node = (struct drm_info_node *)m->private;
1260         struct drm_device *dev = node->minor->dev;
1261         struct amdgpu_device *adev = dev->dev_private;
1262         int r;
1263
1264         r = pm_runtime_get_sync(dev->dev);
1265         if (r < 0) {
1266                 pm_runtime_put_autosuspend(adev->ddev->dev);
1267                 return r;
1268         }
1269
1270         seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1271
1272         pm_runtime_mark_last_busy(dev->dev);
1273         pm_runtime_put_autosuspend(dev->dev);
1274
1275         return 0;
1276 }
1277
1278 static const struct drm_info_list amdgpu_debugfs_list[] = {
1279         {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1280         {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1281         {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1282         {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1283 };
1284
1285 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1286                                           struct dma_fence **fences)
1287 {
1288         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1289         uint32_t sync_seq, last_seq;
1290
1291         last_seq = atomic_read(&ring->fence_drv.last_seq);
1292         sync_seq = ring->fence_drv.sync_seq;
1293
1294         last_seq &= drv->num_fences_mask;
1295         sync_seq &= drv->num_fences_mask;
1296
1297         do {
1298                 struct dma_fence *fence, **ptr;
1299
1300                 ++last_seq;
1301                 last_seq &= drv->num_fences_mask;
1302                 ptr = &drv->fences[last_seq];
1303
1304                 fence = rcu_dereference_protected(*ptr, 1);
1305                 RCU_INIT_POINTER(*ptr, NULL);
1306
1307                 if (!fence)
1308                         continue;
1309
1310                 fences[last_seq] = fence;
1311
1312         } while (last_seq != sync_seq);
1313 }
1314
1315 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1316                                             int length)
1317 {
1318         int i;
1319         struct dma_fence *fence;
1320
1321         for (i = 0; i < length; i++) {
1322                 fence = fences[i];
1323                 if (!fence)
1324                         continue;
1325                 dma_fence_signal(fence);
1326                 dma_fence_put(fence);
1327         }
1328 }
1329
1330 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1331 {
1332         struct drm_sched_job *s_job;
1333         struct dma_fence *fence;
1334
1335         spin_lock(&sched->job_list_lock);
1336         list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1337                 fence = sched->ops->run_job(s_job);
1338                 dma_fence_put(fence);
1339         }
1340         spin_unlock(&sched->job_list_lock);
1341 }
1342
1343 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1344 {
1345         struct amdgpu_job *job;
1346         struct drm_sched_job *s_job;
1347         uint32_t preempt_seq;
1348         struct dma_fence *fence, **ptr;
1349         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1350         struct drm_gpu_scheduler *sched = &ring->sched;
1351
1352         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1353                 return;
1354
1355         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1356         if (preempt_seq <= atomic_read(&drv->last_seq))
1357                 return;
1358
1359         preempt_seq &= drv->num_fences_mask;
1360         ptr = &drv->fences[preempt_seq];
1361         fence = rcu_dereference_protected(*ptr, 1);
1362
1363         spin_lock(&sched->job_list_lock);
1364         list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1365                 job = to_amdgpu_job(s_job);
1366                 if (job->fence == fence)
1367                         /* mark the job as preempted */
1368                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1369         }
1370         spin_unlock(&sched->job_list_lock);
1371 }
1372
1373 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1374 {
1375         int r, resched, length;
1376         struct amdgpu_ring *ring;
1377         struct dma_fence **fences = NULL;
1378         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1379
1380         if (val >= AMDGPU_MAX_RINGS)
1381                 return -EINVAL;
1382
1383         ring = adev->rings[val];
1384
1385         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1386                 return -EINVAL;
1387
1388         /* the last preemption failed */
1389         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1390                 return -EBUSY;
1391
1392         length = ring->fence_drv.num_fences_mask + 1;
1393         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1394         if (!fences)
1395                 return -ENOMEM;
1396
1397         /* Avoid accidently unparking the sched thread during GPU reset */
1398         mutex_lock(&adev->lock_reset);
1399
1400         /* stop the scheduler */
1401         kthread_park(ring->sched.thread);
1402
1403         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1404
1405         /* preempt the IB */
1406         r = amdgpu_ring_preempt_ib(ring);
1407         if (r) {
1408                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1409                 goto failure;
1410         }
1411
1412         amdgpu_fence_process(ring);
1413
1414         if (atomic_read(&ring->fence_drv.last_seq) !=
1415             ring->fence_drv.sync_seq) {
1416                 DRM_INFO("ring %d was preempted\n", ring->idx);
1417
1418                 amdgpu_ib_preempt_mark_partial_job(ring);
1419
1420                 /* swap out the old fences */
1421                 amdgpu_ib_preempt_fences_swap(ring, fences);
1422
1423                 amdgpu_fence_driver_force_completion(ring);
1424
1425                 /* resubmit unfinished jobs */
1426                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1427
1428                 /* wait for jobs finished */
1429                 amdgpu_fence_wait_empty(ring);
1430
1431                 /* signal the old fences */
1432                 amdgpu_ib_preempt_signal_fences(fences, length);
1433         }
1434
1435 failure:
1436         /* restart the scheduler */
1437         kthread_unpark(ring->sched.thread);
1438
1439         mutex_unlock(&adev->lock_reset);
1440
1441         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1442
1443         kfree(fences);
1444
1445         return 0;
1446 }
1447
1448 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1449 {
1450         int ret = 0;
1451         uint32_t max_freq, min_freq;
1452         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1453
1454         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1455                 return -EINVAL;
1456
1457         ret = pm_runtime_get_sync(adev->ddev->dev);
1458         if (ret < 0) {
1459                 pm_runtime_put_autosuspend(adev->ddev->dev);
1460                 return ret;
1461         }
1462
1463         if (is_support_sw_smu(adev)) {
1464                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq, true);
1465                 if (ret || val > max_freq || val < min_freq)
1466                         return -EINVAL;
1467                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val, true);
1468         }
1469
1470         pm_runtime_mark_last_busy(adev->ddev->dev);
1471         pm_runtime_put_autosuspend(adev->ddev->dev);
1472
1473         if (ret)
1474                 return -EINVAL;
1475
1476         return 0;
1477 }
1478
1479 DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1480                         amdgpu_debugfs_ib_preempt, "%llu\n");
1481
1482 DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
1483                         amdgpu_debugfs_sclk_set, "%llu\n");
1484
1485 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1486 {
1487         int r, i;
1488
1489         adev->debugfs_preempt =
1490                 debugfs_create_file("amdgpu_preempt_ib", 0600,
1491                                     adev->ddev->primary->debugfs_root, adev,
1492                                     &fops_ib_preempt);
1493         if (!(adev->debugfs_preempt)) {
1494                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1495                 return -EIO;
1496         }
1497
1498         adev->smu.debugfs_sclk =
1499                 debugfs_create_file("amdgpu_force_sclk", 0200,
1500                                     adev->ddev->primary->debugfs_root, adev,
1501                                     &fops_sclk_set);
1502         if (!(adev->smu.debugfs_sclk)) {
1503                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1504                 return -EIO;
1505         }
1506
1507         /* Register debugfs entries for amdgpu_ttm */
1508         r = amdgpu_ttm_debugfs_init(adev);
1509         if (r) {
1510                 DRM_ERROR("Failed to init debugfs\n");
1511                 return r;
1512         }
1513
1514         r = amdgpu_debugfs_pm_init(adev);
1515         if (r) {
1516                 DRM_ERROR("Failed to register debugfs file for dpm!\n");
1517                 return r;
1518         }
1519
1520         if (amdgpu_debugfs_sa_init(adev)) {
1521                 dev_err(adev->dev, "failed to register debugfs file for SA\n");
1522         }
1523
1524         if (amdgpu_debugfs_fence_init(adev))
1525                 dev_err(adev->dev, "fence debugfs file creation failed\n");
1526
1527         r = amdgpu_debugfs_gem_init(adev);
1528         if (r)
1529                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1530
1531         r = amdgpu_debugfs_regs_init(adev);
1532         if (r)
1533                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1534
1535         r = amdgpu_debugfs_firmware_init(adev);
1536         if (r)
1537                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1538
1539 #if defined(CONFIG_DRM_AMD_DC)
1540         if (amdgpu_device_has_dc_support(adev)) {
1541                 if (dtn_debugfs_init(adev))
1542                         DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
1543         }
1544 #endif
1545
1546         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1547                 struct amdgpu_ring *ring = adev->rings[i];
1548
1549                 if (!ring)
1550                         continue;
1551
1552                 if (amdgpu_debugfs_ring_init(adev, ring)) {
1553                         DRM_ERROR("Failed to register debugfs file for rings !\n");
1554                 }
1555         }
1556
1557         amdgpu_ras_debugfs_create_all(adev);
1558
1559         amdgpu_debugfs_autodump_init(adev);
1560
1561         return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1562                                         ARRAY_SIZE(amdgpu_debugfs_list));
1563 }
1564
1565 #else
1566 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1567 {
1568         return 0;
1569 }
1570 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1571 {
1572         return 0;
1573 }
1574 #endif