drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30 #include <linux/poll.h>
  31 #include <drm/drm_debugfs.h>
  32
  33 #include "amdgpu.h"
  34 #include "amdgpu_pm.h"
  35 #include "amdgpu_dm_debugfs.h"
  36 #include "amdgpu_ras.h"
  37 #include "amdgpu_rap.h"
  38 #include "amdgpu_fw_attestation.h"
  39
  40 /**
  41  * amdgpu_debugfs_add_files - Add simple debugfs entries
  42  *
  43  * @adev:  Device to attach debugfs entries to
  44  * @files:  Array of function callbacks that respond to reads
  45  * @nfiles: Number of callbacks to register
  46  *
  47  */
  48 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
  49                              const struct drm_info_list *files,
  50                              unsigned nfiles)
  51 {
  52         unsigned i;
  53
  54         for (i = 0; i < adev->debugfs_count; i++) {
  55                 if (adev->debugfs[i].files == files) {
  56                         /* Already registered */
  57                         return 0;
  58                 }
  59         }
  60
  61         i = adev->debugfs_count + 1;
  62         if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
  63                 DRM_ERROR("Reached maximum number of debugfs components.\n");
  64                 DRM_ERROR("Report so we increase "
  65                           "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
  66                 return -EINVAL;
  67         }
  68         adev->debugfs[adev->debugfs_count].files = files;
  69         adev->debugfs[adev->debugfs_count].num_files = nfiles;
  70         adev->debugfs_count = i;
  71 #if defined(CONFIG_DEBUG_FS)
  72         drm_debugfs_create_files(files, nfiles,
  73                                  adev_to_drm(adev)->primary->debugfs_root,
  74                                  adev_to_drm(adev)->primary);
  75 #endif
  76         return 0;
  77 }
  78
  79 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
  80 {
  81 #if defined(CONFIG_DEBUG_FS)
  82         unsigned long timeout = 600 * HZ;
  83         int ret;
  84
  85         wake_up_interruptible(&adev->autodump.gpu_hang);
  86
  87         ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
  88         if (ret == 0) {
  89                 pr_err("autodump: timeout, move on to gpu recovery\n");
  90                 return -ETIMEDOUT;
  91         }
  92 #endif
  93         return 0;
  94 }
  95
  96 #if defined(CONFIG_DEBUG_FS)
  97
  98 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
  99 {
 100         struct amdgpu_device *adev = inode->i_private;
 101         int ret;
 102
 103         file->private_data = adev;
 104
 105         ret = down_read_killable(&adev->reset_sem);
 106         if (ret)
 107                 return ret;
 108
 109         if (adev->autodump.dumping.done) {
 110                 reinit_completion(&adev->autodump.dumping);
 111                 ret = 0;
 112         } else {
 113                 ret = -EBUSY;
 114         }
 115
 116         up_read(&adev->reset_sem);
 117
 118         return ret;
 119 }
 120
 121 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
 122 {
 123         struct amdgpu_device *adev = file->private_data;
 124
 125         complete_all(&adev->autodump.dumping);
 126         return 0;
 127 }
 128
 129 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
 130 {
 131         struct amdgpu_device *adev = file->private_data;
 132
 133         poll_wait(file, &adev->autodump.gpu_hang, poll_table);
 134
 135         if (amdgpu_in_reset(adev))
 136                 return POLLIN | POLLRDNORM | POLLWRNORM;
 137
 138         return 0;
 139 }
 140
 141 static const struct file_operations autodump_debug_fops = {
 142         .owner = THIS_MODULE,
 143         .open = amdgpu_debugfs_autodump_open,
 144         .poll = amdgpu_debugfs_autodump_poll,
 145         .release = amdgpu_debugfs_autodump_release,
 146 };
 147
 148 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
 149 {
 150         init_completion(&adev->autodump.dumping);
 151         complete_all(&adev->autodump.dumping);
 152         init_waitqueue_head(&adev->autodump.gpu_hang);
 153
 154         debugfs_create_file("amdgpu_autodump", 0600,
 155                 adev_to_drm(adev)->primary->debugfs_root,
 156                 adev, &autodump_debug_fops);
 157 }
 158
 159 /**
 160  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
 161  *
 162  * @read: True if reading
 163  * @f: open file handle
 164  * @buf: User buffer to write/read to
 165  * @size: Number of bytes to write/read
 166  * @pos:  Offset to seek to
 167  *
 168  * This debugfs entry has special meaning on the offset being sought.
 169  * Various bits have different meanings:
 170  *
 171  * Bit 62:  Indicates a GRBM bank switch is needed
 172  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
 173  *          zero)
 174  * Bits 24..33: The SE or ME selector if needed
 175  * Bits 34..43: The SH (or SA) or PIPE selector if needed
 176  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
 177  *
 178  * Bit 23:  Indicates that the PM power gating lock should be held
 179  *          This is necessary to read registers that might be
 180  *          unreliable during a power gating transistion.
 181  *
 182  * The lower bits are the BYTE offset of the register to read.  This
 183  * allows reading multiple registers in a single call and having
 184  * the returned size reflect that.
 185  */
 186 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
 187                 char __user *buf, size_t size, loff_t *pos)
 188 {
 189         struct amdgpu_device *adev = file_inode(f)->i_private;
 190         ssize_t result = 0;
 191         int r;
 192         bool pm_pg_lock, use_bank, use_ring;
 193         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
 194
 195         pm_pg_lock = use_bank = use_ring = false;
 196         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
 197
 198         if (size & 0x3 || *pos & 0x3 ||
 199                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
 200                 return -EINVAL;
 201
 202         /* are we reading registers for which a PG lock is necessary? */
 203         pm_pg_lock = (*pos >> 23) & 1;
 204
 205         if (*pos & (1ULL << 62)) {
 206                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
 207                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
 208                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
 209
 210                 if (se_bank == 0x3FF)
 211                         se_bank = 0xFFFFFFFF;
 212                 if (sh_bank == 0x3FF)
 213                         sh_bank = 0xFFFFFFFF;
 214                 if (instance_bank == 0x3FF)
 215                         instance_bank = 0xFFFFFFFF;
 216                 use_bank = true;
 217         } else if (*pos & (1ULL << 61)) {
 218
 219                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 220                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 221                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 222                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 223
 224                 use_ring = true;
 225         } else {
 226                 use_bank = use_ring = false;
 227         }
 228
 229         *pos &= (1UL << 22) - 1;
 230
 231         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 232         if (r < 0) {
 233                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 234                 return r;
 235         }
 236
 237         r = amdgpu_virt_enable_access_debugfs(adev);
 238         if (r < 0) {
 239                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 240                 return r;
 241         }
 242
 243         if (use_bank) {
 244                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 245                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 246                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 247                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 248                         amdgpu_virt_disable_access_debugfs(adev);
 249                         return -EINVAL;
 250                 }
 251                 mutex_lock(&adev->grbm_idx_mutex);
 252                 amdgpu_gfx_select_se_sh(adev, se_bank,
 253                                         sh_bank, instance_bank);
 254         } else if (use_ring) {
 255                 mutex_lock(&adev->srbm_mutex);
 256                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 257         }
 258
 259         if (pm_pg_lock)
 260                 mutex_lock(&adev->pm.mutex);
 261
 262         while (size) {
 263                 uint32_t value;
 264
 265                 if (read) {
 266                         value = RREG32(*pos >> 2);
 267                         r = put_user(value, (uint32_t *)buf);
 268                 } else {
 269                         r = get_user(value, (uint32_t *)buf);
 270                         if (!r)
 271                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 272                 }
 273                 if (r) {
 274                         result = r;
 275                         goto end;
 276                 }
 277
 278                 result += 4;
 279                 buf += 4;
 280                 *pos += 4;
 281                 size -= 4;
 282         }
 283
 284 end:
 285         if (use_bank) {
 286                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 287                 mutex_unlock(&adev->grbm_idx_mutex);
 288         } else if (use_ring) {
 289                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 290                 mutex_unlock(&adev->srbm_mutex);
 291         }
 292
 293         if (pm_pg_lock)
 294                 mutex_unlock(&adev->pm.mutex);
 295
 296         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 297         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 298
 299         amdgpu_virt_disable_access_debugfs(adev);
 300         return result;
 301 }
 302
 303 /**
 304  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 305  */
 306 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 307                                         size_t size, loff_t *pos)
 308 {
 309         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 310 }
 311
 312 /**
 313  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 314  */
 315 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 316                                          size_t size, loff_t *pos)
 317 {
 318         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 319 }
 320
 321
 322 /**
 323  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 324  *
 325  * @f: open file handle
 326  * @buf: User buffer to store read data in
 327  * @size: Number of bytes to read
 328  * @pos:  Offset to seek to
 329  *
 330  * The lower bits are the BYTE offset of the register to read.  This
 331  * allows reading multiple registers in a single call and having
 332  * the returned size reflect that.
 333  */
 334 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 335                                         size_t size, loff_t *pos)
 336 {
 337         struct amdgpu_device *adev = file_inode(f)->i_private;
 338         ssize_t result = 0;
 339         int r;
 340
 341         if (size & 0x3 || *pos & 0x3)
 342                 return -EINVAL;
 343
 344         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 345         if (r < 0) {
 346                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 347                 return r;
 348         }
 349
 350         r = amdgpu_virt_enable_access_debugfs(adev);
 351         if (r < 0) {
 352                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 353                 return r;
 354         }
 355
 356         while (size) {
 357                 uint32_t value;
 358
 359                 value = RREG32_PCIE(*pos >> 2);
 360                 r = put_user(value, (uint32_t *)buf);
 361                 if (r) {
 362                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 363                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 364                         amdgpu_virt_disable_access_debugfs(adev);
 365                         return r;
 366                 }
 367
 368                 result += 4;
 369                 buf += 4;
 370                 *pos += 4;
 371                 size -= 4;
 372         }
 373
 374         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 375         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 376
 377         amdgpu_virt_disable_access_debugfs(adev);
 378         return result;
 379 }
 380
 381 /**
 382  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 383  *
 384  * @f: open file handle
 385  * @buf: User buffer to write data from
 386  * @size: Number of bytes to write
 387  * @pos:  Offset to seek to
 388  *
 389  * The lower bits are the BYTE offset of the register to write.  This
 390  * allows writing multiple registers in a single call and having
 391  * the returned size reflect that.
 392  */
 393 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 394                                          size_t size, loff_t *pos)
 395 {
 396         struct amdgpu_device *adev = file_inode(f)->i_private;
 397         ssize_t result = 0;
 398         int r;
 399
 400         if (size & 0x3 || *pos & 0x3)
 401                 return -EINVAL;
 402
 403         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 404         if (r < 0) {
 405                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 406                 return r;
 407         }
 408
 409         r = amdgpu_virt_enable_access_debugfs(adev);
 410         if (r < 0) {
 411                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 412                 return r;
 413         }
 414
 415         while (size) {
 416                 uint32_t value;
 417
 418                 r = get_user(value, (uint32_t *)buf);
 419                 if (r) {
 420                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 421                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 422                         amdgpu_virt_disable_access_debugfs(adev);
 423                         return r;
 424                 }
 425
 426                 WREG32_PCIE(*pos >> 2, value);
 427
 428                 result += 4;
 429                 buf += 4;
 430                 *pos += 4;
 431                 size -= 4;
 432         }
 433
 434         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 435         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 436
 437         amdgpu_virt_disable_access_debugfs(adev);
 438         return result;
 439 }
 440
 441 /**
 442  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 443  *
 444  * @f: open file handle
 445  * @buf: User buffer to store read data in
 446  * @size: Number of bytes to read
 447  * @pos:  Offset to seek to
 448  *
 449  * The lower bits are the BYTE offset of the register to read.  This
 450  * allows reading multiple registers in a single call and having
 451  * the returned size reflect that.
 452  */
 453 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 454                                         size_t size, loff_t *pos)
 455 {
 456         struct amdgpu_device *adev = file_inode(f)->i_private;
 457         ssize_t result = 0;
 458         int r;
 459
 460         if (size & 0x3 || *pos & 0x3)
 461                 return -EINVAL;
 462
 463         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 464         if (r < 0) {
 465                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 466                 return r;
 467         }
 468
 469         r = amdgpu_virt_enable_access_debugfs(adev);
 470         if (r < 0) {
 471                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 472                 return r;
 473         }
 474
 475         while (size) {
 476                 uint32_t value;
 477
 478                 value = RREG32_DIDT(*pos >> 2);
 479                 r = put_user(value, (uint32_t *)buf);
 480                 if (r) {
 481                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 482                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 483                         amdgpu_virt_disable_access_debugfs(adev);
 484                         return r;
 485                 }
 486
 487                 result += 4;
 488                 buf += 4;
 489                 *pos += 4;
 490                 size -= 4;
 491         }
 492
 493         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 494         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 495
 496         amdgpu_virt_disable_access_debugfs(adev);
 497         return result;
 498 }
 499
 500 /**
 501  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 502  *
 503  * @f: open file handle
 504  * @buf: User buffer to write data from
 505  * @size: Number of bytes to write
 506  * @pos:  Offset to seek to
 507  *
 508  * The lower bits are the BYTE offset of the register to write.  This
 509  * allows writing multiple registers in a single call and having
 510  * the returned size reflect that.
 511  */
 512 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 513                                          size_t size, loff_t *pos)
 514 {
 515         struct amdgpu_device *adev = file_inode(f)->i_private;
 516         ssize_t result = 0;
 517         int r;
 518
 519         if (size & 0x3 || *pos & 0x3)
 520                 return -EINVAL;
 521
 522         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 523         if (r < 0) {
 524                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 525                 return r;
 526         }
 527
 528         r = amdgpu_virt_enable_access_debugfs(adev);
 529         if (r < 0) {
 530                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 531                 return r;
 532         }
 533
 534         while (size) {
 535                 uint32_t value;
 536
 537                 r = get_user(value, (uint32_t *)buf);
 538                 if (r) {
 539                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 540                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 541                         amdgpu_virt_disable_access_debugfs(adev);
 542                         return r;
 543                 }
 544
 545                 WREG32_DIDT(*pos >> 2, value);
 546
 547                 result += 4;
 548                 buf += 4;
 549                 *pos += 4;
 550                 size -= 4;
 551         }
 552
 553         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 554         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 555
 556         amdgpu_virt_disable_access_debugfs(adev);
 557         return result;
 558 }
 559
 560 /**
 561  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 562  *
 563  * @f: open file handle
 564  * @buf: User buffer to store read data in
 565  * @size: Number of bytes to read
 566  * @pos:  Offset to seek to
 567  *
 568  * The lower bits are the BYTE offset of the register to read.  This
 569  * allows reading multiple registers in a single call and having
 570  * the returned size reflect that.
 571  */
 572 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 573                                         size_t size, loff_t *pos)
 574 {
 575         struct amdgpu_device *adev = file_inode(f)->i_private;
 576         ssize_t result = 0;
 577         int r;
 578
 579         if (size & 0x3 || *pos & 0x3)
 580                 return -EINVAL;
 581
 582         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 583         if (r < 0) {
 584                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 585                 return r;
 586         }
 587
 588         r = amdgpu_virt_enable_access_debugfs(adev);
 589         if (r < 0) {
 590                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 591                 return r;
 592         }
 593
 594         while (size) {
 595                 uint32_t value;
 596
 597                 value = RREG32_SMC(*pos);
 598                 r = put_user(value, (uint32_t *)buf);
 599                 if (r) {
 600                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 601                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 602                         amdgpu_virt_disable_access_debugfs(adev);
 603                         return r;
 604                 }
 605
 606                 result += 4;
 607                 buf += 4;
 608                 *pos += 4;
 609                 size -= 4;
 610         }
 611
 612         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 613         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 614
 615         amdgpu_virt_disable_access_debugfs(adev);
 616         return result;
 617 }
 618
 619 /**
 620  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 621  *
 622  * @f: open file handle
 623  * @buf: User buffer to write data from
 624  * @size: Number of bytes to write
 625  * @pos:  Offset to seek to
 626  *
 627  * The lower bits are the BYTE offset of the register to write.  This
 628  * allows writing multiple registers in a single call and having
 629  * the returned size reflect that.
 630  */
 631 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 632                                          size_t size, loff_t *pos)
 633 {
 634         struct amdgpu_device *adev = file_inode(f)->i_private;
 635         ssize_t result = 0;
 636         int r;
 637
 638         if (size & 0x3 || *pos & 0x3)
 639                 return -EINVAL;
 640
 641         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 642         if (r < 0) {
 643                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 644                 return r;
 645         }
 646
 647         r = amdgpu_virt_enable_access_debugfs(adev);
 648         if (r < 0) {
 649                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 650                 return r;
 651         }
 652
 653         while (size) {
 654                 uint32_t value;
 655
 656                 r = get_user(value, (uint32_t *)buf);
 657                 if (r) {
 658                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 659                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 660                         amdgpu_virt_disable_access_debugfs(adev);
 661                         return r;
 662                 }
 663
 664                 WREG32_SMC(*pos, value);
 665
 666                 result += 4;
 667                 buf += 4;
 668                 *pos += 4;
 669                 size -= 4;
 670         }
 671
 672         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 673         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 674
 675         amdgpu_virt_disable_access_debugfs(adev);
 676         return result;
 677 }
 678
 679 /**
 680  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 681  *
 682  * @f: open file handle
 683  * @buf: User buffer to store read data in
 684  * @size: Number of bytes to read
 685  * @pos:  Offset to seek to
 686  *
 687  * This file is used to access configuration data in a somewhat
 688  * stable fashion.  The format is a series of DWORDs with the first
 689  * indicating which revision it is.  New content is appended to the
 690  * end so that older software can still read the data.
 691  */
 692
 693 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 694                                         size_t size, loff_t *pos)
 695 {
 696         struct amdgpu_device *adev = file_inode(f)->i_private;
 697         ssize_t result = 0;
 698         int r;
 699         uint32_t *config, no_regs = 0;
 700
 701         if (size & 0x3 || *pos & 0x3)
 702                 return -EINVAL;
 703
 704         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 705         if (!config)
 706                 return -ENOMEM;
 707
 708         /* version, increment each time something is added */
 709         config[no_regs++] = 3;
 710         config[no_regs++] = adev->gfx.config.max_shader_engines;
 711         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 712         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 713         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 714         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 715         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 716         config[no_regs++] = adev->gfx.config.max_gprs;
 717         config[no_regs++] = adev->gfx.config.max_gs_threads;
 718         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 719         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 720         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 721         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 722         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 723         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 724         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 725         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 726         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 727         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 728         config[no_regs++] = adev->gfx.config.num_gpus;
 729         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 730         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 731         config[no_regs++] = adev->gfx.config.gb_addr_config;
 732         config[no_regs++] = adev->gfx.config.num_rbs;
 733
 734         /* rev==1 */
 735         config[no_regs++] = adev->rev_id;
 736         config[no_regs++] = adev->pg_flags;
 737         config[no_regs++] = adev->cg_flags;
 738
 739         /* rev==2 */
 740         config[no_regs++] = adev->family;
 741         config[no_regs++] = adev->external_rev_id;
 742
 743         /* rev==3 */
 744         config[no_regs++] = adev->pdev->device;
 745         config[no_regs++] = adev->pdev->revision;
 746         config[no_regs++] = adev->pdev->subsystem_device;
 747         config[no_regs++] = adev->pdev->subsystem_vendor;
 748
 749         while (size && (*pos < no_regs * 4)) {
 750                 uint32_t value;
 751
 752                 value = config[*pos >> 2];
 753                 r = put_user(value, (uint32_t *)buf);
 754                 if (r) {
 755                         kfree(config);
 756                         return r;
 757                 }
 758
 759                 result += 4;
 760                 buf += 4;
 761                 *pos += 4;
 762                 size -= 4;
 763         }
 764
 765         kfree(config);
 766         return result;
 767 }
 768
 769 /**
 770  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 771  *
 772  * @f: open file handle
 773  * @buf: User buffer to store read data in
 774  * @size: Number of bytes to read
 775  * @pos:  Offset to seek to
 776  *
 777  * The offset is treated as the BYTE address of one of the sensors
 778  * enumerated in amd/include/kgd_pp_interface.h under the
 779  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 780  * you would use the offset 3 * 4 = 12.
 781  */
 782 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 783                                         size_t size, loff_t *pos)
 784 {
 785         struct amdgpu_device *adev = file_inode(f)->i_private;
 786         int idx, x, outsize, r, valuesize;
 787         uint32_t values[16];
 788
 789         if (size & 3 || *pos & 0x3)
 790                 return -EINVAL;
 791
 792         if (!adev->pm.dpm_enabled)
 793                 return -EINVAL;
 794
 795         /* convert offset to sensor number */
 796         idx = *pos >> 2;
 797
 798         valuesize = sizeof(values);
 799
 800         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 801         if (r < 0) {
 802                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 803                 return r;
 804         }
 805
 806         r = amdgpu_virt_enable_access_debugfs(adev);
 807         if (r < 0) {
 808                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 809                 return r;
 810         }
 811
 812         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 813
 814         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 815         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 816
 817         if (r) {
 818                 amdgpu_virt_disable_access_debugfs(adev);
 819                 return r;
 820         }
 821
 822         if (size > valuesize) {
 823                 amdgpu_virt_disable_access_debugfs(adev);
 824                 return -EINVAL;
 825         }
 826
 827         outsize = 0;
 828         x = 0;
 829         if (!r) {
 830                 while (size) {
 831                         r = put_user(values[x++], (int32_t *)buf);
 832                         buf += 4;
 833                         size -= 4;
 834                         outsize += 4;
 835                 }
 836         }
 837
 838         amdgpu_virt_disable_access_debugfs(adev);
 839         return !r ? outsize : r;
 840 }
 841
 842 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 843  *
 844  * @f: open file handle
 845  * @buf: User buffer to store read data in
 846  * @size: Number of bytes to read
 847  * @pos:  Offset to seek to
 848  *
 849  * The offset being sought changes which wave that the status data
 850  * will be returned for.  The bits are used as follows:
 851  *
 852  * Bits 0..6:   Byte offset into data
 853  * Bits 7..14:  SE selector
 854  * Bits 15..22: SH/SA selector
 855  * Bits 23..30: CU/{WGP+SIMD} selector
 856  * Bits 31..36: WAVE ID selector
 857  * Bits 37..44: SIMD ID selector
 858  *
 859  * The returned data begins with one DWORD of version information
 860  * Followed by WAVE STATUS registers relevant to the GFX IP version
 861  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 862  */
 863 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 864                                         size_t size, loff_t *pos)
 865 {
 866         struct amdgpu_device *adev = f->f_inode->i_private;
 867         int r, x;
 868         ssize_t result = 0;
 869         uint32_t offset, se, sh, cu, wave, simd, data[32];
 870
 871         if (size & 3 || *pos & 3)
 872                 return -EINVAL;
 873
 874         /* decode offset */
 875         offset = (*pos & GENMASK_ULL(6, 0));
 876         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 877         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 878         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 879         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 880         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 881
 882         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 883         if (r < 0) {
 884                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 885                 return r;
 886         }
 887
 888         r = amdgpu_virt_enable_access_debugfs(adev);
 889         if (r < 0) {
 890                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 891                 return r;
 892         }
 893
 894         /* switch to the specific se/sh/cu */
 895         mutex_lock(&adev->grbm_idx_mutex);
 896         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 897
 898         x = 0;
 899         if (adev->gfx.funcs->read_wave_data)
 900                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 901
 902         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 903         mutex_unlock(&adev->grbm_idx_mutex);
 904
 905         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 906         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 907
 908         if (!x) {
 909                 amdgpu_virt_disable_access_debugfs(adev);
 910                 return -EINVAL;
 911         }
 912
 913         while (size && (offset < x * 4)) {
 914                 uint32_t value;
 915
 916                 value = data[offset >> 2];
 917                 r = put_user(value, (uint32_t *)buf);
 918                 if (r) {
 919                         amdgpu_virt_disable_access_debugfs(adev);
 920                         return r;
 921                 }
 922
 923                 result += 4;
 924                 buf += 4;
 925                 offset += 4;
 926                 size -= 4;
 927         }
 928
 929         amdgpu_virt_disable_access_debugfs(adev);
 930         return result;
 931 }
 932
 933 /** amdgpu_debugfs_gpr_read - Read wave gprs
 934  *
 935  * @f: open file handle
 936  * @buf: User buffer to store read data in
 937  * @size: Number of bytes to read
 938  * @pos:  Offset to seek to
 939  *
 940  * The offset being sought changes which wave that the status data
 941  * will be returned for.  The bits are used as follows:
 942  *
 943  * Bits 0..11:  Byte offset into data
 944  * Bits 12..19: SE selector
 945  * Bits 20..27: SH/SA selector
 946  * Bits 28..35: CU/{WGP+SIMD} selector
 947  * Bits 36..43: WAVE ID selector
 948  * Bits 37..44: SIMD ID selector
 949  * Bits 52..59: Thread selector
 950  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 951  *
 952  * The return data comes from the SGPR or VGPR register bank for
 953  * the selected operational unit.
 954  */
 955 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 956                                         size_t size, loff_t *pos)
 957 {
 958         struct amdgpu_device *adev = f->f_inode->i_private;
 959         int r;
 960         ssize_t result = 0;
 961         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 962
 963         if (size > 4096 || size & 3 || *pos & 3)
 964                 return -EINVAL;
 965
 966         /* decode offset */
 967         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 968         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 969         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 970         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 971         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 972         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
 973         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
 974         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 975
 976         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 977         if (!data)
 978                 return -ENOMEM;
 979
 980         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 981         if (r < 0)
 982                 goto err;
 983
 984         r = amdgpu_virt_enable_access_debugfs(adev);
 985         if (r < 0)
 986                 goto err;
 987
 988         /* switch to the specific se/sh/cu */
 989         mutex_lock(&adev->grbm_idx_mutex);
 990         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 991
 992         if (bank == 0) {
 993                 if (adev->gfx.funcs->read_wave_vgprs)
 994                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
 995         } else {
 996                 if (adev->gfx.funcs->read_wave_sgprs)
 997                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
 998         }
 999
1000         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1001         mutex_unlock(&adev->grbm_idx_mutex);
1002
1003         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1004         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1005
1006         while (size) {
1007                 uint32_t value;
1008
1009                 value = data[result >> 2];
1010                 r = put_user(value, (uint32_t *)buf);
1011                 if (r) {
1012                         amdgpu_virt_disable_access_debugfs(adev);
1013                         goto err;
1014                 }
1015
1016                 result += 4;
1017                 buf += 4;
1018                 size -= 4;
1019         }
1020
1021         kfree(data);
1022         amdgpu_virt_disable_access_debugfs(adev);
1023         return result;
1024
1025 err:
1026         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1027         kfree(data);
1028         return r;
1029 }
1030
1031 /**
1032  * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
1033  *
1034  * @f: open file handle
1035  * @buf: User buffer to write data from
1036  * @size: Number of bytes to write
1037  * @pos:  Offset to seek to
1038  *
1039  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1040  */
1041 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1042                                          size_t size, loff_t *pos)
1043 {
1044         struct amdgpu_device *adev = file_inode(f)->i_private;
1045         ssize_t result = 0;
1046         int r;
1047
1048         if (size & 0x3 || *pos & 0x3)
1049                 return -EINVAL;
1050
1051         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1052         if (r < 0) {
1053                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1054                 return r;
1055         }
1056
1057         while (size) {
1058                 uint32_t value;
1059
1060                 r = get_user(value, (uint32_t *)buf);
1061                 if (r) {
1062                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1063                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1064                         return r;
1065                 }
1066
1067                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1068
1069                 result += 4;
1070                 buf += 4;
1071                 *pos += 4;
1072                 size -= 4;
1073         }
1074
1075         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1076         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1077
1078         return result;
1079 }
1080
1081
1082 /**
1083  * amdgpu_debugfs_regs_gfxoff_status - read gfxoff status
1084  *
1085  * @f: open file handle
1086  * @buf: User buffer to store read data in
1087  * @size: Number of bytes to read
1088  * @pos:  Offset to seek to
1089  */
1090 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1091                                          size_t size, loff_t *pos)
1092 {
1093         struct amdgpu_device *adev = file_inode(f)->i_private;
1094         ssize_t result = 0;
1095         int r;
1096
1097         if (size & 0x3 || *pos & 0x3)
1098                 return -EINVAL;
1099
1100         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1101         if (r < 0)
1102                 return r;
1103
1104         while (size) {
1105                 uint32_t value;
1106
1107                 r = amdgpu_get_gfx_off_status(adev, &value);
1108                 if (r) {
1109                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1110                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1111                         return r;
1112                 }
1113
1114                 r = put_user(value, (uint32_t *)buf);
1115                 if (r) {
1116                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1117                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1118                         return r;
1119                 }
1120
1121                 result += 4;
1122                 buf += 4;
1123                 *pos += 4;
1124                 size -= 4;
1125         }
1126
1127         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1128         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1129
1130         return result;
1131 }
1132
1133 static const struct file_operations amdgpu_debugfs_regs_fops = {
1134         .owner = THIS_MODULE,
1135         .read = amdgpu_debugfs_regs_read,
1136         .write = amdgpu_debugfs_regs_write,
1137         .llseek = default_llseek
1138 };
1139 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1140         .owner = THIS_MODULE,
1141         .read = amdgpu_debugfs_regs_didt_read,
1142         .write = amdgpu_debugfs_regs_didt_write,
1143         .llseek = default_llseek
1144 };
1145 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1146         .owner = THIS_MODULE,
1147         .read = amdgpu_debugfs_regs_pcie_read,
1148         .write = amdgpu_debugfs_regs_pcie_write,
1149         .llseek = default_llseek
1150 };
1151 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1152         .owner = THIS_MODULE,
1153         .read = amdgpu_debugfs_regs_smc_read,
1154         .write = amdgpu_debugfs_regs_smc_write,
1155         .llseek = default_llseek
1156 };
1157
1158 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1159         .owner = THIS_MODULE,
1160         .read = amdgpu_debugfs_gca_config_read,
1161         .llseek = default_llseek
1162 };
1163
1164 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1165         .owner = THIS_MODULE,
1166         .read = amdgpu_debugfs_sensor_read,
1167         .llseek = default_llseek
1168 };
1169
1170 static const struct file_operations amdgpu_debugfs_wave_fops = {
1171         .owner = THIS_MODULE,
1172         .read = amdgpu_debugfs_wave_read,
1173         .llseek = default_llseek
1174 };
1175 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1176         .owner = THIS_MODULE,
1177         .read = amdgpu_debugfs_gpr_read,
1178         .llseek = default_llseek
1179 };
1180
1181 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1182         .owner = THIS_MODULE,
1183         .read = amdgpu_debugfs_gfxoff_read,
1184         .write = amdgpu_debugfs_gfxoff_write,
1185         .llseek = default_llseek
1186 };
1187
1188 static const struct file_operations *debugfs_regs[] = {
1189         &amdgpu_debugfs_regs_fops,
1190         &amdgpu_debugfs_regs_didt_fops,
1191         &amdgpu_debugfs_regs_pcie_fops,
1192         &amdgpu_debugfs_regs_smc_fops,
1193         &amdgpu_debugfs_gca_config_fops,
1194         &amdgpu_debugfs_sensors_fops,
1195         &amdgpu_debugfs_wave_fops,
1196         &amdgpu_debugfs_gpr_fops,
1197         &amdgpu_debugfs_gfxoff_fops,
1198 };
1199
1200 static const char *debugfs_regs_names[] = {
1201         "amdgpu_regs",
1202         "amdgpu_regs_didt",
1203         "amdgpu_regs_pcie",
1204         "amdgpu_regs_smc",
1205         "amdgpu_gca_config",
1206         "amdgpu_sensors",
1207         "amdgpu_wave",
1208         "amdgpu_gpr",
1209         "amdgpu_gfxoff",
1210 };
1211
1212 /**
1213  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1214  *                              register access.
1215  *
1216  * @adev: The device to attach the debugfs entries to
1217  */
1218 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1219 {
1220         struct drm_minor *minor = adev_to_drm(adev)->primary;
1221         struct dentry *ent, *root = minor->debugfs_root;
1222         unsigned int i;
1223
1224         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1225                 ent = debugfs_create_file(debugfs_regs_names[i],
1226                                           S_IFREG | S_IRUGO, root,
1227                                           adev, debugfs_regs[i]);
1228                 if (!i && !IS_ERR_OR_NULL(ent))
1229                         i_size_write(ent->d_inode, adev->rmmio_size);
1230                 adev->debugfs_regs[i] = ent;
1231         }
1232
1233         return 0;
1234 }
1235
1236 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
1237 {
1238         struct drm_info_node *node = (struct drm_info_node *) m->private;
1239         struct drm_device *dev = node->minor->dev;
1240         struct amdgpu_device *adev = drm_to_adev(dev);
1241         int r = 0, i;
1242
1243         r = pm_runtime_get_sync(dev->dev);
1244         if (r < 0) {
1245                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1246                 return r;
1247         }
1248
1249         /* Avoid accidently unparking the sched thread during GPU reset */
1250         r = down_read_killable(&adev->reset_sem);
1251         if (r)
1252                 return r;
1253
1254         /* hold on the scheduler */
1255         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1256                 struct amdgpu_ring *ring = adev->rings[i];
1257
1258                 if (!ring || !ring->sched.thread)
1259                         continue;
1260                 kthread_park(ring->sched.thread);
1261         }
1262
1263         seq_printf(m, "run ib test:\n");
1264         r = amdgpu_ib_ring_tests(adev);
1265         if (r)
1266                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1267         else
1268                 seq_printf(m, "ib ring tests passed.\n");
1269
1270         /* go on the scheduler */
1271         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1272                 struct amdgpu_ring *ring = adev->rings[i];
1273
1274                 if (!ring || !ring->sched.thread)
1275                         continue;
1276                 kthread_unpark(ring->sched.thread);
1277         }
1278
1279         up_read(&adev->reset_sem);
1280
1281         pm_runtime_mark_last_busy(dev->dev);
1282         pm_runtime_put_autosuspend(dev->dev);
1283
1284         return 0;
1285 }
1286
1287 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1288 {
1289         struct drm_info_node *node = (struct drm_info_node *) m->private;
1290         struct drm_device *dev = node->minor->dev;
1291         struct amdgpu_device *adev = drm_to_adev(dev);
1292
1293         seq_write(m, adev->bios, adev->bios_size);
1294         return 0;
1295 }
1296
1297 static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1298 {
1299         struct drm_info_node *node = (struct drm_info_node *)m->private;
1300         struct drm_device *dev = node->minor->dev;
1301         struct amdgpu_device *adev = drm_to_adev(dev);
1302         int r;
1303
1304         r = pm_runtime_get_sync(dev->dev);
1305         if (r < 0) {
1306                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1307                 return r;
1308         }
1309
1310         seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1311
1312         pm_runtime_mark_last_busy(dev->dev);
1313         pm_runtime_put_autosuspend(dev->dev);
1314
1315         return 0;
1316 }
1317
1318 static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1319 {
1320         struct drm_info_node *node = (struct drm_info_node *)m->private;
1321         struct drm_device *dev = node->minor->dev;
1322         struct amdgpu_device *adev = drm_to_adev(dev);
1323         int r;
1324
1325         r = pm_runtime_get_sync(dev->dev);
1326         if (r < 0) {
1327                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1328                 return r;
1329         }
1330
1331         seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1332
1333         pm_runtime_mark_last_busy(dev->dev);
1334         pm_runtime_put_autosuspend(dev->dev);
1335
1336         return 0;
1337 }
1338
1339 static int amdgpu_debugfs_vm_info(struct seq_file *m, void *data)
1340 {
1341         struct drm_info_node *node = (struct drm_info_node *)m->private;
1342         struct drm_device *dev = node->minor->dev;
1343         struct drm_file *file;
1344         int r;
1345
1346         r = mutex_lock_interruptible(&dev->filelist_mutex);
1347         if (r)
1348                 return r;
1349
1350         list_for_each_entry(file, &dev->filelist, lhead) {
1351                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1352                 struct amdgpu_vm *vm = &fpriv->vm;
1353
1354                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1355                                 vm->task_info.pid, vm->task_info.process_name);
1356                 r = amdgpu_bo_reserve(vm->root.base.bo, true);
1357                 if (r)
1358                         break;
1359                 amdgpu_debugfs_vm_bo_info(vm, m);
1360                 amdgpu_bo_unreserve(vm->root.base.bo);
1361         }
1362
1363         mutex_unlock(&dev->filelist_mutex);
1364
1365         return r;
1366 }
1367
1368 static const struct drm_info_list amdgpu_debugfs_list[] = {
1369         {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1370         {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1371         {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1372         {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1373         {"amdgpu_vm_info", &amdgpu_debugfs_vm_info},
1374 };
1375
1376 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1377                                           struct dma_fence **fences)
1378 {
1379         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1380         uint32_t sync_seq, last_seq;
1381
1382         last_seq = atomic_read(&ring->fence_drv.last_seq);
1383         sync_seq = ring->fence_drv.sync_seq;
1384
1385         last_seq &= drv->num_fences_mask;
1386         sync_seq &= drv->num_fences_mask;
1387
1388         do {
1389                 struct dma_fence *fence, **ptr;
1390
1391                 ++last_seq;
1392                 last_seq &= drv->num_fences_mask;
1393                 ptr = &drv->fences[last_seq];
1394
1395                 fence = rcu_dereference_protected(*ptr, 1);
1396                 RCU_INIT_POINTER(*ptr, NULL);
1397
1398                 if (!fence)
1399                         continue;
1400
1401                 fences[last_seq] = fence;
1402
1403         } while (last_seq != sync_seq);
1404 }
1405
1406 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1407                                             int length)
1408 {
1409         int i;
1410         struct dma_fence *fence;
1411
1412         for (i = 0; i < length; i++) {
1413                 fence = fences[i];
1414                 if (!fence)
1415                         continue;
1416                 dma_fence_signal(fence);
1417                 dma_fence_put(fence);
1418         }
1419 }
1420
1421 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1422 {
1423         struct drm_sched_job *s_job;
1424         struct dma_fence *fence;
1425
1426         spin_lock(&sched->job_list_lock);
1427         list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1428                 fence = sched->ops->run_job(s_job);
1429                 dma_fence_put(fence);
1430         }
1431         spin_unlock(&sched->job_list_lock);
1432 }
1433
1434 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1435 {
1436         struct amdgpu_job *job;
1437         struct drm_sched_job *s_job, *tmp;
1438         uint32_t preempt_seq;
1439         struct dma_fence *fence, **ptr;
1440         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1441         struct drm_gpu_scheduler *sched = &ring->sched;
1442         bool preempted = true;
1443
1444         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1445                 return;
1446
1447         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1448         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1449                 preempted = false;
1450                 goto no_preempt;
1451         }
1452
1453         preempt_seq &= drv->num_fences_mask;
1454         ptr = &drv->fences[preempt_seq];
1455         fence = rcu_dereference_protected(*ptr, 1);
1456
1457 no_preempt:
1458         spin_lock(&sched->job_list_lock);
1459         list_for_each_entry_safe(s_job, tmp, &sched->ring_mirror_list, node) {
1460                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1461                         /* remove job from ring_mirror_list */
1462                         list_del_init(&s_job->node);
1463                         sched->ops->free_job(s_job);
1464                         continue;
1465                 }
1466                 job = to_amdgpu_job(s_job);
1467                 if (preempted && job->fence == fence)
1468                         /* mark the job as preempted */
1469                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1470         }
1471         spin_unlock(&sched->job_list_lock);
1472 }
1473
1474 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1475 {
1476         int r, resched, length;
1477         struct amdgpu_ring *ring;
1478         struct dma_fence **fences = NULL;
1479         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1480
1481         if (val >= AMDGPU_MAX_RINGS)
1482                 return -EINVAL;
1483
1484         ring = adev->rings[val];
1485
1486         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1487                 return -EINVAL;
1488
1489         /* the last preemption failed */
1490         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1491                 return -EBUSY;
1492
1493         length = ring->fence_drv.num_fences_mask + 1;
1494         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1495         if (!fences)
1496                 return -ENOMEM;
1497
1498         /* Avoid accidently unparking the sched thread during GPU reset */
1499         r = down_read_killable(&adev->reset_sem);
1500         if (r)
1501                 goto pro_end;
1502
1503         /* stop the scheduler */
1504         kthread_park(ring->sched.thread);
1505
1506         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1507
1508         /* preempt the IB */
1509         r = amdgpu_ring_preempt_ib(ring);
1510         if (r) {
1511                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1512                 goto failure;
1513         }
1514
1515         amdgpu_fence_process(ring);
1516
1517         if (atomic_read(&ring->fence_drv.last_seq) !=
1518             ring->fence_drv.sync_seq) {
1519                 DRM_INFO("ring %d was preempted\n", ring->idx);
1520
1521                 amdgpu_ib_preempt_mark_partial_job(ring);
1522
1523                 /* swap out the old fences */
1524                 amdgpu_ib_preempt_fences_swap(ring, fences);
1525
1526                 amdgpu_fence_driver_force_completion(ring);
1527
1528                 /* resubmit unfinished jobs */
1529                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1530
1531                 /* wait for jobs finished */
1532                 amdgpu_fence_wait_empty(ring);
1533
1534                 /* signal the old fences */
1535                 amdgpu_ib_preempt_signal_fences(fences, length);
1536         }
1537
1538 failure:
1539         /* restart the scheduler */
1540         kthread_unpark(ring->sched.thread);
1541
1542         up_read(&adev->reset_sem);
1543
1544         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1545
1546 pro_end:
1547         kfree(fences);
1548
1549         return r;
1550 }
1551
1552 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1553 {
1554         int ret = 0;
1555         uint32_t max_freq, min_freq;
1556         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1557
1558         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1559                 return -EINVAL;
1560
1561         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1562         if (ret < 0) {
1563                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1564                 return ret;
1565         }
1566
1567         if (is_support_sw_smu(adev)) {
1568                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1569                 if (ret || val > max_freq || val < min_freq)
1570                         return -EINVAL;
1571                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1572         } else {
1573                 return 0;
1574         }
1575
1576         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1577         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1578
1579         if (ret)
1580                 return -EINVAL;
1581
1582         return 0;
1583 }
1584
1585 DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1586                         amdgpu_debugfs_ib_preempt, "%llu\n");
1587
1588 DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
1589                         amdgpu_debugfs_sclk_set, "%llu\n");
1590
1591 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1592 {
1593         int r, i;
1594
1595         adev->debugfs_preempt =
1596                 debugfs_create_file("amdgpu_preempt_ib", 0600,
1597                                     adev_to_drm(adev)->primary->debugfs_root, adev,
1598                                     &fops_ib_preempt);
1599         if (!(adev->debugfs_preempt)) {
1600                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1601                 return -EIO;
1602         }
1603
1604         adev->smu.debugfs_sclk =
1605                 debugfs_create_file("amdgpu_force_sclk", 0200,
1606                                     adev_to_drm(adev)->primary->debugfs_root, adev,
1607                                     &fops_sclk_set);
1608         if (!(adev->smu.debugfs_sclk)) {
1609                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1610                 return -EIO;
1611         }
1612
1613         /* Register debugfs entries for amdgpu_ttm */
1614         r = amdgpu_ttm_debugfs_init(adev);
1615         if (r) {
1616                 DRM_ERROR("Failed to init debugfs\n");
1617                 return r;
1618         }
1619
1620         r = amdgpu_debugfs_pm_init(adev);
1621         if (r) {
1622                 DRM_ERROR("Failed to register debugfs file for dpm!\n");
1623                 return r;
1624         }
1625
1626         if (amdgpu_debugfs_sa_init(adev)) {
1627                 dev_err(adev->dev, "failed to register debugfs file for SA\n");
1628         }
1629
1630         if (amdgpu_debugfs_fence_init(adev))
1631                 dev_err(adev->dev, "fence debugfs file creation failed\n");
1632
1633         r = amdgpu_debugfs_gem_init(adev);
1634         if (r)
1635                 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1636
1637         r = amdgpu_debugfs_regs_init(adev);
1638         if (r)
1639                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1640
1641         r = amdgpu_debugfs_firmware_init(adev);
1642         if (r)
1643                 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1644
1645 #if defined(CONFIG_DRM_AMD_DC)
1646         if (amdgpu_device_has_dc_support(adev)) {
1647                 if (dtn_debugfs_init(adev))
1648                         DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
1649         }
1650 #endif
1651
1652         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1653                 struct amdgpu_ring *ring = adev->rings[i];
1654
1655                 if (!ring)
1656                         continue;
1657
1658                 if (amdgpu_debugfs_ring_init(adev, ring)) {
1659                         DRM_ERROR("Failed to register debugfs file for rings !\n");
1660                 }
1661         }
1662
1663         amdgpu_ras_debugfs_create_all(adev);
1664
1665         amdgpu_debugfs_autodump_init(adev);
1666
1667         amdgpu_rap_debugfs_init(adev);
1668
1669         amdgpu_fw_attestation_debugfs_init(adev);
1670
1671         return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1672                                         ARRAY_SIZE(amdgpu_debugfs_list));
1673 }
1674
1675 #else
1676 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1677 {
1678         return 0;
1679 }
1680 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1681 {
1682         return 0;
1683 }
1684 #endif