drivers/gpu/drm/amd/amdgpu/amdgpu_debugfs.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  */
  25
  26 #include <linux/kthread.h>
  27 #include <linux/pci.h>
  28 #include <linux/uaccess.h>
  29 #include <linux/pm_runtime.h>
  30
  31 #include "amdgpu.h"
  32 #include "amdgpu_pm.h"
  33 #include "amdgpu_dm_debugfs.h"
  34 #include "amdgpu_ras.h"
  35 #include "amdgpu_rap.h"
  36 #include "amdgpu_securedisplay.h"
  37 #include "amdgpu_fw_attestation.h"
  38 #include "amdgpu_umr.h"
  39
  40 #if defined(CONFIG_DEBUG_FS)
  41
  42 /**
  43  * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
  44  *
  45  * @read: True if reading
  46  * @f: open file handle
  47  * @buf: User buffer to write/read to
  48  * @size: Number of bytes to write/read
  49  * @pos:  Offset to seek to
  50  *
  51  * This debugfs entry has special meaning on the offset being sought.
  52  * Various bits have different meanings:
  53  *
  54  * Bit 62:  Indicates a GRBM bank switch is needed
  55  * Bit 61:  Indicates a SRBM bank switch is needed (implies bit 62 is
  56  *          zero)
  57  * Bits 24..33: The SE or ME selector if needed
  58  * Bits 34..43: The SH (or SA) or PIPE selector if needed
  59  * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
  60  *
  61  * Bit 23:  Indicates that the PM power gating lock should be held
  62  *          This is necessary to read registers that might be
  63  *          unreliable during a power gating transistion.
  64  *
  65  * The lower bits are the BYTE offset of the register to read.  This
  66  * allows reading multiple registers in a single call and having
  67  * the returned size reflect that.
  68  */
  69 static int  amdgpu_debugfs_process_reg_op(bool read, struct file *f,
  70                 char __user *buf, size_t size, loff_t *pos)
  71 {
  72         struct amdgpu_device *adev = file_inode(f)->i_private;
  73         ssize_t result = 0;
  74         int r;
  75         bool pm_pg_lock, use_bank, use_ring;
  76         unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
  77
  78         pm_pg_lock = use_bank = use_ring = false;
  79         instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
  80
  81         if (size & 0x3 || *pos & 0x3 ||
  82                         ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
  83                 return -EINVAL;
  84
  85         /* are we reading registers for which a PG lock is necessary? */
  86         pm_pg_lock = (*pos >> 23) & 1;
  87
  88         if (*pos & (1ULL << 62)) {
  89                 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
  90                 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
  91                 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
  92
  93                 if (se_bank == 0x3FF)
  94                         se_bank = 0xFFFFFFFF;
  95                 if (sh_bank == 0x3FF)
  96                         sh_bank = 0xFFFFFFFF;
  97                 if (instance_bank == 0x3FF)
  98                         instance_bank = 0xFFFFFFFF;
  99                 use_bank = true;
 100         } else if (*pos & (1ULL << 61)) {
 101
 102                 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
 103                 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
 104                 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
 105                 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
 106
 107                 use_ring = true;
 108         } else {
 109                 use_bank = use_ring = false;
 110         }
 111
 112         *pos &= (1UL << 22) - 1;
 113
 114         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 115         if (r < 0) {
 116                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 117                 return r;
 118         }
 119
 120         r = amdgpu_virt_enable_access_debugfs(adev);
 121         if (r < 0) {
 122                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 123                 return r;
 124         }
 125
 126         if (use_bank) {
 127                 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
 128                     (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
 129                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 130                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 131                         amdgpu_virt_disable_access_debugfs(adev);
 132                         return -EINVAL;
 133                 }
 134                 mutex_lock(&adev->grbm_idx_mutex);
 135                 amdgpu_gfx_select_se_sh(adev, se_bank,
 136                                         sh_bank, instance_bank);
 137         } else if (use_ring) {
 138                 mutex_lock(&adev->srbm_mutex);
 139                 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
 140         }
 141
 142         if (pm_pg_lock)
 143                 mutex_lock(&adev->pm.mutex);
 144
 145         while (size) {
 146                 uint32_t value;
 147
 148                 if (read) {
 149                         value = RREG32(*pos >> 2);
 150                         r = put_user(value, (uint32_t *)buf);
 151                 } else {
 152                         r = get_user(value, (uint32_t *)buf);
 153                         if (!r)
 154                                 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value);
 155                 }
 156                 if (r) {
 157                         result = r;
 158                         goto end;
 159                 }
 160
 161                 result += 4;
 162                 buf += 4;
 163                 *pos += 4;
 164                 size -= 4;
 165         }
 166
 167 end:
 168         if (use_bank) {
 169                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 170                 mutex_unlock(&adev->grbm_idx_mutex);
 171         } else if (use_ring) {
 172                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 173                 mutex_unlock(&adev->srbm_mutex);
 174         }
 175
 176         if (pm_pg_lock)
 177                 mutex_unlock(&adev->pm.mutex);
 178
 179         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 180         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 181
 182         amdgpu_virt_disable_access_debugfs(adev);
 183         return result;
 184 }
 185
 186 /*
 187  * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
 188  */
 189 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
 190                                         size_t size, loff_t *pos)
 191 {
 192         return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
 193 }
 194
 195 /*
 196  * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
 197  */
 198 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
 199                                          size_t size, loff_t *pos)
 200 {
 201         return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
 202 }
 203
 204 static int amdgpu_debugfs_regs2_open(struct inode *inode, struct file *file)
 205 {
 206         struct amdgpu_debugfs_regs2_data *rd;
 207
 208         rd = kzalloc(sizeof *rd, GFP_KERNEL);
 209         if (!rd)
 210                 return -ENOMEM;
 211         rd->adev = file_inode(file)->i_private;
 212         file->private_data = rd;
 213         mutex_init(&rd->lock);
 214
 215         return 0;
 216 }
 217
 218 static int amdgpu_debugfs_regs2_release(struct inode *inode, struct file *file)
 219 {
 220         struct amdgpu_debugfs_regs2_data *rd = file->private_data;
 221         mutex_destroy(&rd->lock);
 222         kfree(file->private_data);
 223         return 0;
 224 }
 225
 226 static ssize_t amdgpu_debugfs_regs2_op(struct file *f, char __user *buf, u32 offset, size_t size, int write_en)
 227 {
 228         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 229         struct amdgpu_device *adev = rd->adev;
 230         ssize_t result = 0;
 231         int r;
 232         uint32_t value;
 233
 234         if (size & 0x3 || offset & 0x3)
 235                 return -EINVAL;
 236
 237         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 238         if (r < 0) {
 239                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 240                 return r;
 241         }
 242
 243         r = amdgpu_virt_enable_access_debugfs(adev);
 244         if (r < 0) {
 245                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 246                 return r;
 247         }
 248
 249         mutex_lock(&rd->lock);
 250
 251         if (rd->id.use_grbm) {
 252                 if ((rd->id.grbm.sh != 0xFFFFFFFF && rd->id.grbm.sh >= adev->gfx.config.max_sh_per_se) ||
 253                     (rd->id.grbm.se != 0xFFFFFFFF && rd->id.grbm.se >= adev->gfx.config.max_shader_engines)) {
 254                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 255                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 256                         amdgpu_virt_disable_access_debugfs(adev);
 257                         mutex_unlock(&rd->lock);
 258                         return -EINVAL;
 259                 }
 260                 mutex_lock(&adev->grbm_idx_mutex);
 261                 amdgpu_gfx_select_se_sh(adev, rd->id.grbm.se,
 262                                                                 rd->id.grbm.sh,
 263                                                                 rd->id.grbm.instance);
 264         }
 265
 266         if (rd->id.use_srbm) {
 267                 mutex_lock(&adev->srbm_mutex);
 268                 amdgpu_gfx_select_me_pipe_q(adev, rd->id.srbm.me, rd->id.srbm.pipe,
 269                                                                         rd->id.srbm.queue, rd->id.srbm.vmid);
 270         }
 271
 272         if (rd->id.pg_lock)
 273                 mutex_lock(&adev->pm.mutex);
 274
 275         while (size) {
 276                 if (!write_en) {
 277                         value = RREG32(offset >> 2);
 278                         r = put_user(value, (uint32_t *)buf);
 279                 } else {
 280                         r = get_user(value, (uint32_t *)buf);
 281                         if (!r)
 282                                 amdgpu_mm_wreg_mmio_rlc(adev, offset >> 2, value);
 283                 }
 284                 if (r) {
 285                         result = r;
 286                         goto end;
 287                 }
 288                 offset += 4;
 289                 size -= 4;
 290                 result += 4;
 291                 buf += 4;
 292         }
 293 end:
 294         if (rd->id.use_grbm) {
 295                 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
 296                 mutex_unlock(&adev->grbm_idx_mutex);
 297         }
 298
 299         if (rd->id.use_srbm) {
 300                 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
 301                 mutex_unlock(&adev->srbm_mutex);
 302         }
 303
 304         if (rd->id.pg_lock)
 305                 mutex_unlock(&adev->pm.mutex);
 306
 307         mutex_unlock(&rd->lock);
 308
 309         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 310         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 311
 312         amdgpu_virt_disable_access_debugfs(adev);
 313         return result;
 314 }
 315
 316 static long amdgpu_debugfs_regs2_ioctl(struct file *f, unsigned int cmd, unsigned long data)
 317 {
 318         struct amdgpu_debugfs_regs2_data *rd = f->private_data;
 319         int r;
 320
 321         switch (cmd) {
 322         case AMDGPU_DEBUGFS_REGS2_IOC_SET_STATE:
 323                 mutex_lock(&rd->lock);
 324                 r = copy_from_user(&rd->id, (struct amdgpu_debugfs_regs2_iocdata *)data, sizeof rd->id);
 325                 mutex_unlock(&rd->lock);
 326                 return r ? -EINVAL : 0;
 327         default:
 328                 return -EINVAL;
 329         }
 330         return 0;
 331 }
 332
 333 static ssize_t amdgpu_debugfs_regs2_read(struct file *f, char __user *buf, size_t size, loff_t *pos)
 334 {
 335         return amdgpu_debugfs_regs2_op(f, buf, *pos, size, 0);
 336 }
 337
 338 static ssize_t amdgpu_debugfs_regs2_write(struct file *f, const char __user *buf, size_t size, loff_t *pos)
 339 {
 340         return amdgpu_debugfs_regs2_op(f, (char __user *)buf, *pos, size, 1);
 341 }
 342
 343
 344 /**
 345  * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
 346  *
 347  * @f: open file handle
 348  * @buf: User buffer to store read data in
 349  * @size: Number of bytes to read
 350  * @pos:  Offset to seek to
 351  *
 352  * The lower bits are the BYTE offset of the register to read.  This
 353  * allows reading multiple registers in a single call and having
 354  * the returned size reflect that.
 355  */
 356 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
 357                                         size_t size, loff_t *pos)
 358 {
 359         struct amdgpu_device *adev = file_inode(f)->i_private;
 360         ssize_t result = 0;
 361         int r;
 362
 363         if (size & 0x3 || *pos & 0x3)
 364                 return -EINVAL;
 365
 366         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 367         if (r < 0) {
 368                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 369                 return r;
 370         }
 371
 372         r = amdgpu_virt_enable_access_debugfs(adev);
 373         if (r < 0) {
 374                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 375                 return r;
 376         }
 377
 378         while (size) {
 379                 uint32_t value;
 380
 381                 value = RREG32_PCIE(*pos);
 382                 r = put_user(value, (uint32_t *)buf);
 383                 if (r) {
 384                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 385                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 386                         amdgpu_virt_disable_access_debugfs(adev);
 387                         return r;
 388                 }
 389
 390                 result += 4;
 391                 buf += 4;
 392                 *pos += 4;
 393                 size -= 4;
 394         }
 395
 396         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 397         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 398
 399         amdgpu_virt_disable_access_debugfs(adev);
 400         return result;
 401 }
 402
 403 /**
 404  * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
 405  *
 406  * @f: open file handle
 407  * @buf: User buffer to write data from
 408  * @size: Number of bytes to write
 409  * @pos:  Offset to seek to
 410  *
 411  * The lower bits are the BYTE offset of the register to write.  This
 412  * allows writing multiple registers in a single call and having
 413  * the returned size reflect that.
 414  */
 415 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
 416                                          size_t size, loff_t *pos)
 417 {
 418         struct amdgpu_device *adev = file_inode(f)->i_private;
 419         ssize_t result = 0;
 420         int r;
 421
 422         if (size & 0x3 || *pos & 0x3)
 423                 return -EINVAL;
 424
 425         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 426         if (r < 0) {
 427                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 428                 return r;
 429         }
 430
 431         r = amdgpu_virt_enable_access_debugfs(adev);
 432         if (r < 0) {
 433                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 434                 return r;
 435         }
 436
 437         while (size) {
 438                 uint32_t value;
 439
 440                 r = get_user(value, (uint32_t *)buf);
 441                 if (r) {
 442                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 443                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 444                         amdgpu_virt_disable_access_debugfs(adev);
 445                         return r;
 446                 }
 447
 448                 WREG32_PCIE(*pos, value);
 449
 450                 result += 4;
 451                 buf += 4;
 452                 *pos += 4;
 453                 size -= 4;
 454         }
 455
 456         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 457         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 458
 459         amdgpu_virt_disable_access_debugfs(adev);
 460         return result;
 461 }
 462
 463 /**
 464  * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
 465  *
 466  * @f: open file handle
 467  * @buf: User buffer to store read data in
 468  * @size: Number of bytes to read
 469  * @pos:  Offset to seek to
 470  *
 471  * The lower bits are the BYTE offset of the register to read.  This
 472  * allows reading multiple registers in a single call and having
 473  * the returned size reflect that.
 474  */
 475 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
 476                                         size_t size, loff_t *pos)
 477 {
 478         struct amdgpu_device *adev = file_inode(f)->i_private;
 479         ssize_t result = 0;
 480         int r;
 481
 482         if (size & 0x3 || *pos & 0x3)
 483                 return -EINVAL;
 484
 485         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 486         if (r < 0) {
 487                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 488                 return r;
 489         }
 490
 491         r = amdgpu_virt_enable_access_debugfs(adev);
 492         if (r < 0) {
 493                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 494                 return r;
 495         }
 496
 497         while (size) {
 498                 uint32_t value;
 499
 500                 value = RREG32_DIDT(*pos >> 2);
 501                 r = put_user(value, (uint32_t *)buf);
 502                 if (r) {
 503                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 504                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 505                         amdgpu_virt_disable_access_debugfs(adev);
 506                         return r;
 507                 }
 508
 509                 result += 4;
 510                 buf += 4;
 511                 *pos += 4;
 512                 size -= 4;
 513         }
 514
 515         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 516         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 517
 518         amdgpu_virt_disable_access_debugfs(adev);
 519         return result;
 520 }
 521
 522 /**
 523  * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
 524  *
 525  * @f: open file handle
 526  * @buf: User buffer to write data from
 527  * @size: Number of bytes to write
 528  * @pos:  Offset to seek to
 529  *
 530  * The lower bits are the BYTE offset of the register to write.  This
 531  * allows writing multiple registers in a single call and having
 532  * the returned size reflect that.
 533  */
 534 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
 535                                          size_t size, loff_t *pos)
 536 {
 537         struct amdgpu_device *adev = file_inode(f)->i_private;
 538         ssize_t result = 0;
 539         int r;
 540
 541         if (size & 0x3 || *pos & 0x3)
 542                 return -EINVAL;
 543
 544         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 545         if (r < 0) {
 546                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 547                 return r;
 548         }
 549
 550         r = amdgpu_virt_enable_access_debugfs(adev);
 551         if (r < 0) {
 552                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 553                 return r;
 554         }
 555
 556         while (size) {
 557                 uint32_t value;
 558
 559                 r = get_user(value, (uint32_t *)buf);
 560                 if (r) {
 561                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 562                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 563                         amdgpu_virt_disable_access_debugfs(adev);
 564                         return r;
 565                 }
 566
 567                 WREG32_DIDT(*pos >> 2, value);
 568
 569                 result += 4;
 570                 buf += 4;
 571                 *pos += 4;
 572                 size -= 4;
 573         }
 574
 575         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 576         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 577
 578         amdgpu_virt_disable_access_debugfs(adev);
 579         return result;
 580 }
 581
 582 /**
 583  * amdgpu_debugfs_regs_smc_read - Read from a SMC register
 584  *
 585  * @f: open file handle
 586  * @buf: User buffer to store read data in
 587  * @size: Number of bytes to read
 588  * @pos:  Offset to seek to
 589  *
 590  * The lower bits are the BYTE offset of the register to read.  This
 591  * allows reading multiple registers in a single call and having
 592  * the returned size reflect that.
 593  */
 594 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
 595                                         size_t size, loff_t *pos)
 596 {
 597         struct amdgpu_device *adev = file_inode(f)->i_private;
 598         ssize_t result = 0;
 599         int r;
 600
 601         if (size & 0x3 || *pos & 0x3)
 602                 return -EINVAL;
 603
 604         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 605         if (r < 0) {
 606                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 607                 return r;
 608         }
 609
 610         r = amdgpu_virt_enable_access_debugfs(adev);
 611         if (r < 0) {
 612                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 613                 return r;
 614         }
 615
 616         while (size) {
 617                 uint32_t value;
 618
 619                 value = RREG32_SMC(*pos);
 620                 r = put_user(value, (uint32_t *)buf);
 621                 if (r) {
 622                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 623                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 624                         amdgpu_virt_disable_access_debugfs(adev);
 625                         return r;
 626                 }
 627
 628                 result += 4;
 629                 buf += 4;
 630                 *pos += 4;
 631                 size -= 4;
 632         }
 633
 634         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 635         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 636
 637         amdgpu_virt_disable_access_debugfs(adev);
 638         return result;
 639 }
 640
 641 /**
 642  * amdgpu_debugfs_regs_smc_write - Write to a SMC register
 643  *
 644  * @f: open file handle
 645  * @buf: User buffer to write data from
 646  * @size: Number of bytes to write
 647  * @pos:  Offset to seek to
 648  *
 649  * The lower bits are the BYTE offset of the register to write.  This
 650  * allows writing multiple registers in a single call and having
 651  * the returned size reflect that.
 652  */
 653 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
 654                                          size_t size, loff_t *pos)
 655 {
 656         struct amdgpu_device *adev = file_inode(f)->i_private;
 657         ssize_t result = 0;
 658         int r;
 659
 660         if (size & 0x3 || *pos & 0x3)
 661                 return -EINVAL;
 662
 663         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 664         if (r < 0) {
 665                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 666                 return r;
 667         }
 668
 669         r = amdgpu_virt_enable_access_debugfs(adev);
 670         if (r < 0) {
 671                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 672                 return r;
 673         }
 674
 675         while (size) {
 676                 uint32_t value;
 677
 678                 r = get_user(value, (uint32_t *)buf);
 679                 if (r) {
 680                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 681                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 682                         amdgpu_virt_disable_access_debugfs(adev);
 683                         return r;
 684                 }
 685
 686                 WREG32_SMC(*pos, value);
 687
 688                 result += 4;
 689                 buf += 4;
 690                 *pos += 4;
 691                 size -= 4;
 692         }
 693
 694         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 695         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 696
 697         amdgpu_virt_disable_access_debugfs(adev);
 698         return result;
 699 }
 700
 701 /**
 702  * amdgpu_debugfs_gca_config_read - Read from gfx config data
 703  *
 704  * @f: open file handle
 705  * @buf: User buffer to store read data in
 706  * @size: Number of bytes to read
 707  * @pos:  Offset to seek to
 708  *
 709  * This file is used to access configuration data in a somewhat
 710  * stable fashion.  The format is a series of DWORDs with the first
 711  * indicating which revision it is.  New content is appended to the
 712  * end so that older software can still read the data.
 713  */
 714
 715 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
 716                                         size_t size, loff_t *pos)
 717 {
 718         struct amdgpu_device *adev = file_inode(f)->i_private;
 719         ssize_t result = 0;
 720         int r;
 721         uint32_t *config, no_regs = 0;
 722
 723         if (size & 0x3 || *pos & 0x3)
 724                 return -EINVAL;
 725
 726         config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
 727         if (!config)
 728                 return -ENOMEM;
 729
 730         /* version, increment each time something is added */
 731         config[no_regs++] = 3;
 732         config[no_regs++] = adev->gfx.config.max_shader_engines;
 733         config[no_regs++] = adev->gfx.config.max_tile_pipes;
 734         config[no_regs++] = adev->gfx.config.max_cu_per_sh;
 735         config[no_regs++] = adev->gfx.config.max_sh_per_se;
 736         config[no_regs++] = adev->gfx.config.max_backends_per_se;
 737         config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
 738         config[no_regs++] = adev->gfx.config.max_gprs;
 739         config[no_regs++] = adev->gfx.config.max_gs_threads;
 740         config[no_regs++] = adev->gfx.config.max_hw_contexts;
 741         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
 742         config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
 743         config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
 744         config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
 745         config[no_regs++] = adev->gfx.config.num_tile_pipes;
 746         config[no_regs++] = adev->gfx.config.backend_enable_mask;
 747         config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
 748         config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
 749         config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
 750         config[no_regs++] = adev->gfx.config.num_gpus;
 751         config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
 752         config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
 753         config[no_regs++] = adev->gfx.config.gb_addr_config;
 754         config[no_regs++] = adev->gfx.config.num_rbs;
 755
 756         /* rev==1 */
 757         config[no_regs++] = adev->rev_id;
 758         config[no_regs++] = adev->pg_flags;
 759         config[no_regs++] = adev->cg_flags;
 760
 761         /* rev==2 */
 762         config[no_regs++] = adev->family;
 763         config[no_regs++] = adev->external_rev_id;
 764
 765         /* rev==3 */
 766         config[no_regs++] = adev->pdev->device;
 767         config[no_regs++] = adev->pdev->revision;
 768         config[no_regs++] = adev->pdev->subsystem_device;
 769         config[no_regs++] = adev->pdev->subsystem_vendor;
 770
 771         while (size && (*pos < no_regs * 4)) {
 772                 uint32_t value;
 773
 774                 value = config[*pos >> 2];
 775                 r = put_user(value, (uint32_t *)buf);
 776                 if (r) {
 777                         kfree(config);
 778                         return r;
 779                 }
 780
 781                 result += 4;
 782                 buf += 4;
 783                 *pos += 4;
 784                 size -= 4;
 785         }
 786
 787         kfree(config);
 788         return result;
 789 }
 790
 791 /**
 792  * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
 793  *
 794  * @f: open file handle
 795  * @buf: User buffer to store read data in
 796  * @size: Number of bytes to read
 797  * @pos:  Offset to seek to
 798  *
 799  * The offset is treated as the BYTE address of one of the sensors
 800  * enumerated in amd/include/kgd_pp_interface.h under the
 801  * 'amd_pp_sensors' enumeration.  For instance to read the UVD VCLK
 802  * you would use the offset 3 * 4 = 12.
 803  */
 804 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
 805                                         size_t size, loff_t *pos)
 806 {
 807         struct amdgpu_device *adev = file_inode(f)->i_private;
 808         int idx, x, outsize, r, valuesize;
 809         uint32_t values[16];
 810
 811         if (size & 3 || *pos & 0x3)
 812                 return -EINVAL;
 813
 814         if (!adev->pm.dpm_enabled)
 815                 return -EINVAL;
 816
 817         /* convert offset to sensor number */
 818         idx = *pos >> 2;
 819
 820         valuesize = sizeof(values);
 821
 822         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 823         if (r < 0) {
 824                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 825                 return r;
 826         }
 827
 828         r = amdgpu_virt_enable_access_debugfs(adev);
 829         if (r < 0) {
 830                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 831                 return r;
 832         }
 833
 834         r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
 835
 836         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 837         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 838
 839         if (r) {
 840                 amdgpu_virt_disable_access_debugfs(adev);
 841                 return r;
 842         }
 843
 844         if (size > valuesize) {
 845                 amdgpu_virt_disable_access_debugfs(adev);
 846                 return -EINVAL;
 847         }
 848
 849         outsize = 0;
 850         x = 0;
 851         if (!r) {
 852                 while (size) {
 853                         r = put_user(values[x++], (int32_t *)buf);
 854                         buf += 4;
 855                         size -= 4;
 856                         outsize += 4;
 857                 }
 858         }
 859
 860         amdgpu_virt_disable_access_debugfs(adev);
 861         return !r ? outsize : r;
 862 }
 863
 864 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
 865  *
 866  * @f: open file handle
 867  * @buf: User buffer to store read data in
 868  * @size: Number of bytes to read
 869  * @pos:  Offset to seek to
 870  *
 871  * The offset being sought changes which wave that the status data
 872  * will be returned for.  The bits are used as follows:
 873  *
 874  * Bits 0..6:   Byte offset into data
 875  * Bits 7..14:  SE selector
 876  * Bits 15..22: SH/SA selector
 877  * Bits 23..30: CU/{WGP+SIMD} selector
 878  * Bits 31..36: WAVE ID selector
 879  * Bits 37..44: SIMD ID selector
 880  *
 881  * The returned data begins with one DWORD of version information
 882  * Followed by WAVE STATUS registers relevant to the GFX IP version
 883  * being used.  See gfx_v8_0_read_wave_data() for an example output.
 884  */
 885 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
 886                                         size_t size, loff_t *pos)
 887 {
 888         struct amdgpu_device *adev = f->f_inode->i_private;
 889         int r, x;
 890         ssize_t result = 0;
 891         uint32_t offset, se, sh, cu, wave, simd, data[32];
 892
 893         if (size & 3 || *pos & 3)
 894                 return -EINVAL;
 895
 896         /* decode offset */
 897         offset = (*pos & GENMASK_ULL(6, 0));
 898         se = (*pos & GENMASK_ULL(14, 7)) >> 7;
 899         sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
 900         cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
 901         wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
 902         simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
 903
 904         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
 905         if (r < 0) {
 906                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 907                 return r;
 908         }
 909
 910         r = amdgpu_virt_enable_access_debugfs(adev);
 911         if (r < 0) {
 912                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 913                 return r;
 914         }
 915
 916         /* switch to the specific se/sh/cu */
 917         mutex_lock(&adev->grbm_idx_mutex);
 918         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
 919
 920         x = 0;
 921         if (adev->gfx.funcs->read_wave_data)
 922                 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
 923
 924         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
 925         mutex_unlock(&adev->grbm_idx_mutex);
 926
 927         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
 928         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
 929
 930         if (!x) {
 931                 amdgpu_virt_disable_access_debugfs(adev);
 932                 return -EINVAL;
 933         }
 934
 935         while (size && (offset < x * 4)) {
 936                 uint32_t value;
 937
 938                 value = data[offset >> 2];
 939                 r = put_user(value, (uint32_t *)buf);
 940                 if (r) {
 941                         amdgpu_virt_disable_access_debugfs(adev);
 942                         return r;
 943                 }
 944
 945                 result += 4;
 946                 buf += 4;
 947                 offset += 4;
 948                 size -= 4;
 949         }
 950
 951         amdgpu_virt_disable_access_debugfs(adev);
 952         return result;
 953 }
 954
 955 /** amdgpu_debugfs_gpr_read - Read wave gprs
 956  *
 957  * @f: open file handle
 958  * @buf: User buffer to store read data in
 959  * @size: Number of bytes to read
 960  * @pos:  Offset to seek to
 961  *
 962  * The offset being sought changes which wave that the status data
 963  * will be returned for.  The bits are used as follows:
 964  *
 965  * Bits 0..11:  Byte offset into data
 966  * Bits 12..19: SE selector
 967  * Bits 20..27: SH/SA selector
 968  * Bits 28..35: CU/{WGP+SIMD} selector
 969  * Bits 36..43: WAVE ID selector
 970  * Bits 37..44: SIMD ID selector
 971  * Bits 52..59: Thread selector
 972  * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
 973  *
 974  * The return data comes from the SGPR or VGPR register bank for
 975  * the selected operational unit.
 976  */
 977 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
 978                                         size_t size, loff_t *pos)
 979 {
 980         struct amdgpu_device *adev = f->f_inode->i_private;
 981         int r;
 982         ssize_t result = 0;
 983         uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
 984
 985         if (size > 4096 || size & 3 || *pos & 3)
 986                 return -EINVAL;
 987
 988         /* decode offset */
 989         offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
 990         se = (*pos & GENMASK_ULL(19, 12)) >> 12;
 991         sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
 992         cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
 993         wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
 994         simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
 995         thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
 996         bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
 997
 998         data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
 999         if (!data)
1000                 return -ENOMEM;
1001
1002         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1003         if (r < 0)
1004                 goto err;
1005
1006         r = amdgpu_virt_enable_access_debugfs(adev);
1007         if (r < 0)
1008                 goto err;
1009
1010         /* switch to the specific se/sh/cu */
1011         mutex_lock(&adev->grbm_idx_mutex);
1012         amdgpu_gfx_select_se_sh(adev, se, sh, cu);
1013
1014         if (bank == 0) {
1015                 if (adev->gfx.funcs->read_wave_vgprs)
1016                         adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
1017         } else {
1018                 if (adev->gfx.funcs->read_wave_sgprs)
1019                         adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
1020         }
1021
1022         amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
1023         mutex_unlock(&adev->grbm_idx_mutex);
1024
1025         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1026         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1027
1028         while (size) {
1029                 uint32_t value;
1030
1031                 value = data[result >> 2];
1032                 r = put_user(value, (uint32_t *)buf);
1033                 if (r) {
1034                         amdgpu_virt_disable_access_debugfs(adev);
1035                         goto err;
1036                 }
1037
1038                 result += 4;
1039                 buf += 4;
1040                 size -= 4;
1041         }
1042
1043         kfree(data);
1044         amdgpu_virt_disable_access_debugfs(adev);
1045         return result;
1046
1047 err:
1048         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1049         kfree(data);
1050         return r;
1051 }
1052
1053 /**
1054  * amdgpu_debugfs_gfxoff_write - Enable/disable GFXOFF
1055  *
1056  * @f: open file handle
1057  * @buf: User buffer to write data from
1058  * @size: Number of bytes to write
1059  * @pos:  Offset to seek to
1060  *
1061  * Write a 32-bit zero to disable or a 32-bit non-zero to enable
1062  */
1063 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
1064                                          size_t size, loff_t *pos)
1065 {
1066         struct amdgpu_device *adev = file_inode(f)->i_private;
1067         ssize_t result = 0;
1068         int r;
1069
1070         if (size & 0x3 || *pos & 0x3)
1071                 return -EINVAL;
1072
1073         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1074         if (r < 0) {
1075                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1076                 return r;
1077         }
1078
1079         while (size) {
1080                 uint32_t value;
1081
1082                 r = get_user(value, (uint32_t *)buf);
1083                 if (r) {
1084                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1085                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1086                         return r;
1087                 }
1088
1089                 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1090
1091                 result += 4;
1092                 buf += 4;
1093                 *pos += 4;
1094                 size -= 4;
1095         }
1096
1097         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1098         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1099
1100         return result;
1101 }
1102
1103
1104 /**
1105  * amdgpu_debugfs_gfxoff_read - read gfxoff status
1106  *
1107  * @f: open file handle
1108  * @buf: User buffer to store read data in
1109  * @size: Number of bytes to read
1110  * @pos:  Offset to seek to
1111  */
1112 static ssize_t amdgpu_debugfs_gfxoff_read(struct file *f, char __user *buf,
1113                                          size_t size, loff_t *pos)
1114 {
1115         struct amdgpu_device *adev = file_inode(f)->i_private;
1116         ssize_t result = 0;
1117         int r;
1118
1119         if (size & 0x3 || *pos & 0x3)
1120                 return -EINVAL;
1121
1122         r = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1123         if (r < 0)
1124                 return r;
1125
1126         while (size) {
1127                 uint32_t value;
1128
1129                 r = amdgpu_get_gfx_off_status(adev, &value);
1130                 if (r) {
1131                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1132                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1133                         return r;
1134                 }
1135
1136                 r = put_user(value, (uint32_t *)buf);
1137                 if (r) {
1138                         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1139                         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1140                         return r;
1141                 }
1142
1143                 result += 4;
1144                 buf += 4;
1145                 *pos += 4;
1146                 size -= 4;
1147         }
1148
1149         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1150         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1151
1152         return result;
1153 }
1154
1155 static const struct file_operations amdgpu_debugfs_regs2_fops = {
1156         .owner = THIS_MODULE,
1157         .unlocked_ioctl = amdgpu_debugfs_regs2_ioctl,
1158         .read = amdgpu_debugfs_regs2_read,
1159         .write = amdgpu_debugfs_regs2_write,
1160         .open = amdgpu_debugfs_regs2_open,
1161         .release = amdgpu_debugfs_regs2_release,
1162         .llseek = default_llseek
1163 };
1164
1165 static const struct file_operations amdgpu_debugfs_regs_fops = {
1166         .owner = THIS_MODULE,
1167         .read = amdgpu_debugfs_regs_read,
1168         .write = amdgpu_debugfs_regs_write,
1169         .llseek = default_llseek
1170 };
1171 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1172         .owner = THIS_MODULE,
1173         .read = amdgpu_debugfs_regs_didt_read,
1174         .write = amdgpu_debugfs_regs_didt_write,
1175         .llseek = default_llseek
1176 };
1177 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1178         .owner = THIS_MODULE,
1179         .read = amdgpu_debugfs_regs_pcie_read,
1180         .write = amdgpu_debugfs_regs_pcie_write,
1181         .llseek = default_llseek
1182 };
1183 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1184         .owner = THIS_MODULE,
1185         .read = amdgpu_debugfs_regs_smc_read,
1186         .write = amdgpu_debugfs_regs_smc_write,
1187         .llseek = default_llseek
1188 };
1189
1190 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1191         .owner = THIS_MODULE,
1192         .read = amdgpu_debugfs_gca_config_read,
1193         .llseek = default_llseek
1194 };
1195
1196 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1197         .owner = THIS_MODULE,
1198         .read = amdgpu_debugfs_sensor_read,
1199         .llseek = default_llseek
1200 };
1201
1202 static const struct file_operations amdgpu_debugfs_wave_fops = {
1203         .owner = THIS_MODULE,
1204         .read = amdgpu_debugfs_wave_read,
1205         .llseek = default_llseek
1206 };
1207 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1208         .owner = THIS_MODULE,
1209         .read = amdgpu_debugfs_gpr_read,
1210         .llseek = default_llseek
1211 };
1212
1213 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1214         .owner = THIS_MODULE,
1215         .read = amdgpu_debugfs_gfxoff_read,
1216         .write = amdgpu_debugfs_gfxoff_write,
1217         .llseek = default_llseek
1218 };
1219
1220 static const struct file_operations *debugfs_regs[] = {
1221         &amdgpu_debugfs_regs_fops,
1222         &amdgpu_debugfs_regs2_fops,
1223         &amdgpu_debugfs_regs_didt_fops,
1224         &amdgpu_debugfs_regs_pcie_fops,
1225         &amdgpu_debugfs_regs_smc_fops,
1226         &amdgpu_debugfs_gca_config_fops,
1227         &amdgpu_debugfs_sensors_fops,
1228         &amdgpu_debugfs_wave_fops,
1229         &amdgpu_debugfs_gpr_fops,
1230         &amdgpu_debugfs_gfxoff_fops,
1231 };
1232
1233 static const char *debugfs_regs_names[] = {
1234         "amdgpu_regs",
1235         "amdgpu_regs2",
1236         "amdgpu_regs_didt",
1237         "amdgpu_regs_pcie",
1238         "amdgpu_regs_smc",
1239         "amdgpu_gca_config",
1240         "amdgpu_sensors",
1241         "amdgpu_wave",
1242         "amdgpu_gpr",
1243         "amdgpu_gfxoff",
1244 };
1245
1246 /**
1247  * amdgpu_debugfs_regs_init -   Initialize debugfs entries that provide
1248  *                              register access.
1249  *
1250  * @adev: The device to attach the debugfs entries to
1251  */
1252 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1253 {
1254         struct drm_minor *minor = adev_to_drm(adev)->primary;
1255         struct dentry *ent, *root = minor->debugfs_root;
1256         unsigned int i;
1257
1258         for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1259                 ent = debugfs_create_file(debugfs_regs_names[i],
1260                                           S_IFREG | S_IRUGO, root,
1261                                           adev, debugfs_regs[i]);
1262                 if (!i && !IS_ERR_OR_NULL(ent))
1263                         i_size_write(ent->d_inode, adev->rmmio_size);
1264         }
1265
1266         return 0;
1267 }
1268
1269 static int amdgpu_debugfs_test_ib_show(struct seq_file *m, void *unused)
1270 {
1271         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1272         struct drm_device *dev = adev_to_drm(adev);
1273         int r = 0, i;
1274
1275         r = pm_runtime_get_sync(dev->dev);
1276         if (r < 0) {
1277                 pm_runtime_put_autosuspend(dev->dev);
1278                 return r;
1279         }
1280
1281         /* Avoid accidently unparking the sched thread during GPU reset */
1282         r = down_write_killable(&adev->reset_sem);
1283         if (r)
1284                 return r;
1285
1286         /* hold on the scheduler */
1287         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1288                 struct amdgpu_ring *ring = adev->rings[i];
1289
1290                 if (!ring || !ring->sched.thread)
1291                         continue;
1292                 kthread_park(ring->sched.thread);
1293         }
1294
1295         seq_printf(m, "run ib test:\n");
1296         r = amdgpu_ib_ring_tests(adev);
1297         if (r)
1298                 seq_printf(m, "ib ring tests failed (%d).\n", r);
1299         else
1300                 seq_printf(m, "ib ring tests passed.\n");
1301
1302         /* go on the scheduler */
1303         for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1304                 struct amdgpu_ring *ring = adev->rings[i];
1305
1306                 if (!ring || !ring->sched.thread)
1307                         continue;
1308                 kthread_unpark(ring->sched.thread);
1309         }
1310
1311         up_write(&adev->reset_sem);
1312
1313         pm_runtime_mark_last_busy(dev->dev);
1314         pm_runtime_put_autosuspend(dev->dev);
1315
1316         return 0;
1317 }
1318
1319 static int amdgpu_debugfs_evict_vram(void *data, u64 *val)
1320 {
1321         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1322         struct drm_device *dev = adev_to_drm(adev);
1323         int r;
1324
1325         r = pm_runtime_get_sync(dev->dev);
1326         if (r < 0) {
1327                 pm_runtime_put_autosuspend(dev->dev);
1328                 return r;
1329         }
1330
1331         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
1332
1333         pm_runtime_mark_last_busy(dev->dev);
1334         pm_runtime_put_autosuspend(dev->dev);
1335
1336         return 0;
1337 }
1338
1339
1340 static int amdgpu_debugfs_evict_gtt(void *data, u64 *val)
1341 {
1342         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1343         struct drm_device *dev = adev_to_drm(adev);
1344         int r;
1345
1346         r = pm_runtime_get_sync(dev->dev);
1347         if (r < 0) {
1348                 pm_runtime_put_autosuspend(dev->dev);
1349                 return r;
1350         }
1351
1352         *val = amdgpu_ttm_evict_resources(adev, TTM_PL_TT);
1353
1354         pm_runtime_mark_last_busy(dev->dev);
1355         pm_runtime_put_autosuspend(dev->dev);
1356
1357         return 0;
1358 }
1359
1360
1361 static int amdgpu_debugfs_vm_info_show(struct seq_file *m, void *unused)
1362 {
1363         struct amdgpu_device *adev = (struct amdgpu_device *)m->private;
1364         struct drm_device *dev = adev_to_drm(adev);
1365         struct drm_file *file;
1366         int r;
1367
1368         r = mutex_lock_interruptible(&dev->filelist_mutex);
1369         if (r)
1370                 return r;
1371
1372         list_for_each_entry(file, &dev->filelist, lhead) {
1373                 struct amdgpu_fpriv *fpriv = file->driver_priv;
1374                 struct amdgpu_vm *vm = &fpriv->vm;
1375
1376                 seq_printf(m, "pid:%d\tProcess:%s ----------\n",
1377                                 vm->task_info.pid, vm->task_info.process_name);
1378                 r = amdgpu_bo_reserve(vm->root.bo, true);
1379                 if (r)
1380                         break;
1381                 amdgpu_debugfs_vm_bo_info(vm, m);
1382                 amdgpu_bo_unreserve(vm->root.bo);
1383         }
1384
1385         mutex_unlock(&dev->filelist_mutex);
1386
1387         return r;
1388 }
1389
1390 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_test_ib);
1391 DEFINE_SHOW_ATTRIBUTE(amdgpu_debugfs_vm_info);
1392 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_vram_fops, amdgpu_debugfs_evict_vram,
1393                          NULL, "%lld\n");
1394 DEFINE_DEBUGFS_ATTRIBUTE(amdgpu_evict_gtt_fops, amdgpu_debugfs_evict_gtt,
1395                          NULL, "%lld\n");
1396
1397 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1398                                           struct dma_fence **fences)
1399 {
1400         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1401         uint32_t sync_seq, last_seq;
1402
1403         last_seq = atomic_read(&ring->fence_drv.last_seq);
1404         sync_seq = ring->fence_drv.sync_seq;
1405
1406         last_seq &= drv->num_fences_mask;
1407         sync_seq &= drv->num_fences_mask;
1408
1409         do {
1410                 struct dma_fence *fence, **ptr;
1411
1412                 ++last_seq;
1413                 last_seq &= drv->num_fences_mask;
1414                 ptr = &drv->fences[last_seq];
1415
1416                 fence = rcu_dereference_protected(*ptr, 1);
1417                 RCU_INIT_POINTER(*ptr, NULL);
1418
1419                 if (!fence)
1420                         continue;
1421
1422                 fences[last_seq] = fence;
1423
1424         } while (last_seq != sync_seq);
1425 }
1426
1427 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1428                                             int length)
1429 {
1430         int i;
1431         struct dma_fence *fence;
1432
1433         for (i = 0; i < length; i++) {
1434                 fence = fences[i];
1435                 if (!fence)
1436                         continue;
1437                 dma_fence_signal(fence);
1438                 dma_fence_put(fence);
1439         }
1440 }
1441
1442 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1443 {
1444         struct drm_sched_job *s_job;
1445         struct dma_fence *fence;
1446
1447         spin_lock(&sched->job_list_lock);
1448         list_for_each_entry(s_job, &sched->pending_list, list) {
1449                 fence = sched->ops->run_job(s_job);
1450                 dma_fence_put(fence);
1451         }
1452         spin_unlock(&sched->job_list_lock);
1453 }
1454
1455 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1456 {
1457         struct amdgpu_job *job;
1458         struct drm_sched_job *s_job, *tmp;
1459         uint32_t preempt_seq;
1460         struct dma_fence *fence, **ptr;
1461         struct amdgpu_fence_driver *drv = &ring->fence_drv;
1462         struct drm_gpu_scheduler *sched = &ring->sched;
1463         bool preempted = true;
1464
1465         if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1466                 return;
1467
1468         preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1469         if (preempt_seq <= atomic_read(&drv->last_seq)) {
1470                 preempted = false;
1471                 goto no_preempt;
1472         }
1473
1474         preempt_seq &= drv->num_fences_mask;
1475         ptr = &drv->fences[preempt_seq];
1476         fence = rcu_dereference_protected(*ptr, 1);
1477
1478 no_preempt:
1479         spin_lock(&sched->job_list_lock);
1480         list_for_each_entry_safe(s_job, tmp, &sched->pending_list, list) {
1481                 if (dma_fence_is_signaled(&s_job->s_fence->finished)) {
1482                         /* remove job from ring_mirror_list */
1483                         list_del_init(&s_job->list);
1484                         sched->ops->free_job(s_job);
1485                         continue;
1486                 }
1487                 job = to_amdgpu_job(s_job);
1488                 if (preempted && (&job->hw_fence) == fence)
1489                         /* mark the job as preempted */
1490                         job->preemption_status |= AMDGPU_IB_PREEMPTED;
1491         }
1492         spin_unlock(&sched->job_list_lock);
1493 }
1494
1495 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1496 {
1497         int r, resched, length;
1498         struct amdgpu_ring *ring;
1499         struct dma_fence **fences = NULL;
1500         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1501
1502         if (val >= AMDGPU_MAX_RINGS)
1503                 return -EINVAL;
1504
1505         ring = adev->rings[val];
1506
1507         if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1508                 return -EINVAL;
1509
1510         /* the last preemption failed */
1511         if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1512                 return -EBUSY;
1513
1514         length = ring->fence_drv.num_fences_mask + 1;
1515         fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1516         if (!fences)
1517                 return -ENOMEM;
1518
1519         /* Avoid accidently unparking the sched thread during GPU reset */
1520         r = down_read_killable(&adev->reset_sem);
1521         if (r)
1522                 goto pro_end;
1523
1524         /* stop the scheduler */
1525         kthread_park(ring->sched.thread);
1526
1527         resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1528
1529         /* preempt the IB */
1530         r = amdgpu_ring_preempt_ib(ring);
1531         if (r) {
1532                 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1533                 goto failure;
1534         }
1535
1536         amdgpu_fence_process(ring);
1537
1538         if (atomic_read(&ring->fence_drv.last_seq) !=
1539             ring->fence_drv.sync_seq) {
1540                 DRM_INFO("ring %d was preempted\n", ring->idx);
1541
1542                 amdgpu_ib_preempt_mark_partial_job(ring);
1543
1544                 /* swap out the old fences */
1545                 amdgpu_ib_preempt_fences_swap(ring, fences);
1546
1547                 amdgpu_fence_driver_force_completion(ring);
1548
1549                 /* resubmit unfinished jobs */
1550                 amdgpu_ib_preempt_job_recovery(&ring->sched);
1551
1552                 /* wait for jobs finished */
1553                 amdgpu_fence_wait_empty(ring);
1554
1555                 /* signal the old fences */
1556                 amdgpu_ib_preempt_signal_fences(fences, length);
1557         }
1558
1559 failure:
1560         /* restart the scheduler */
1561         kthread_unpark(ring->sched.thread);
1562
1563         up_read(&adev->reset_sem);
1564
1565         ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1566
1567 pro_end:
1568         kfree(fences);
1569
1570         return r;
1571 }
1572
1573 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1574 {
1575         int ret = 0;
1576         uint32_t max_freq, min_freq;
1577         struct amdgpu_device *adev = (struct amdgpu_device *)data;
1578
1579         if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1580                 return -EINVAL;
1581
1582         ret = pm_runtime_get_sync(adev_to_drm(adev)->dev);
1583         if (ret < 0) {
1584                 pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1585                 return ret;
1586         }
1587
1588         if (is_support_sw_smu(adev)) {
1589                 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq);
1590                 if (ret || val > max_freq || val < min_freq)
1591                         return -EINVAL;
1592                 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val);
1593         } else {
1594                 return 0;
1595         }
1596
1597         pm_runtime_mark_last_busy(adev_to_drm(adev)->dev);
1598         pm_runtime_put_autosuspend(adev_to_drm(adev)->dev);
1599
1600         if (ret)
1601                 return -EINVAL;
1602
1603         return 0;
1604 }
1605
1606 DEFINE_DEBUGFS_ATTRIBUTE(fops_ib_preempt, NULL,
1607                         amdgpu_debugfs_ib_preempt, "%llu\n");
1608
1609 DEFINE_DEBUGFS_ATTRIBUTE(fops_sclk_set, NULL,
1610                         amdgpu_debugfs_sclk_set, "%llu\n");
1611
1612 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1613 {
1614         struct dentry *root = adev_to_drm(adev)->primary->debugfs_root;
1615         struct dentry *ent;
1616         int r, i;
1617
1618         if (!debugfs_initialized())
1619                 return 0;
1620
1621         ent = debugfs_create_file("amdgpu_preempt_ib", 0600, root, adev,
1622                                   &fops_ib_preempt);
1623         if (IS_ERR(ent)) {
1624                 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1625                 return PTR_ERR(ent);
1626         }
1627
1628         ent = debugfs_create_file("amdgpu_force_sclk", 0200, root, adev,
1629                                   &fops_sclk_set);
1630         if (IS_ERR(ent)) {
1631                 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1632                 return PTR_ERR(ent);
1633         }
1634
1635         /* Register debugfs entries for amdgpu_ttm */
1636         amdgpu_ttm_debugfs_init(adev);
1637         amdgpu_debugfs_pm_init(adev);
1638         amdgpu_debugfs_sa_init(adev);
1639         amdgpu_debugfs_fence_init(adev);
1640         amdgpu_debugfs_gem_init(adev);
1641
1642         r = amdgpu_debugfs_regs_init(adev);
1643         if (r)
1644                 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1645
1646         amdgpu_debugfs_firmware_init(adev);
1647
1648 #if defined(CONFIG_DRM_AMD_DC)
1649         if (amdgpu_device_has_dc_support(adev))
1650                 dtn_debugfs_init(adev);
1651 #endif
1652
1653         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1654                 struct amdgpu_ring *ring = adev->rings[i];
1655
1656                 if (!ring)
1657                         continue;
1658
1659                 amdgpu_debugfs_ring_init(adev, ring);
1660         }
1661
1662         amdgpu_ras_debugfs_create_all(adev);
1663         amdgpu_rap_debugfs_init(adev);
1664         amdgpu_securedisplay_debugfs_init(adev);
1665         amdgpu_fw_attestation_debugfs_init(adev);
1666
1667         debugfs_create_file("amdgpu_evict_vram", 0444, root, adev,
1668                             &amdgpu_evict_vram_fops);
1669         debugfs_create_file("amdgpu_evict_gtt", 0444, root, adev,
1670                             &amdgpu_evict_gtt_fops);
1671         debugfs_create_file("amdgpu_test_ib", 0444, root, adev,
1672                             &amdgpu_debugfs_test_ib_fops);
1673         debugfs_create_file("amdgpu_vm_info", 0444, root, adev,
1674                             &amdgpu_debugfs_vm_info_fops);
1675
1676         adev->debugfs_vbios_blob.data = adev->bios;
1677         adev->debugfs_vbios_blob.size = adev->bios_size;
1678         debugfs_create_blob("amdgpu_vbios", 0444, root,
1679                             &adev->debugfs_vbios_blob);
1680
1681         adev->debugfs_discovery_blob.data = adev->mman.discovery_bin;
1682         adev->debugfs_discovery_blob.size = adev->mman.discovery_tmr_size;
1683         debugfs_create_blob("amdgpu_discovery", 0444, root,
1684                             &adev->debugfs_discovery_blob);
1685
1686         return 0;
1687 }
1688
1689 #else
1690 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1691 {
1692         return 0;
1693 }
1694 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1695 {
1696         return 0;
1697 }
1698 #endif