2 * Copyright 2008 Advanced Micro Devices, Inc.
3 * Copyright 2008 Red Hat Inc.
4 * Copyright 2009 Jerome Glisse.
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the "Software"),
8 * to deal in the Software without restriction, including without limitation
9 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
10 * and/or sell copies of the Software, and to permit persons to whom the
11 * Software is furnished to do so, subject to the following conditions:
13 * The above copyright notice and this permission notice shall be included in
14 * all copies or substantial portions of the Software.
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
20 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22 * OTHER DEALINGS IN THE SOFTWARE.
26 #include <linux/kthread.h>
27 #include <linux/pci.h>
28 #include <linux/uaccess.h>
29 #include <linux/pm_runtime.h>
30 #include <linux/poll.h>
31 #include <drm/drm_debugfs.h>
34 #include "amdgpu_pm.h"
35 #include "amdgpu_dm_debugfs.h"
36 #include "amdgpu_ras.h"
39 * amdgpu_debugfs_add_files - Add simple debugfs entries
41 * @adev: Device to attach debugfs entries to
42 * @files: Array of function callbacks that respond to reads
43 * @nfiles: Number of callbacks to register
46 int amdgpu_debugfs_add_files(struct amdgpu_device *adev,
47 const struct drm_info_list *files,
52 for (i = 0; i < adev->debugfs_count; i++) {
53 if (adev->debugfs[i].files == files) {
54 /* Already registered */
59 i = adev->debugfs_count + 1;
60 if (i > AMDGPU_DEBUGFS_MAX_COMPONENTS) {
61 DRM_ERROR("Reached maximum number of debugfs components.\n");
62 DRM_ERROR("Report so we increase "
63 "AMDGPU_DEBUGFS_MAX_COMPONENTS.\n");
66 adev->debugfs[adev->debugfs_count].files = files;
67 adev->debugfs[adev->debugfs_count].num_files = nfiles;
68 adev->debugfs_count = i;
69 #if defined(CONFIG_DEBUG_FS)
70 drm_debugfs_create_files(files, nfiles,
71 adev->ddev->primary->debugfs_root,
77 int amdgpu_debugfs_wait_dump(struct amdgpu_device *adev)
79 #if defined(CONFIG_DEBUG_FS)
80 unsigned long timeout = 600 * HZ;
83 wake_up_interruptible(&adev->autodump.gpu_hang);
85 ret = wait_for_completion_interruptible_timeout(&adev->autodump.dumping, timeout);
87 pr_err("autodump: timeout, move on to gpu recovery\n");
94 #if defined(CONFIG_DEBUG_FS)
96 static int amdgpu_debugfs_autodump_open(struct inode *inode, struct file *file)
98 struct amdgpu_device *adev = inode->i_private;
101 file->private_data = adev;
103 mutex_lock(&adev->lock_reset);
104 if (adev->autodump.dumping.done) {
105 reinit_completion(&adev->autodump.dumping);
110 mutex_unlock(&adev->lock_reset);
115 static int amdgpu_debugfs_autodump_release(struct inode *inode, struct file *file)
117 struct amdgpu_device *adev = file->private_data;
119 complete_all(&adev->autodump.dumping);
123 static unsigned int amdgpu_debugfs_autodump_poll(struct file *file, struct poll_table_struct *poll_table)
125 struct amdgpu_device *adev = file->private_data;
127 poll_wait(file, &adev->autodump.gpu_hang, poll_table);
129 if (adev->in_gpu_reset)
130 return POLLIN | POLLRDNORM | POLLWRNORM;
135 static const struct file_operations autodump_debug_fops = {
136 .owner = THIS_MODULE,
137 .open = amdgpu_debugfs_autodump_open,
138 .poll = amdgpu_debugfs_autodump_poll,
139 .release = amdgpu_debugfs_autodump_release,
142 static void amdgpu_debugfs_autodump_init(struct amdgpu_device *adev)
144 init_completion(&adev->autodump.dumping);
145 complete_all(&adev->autodump.dumping);
146 init_waitqueue_head(&adev->autodump.gpu_hang);
148 debugfs_create_file("amdgpu_autodump", 0600,
149 adev->ddev->primary->debugfs_root,
150 adev, &autodump_debug_fops);
154 * amdgpu_debugfs_process_reg_op - Handle MMIO register reads/writes
156 * @read: True if reading
157 * @f: open file handle
158 * @buf: User buffer to write/read to
159 * @size: Number of bytes to write/read
160 * @pos: Offset to seek to
162 * This debugfs entry has special meaning on the offset being sought.
163 * Various bits have different meanings:
165 * Bit 62: Indicates a GRBM bank switch is needed
166 * Bit 61: Indicates a SRBM bank switch is needed (implies bit 62 is
168 * Bits 24..33: The SE or ME selector if needed
169 * Bits 34..43: The SH (or SA) or PIPE selector if needed
170 * Bits 44..53: The INSTANCE (or CU/WGP) or QUEUE selector if needed
172 * Bit 23: Indicates that the PM power gating lock should be held
173 * This is necessary to read registers that might be
174 * unreliable during a power gating transistion.
176 * The lower bits are the BYTE offset of the register to read. This
177 * allows reading multiple registers in a single call and having
178 * the returned size reflect that.
180 static int amdgpu_debugfs_process_reg_op(bool read, struct file *f,
181 char __user *buf, size_t size, loff_t *pos)
183 struct amdgpu_device *adev = file_inode(f)->i_private;
186 bool pm_pg_lock, use_bank, use_ring;
187 unsigned instance_bank, sh_bank, se_bank, me, pipe, queue, vmid;
189 pm_pg_lock = use_bank = use_ring = false;
190 instance_bank = sh_bank = se_bank = me = pipe = queue = vmid = 0;
192 if (size & 0x3 || *pos & 0x3 ||
193 ((*pos & (1ULL << 62)) && (*pos & (1ULL << 61))))
196 /* are we reading registers for which a PG lock is necessary? */
197 pm_pg_lock = (*pos >> 23) & 1;
199 if (*pos & (1ULL << 62)) {
200 se_bank = (*pos & GENMASK_ULL(33, 24)) >> 24;
201 sh_bank = (*pos & GENMASK_ULL(43, 34)) >> 34;
202 instance_bank = (*pos & GENMASK_ULL(53, 44)) >> 44;
204 if (se_bank == 0x3FF)
205 se_bank = 0xFFFFFFFF;
206 if (sh_bank == 0x3FF)
207 sh_bank = 0xFFFFFFFF;
208 if (instance_bank == 0x3FF)
209 instance_bank = 0xFFFFFFFF;
211 } else if (*pos & (1ULL << 61)) {
213 me = (*pos & GENMASK_ULL(33, 24)) >> 24;
214 pipe = (*pos & GENMASK_ULL(43, 34)) >> 34;
215 queue = (*pos & GENMASK_ULL(53, 44)) >> 44;
216 vmid = (*pos & GENMASK_ULL(58, 54)) >> 54;
220 use_bank = use_ring = false;
223 *pos &= (1UL << 22) - 1;
225 r = pm_runtime_get_sync(adev->ddev->dev);
229 r = amdgpu_virt_enable_access_debugfs(adev);
234 if ((sh_bank != 0xFFFFFFFF && sh_bank >= adev->gfx.config.max_sh_per_se) ||
235 (se_bank != 0xFFFFFFFF && se_bank >= adev->gfx.config.max_shader_engines)) {
236 pm_runtime_mark_last_busy(adev->ddev->dev);
237 pm_runtime_put_autosuspend(adev->ddev->dev);
238 amdgpu_virt_disable_access_debugfs(adev);
241 mutex_lock(&adev->grbm_idx_mutex);
242 amdgpu_gfx_select_se_sh(adev, se_bank,
243 sh_bank, instance_bank);
244 } else if (use_ring) {
245 mutex_lock(&adev->srbm_mutex);
246 amdgpu_gfx_select_me_pipe_q(adev, me, pipe, queue, vmid);
250 mutex_lock(&adev->pm.mutex);
256 value = RREG32(*pos >> 2);
257 r = put_user(value, (uint32_t *)buf);
259 r = get_user(value, (uint32_t *)buf);
261 amdgpu_mm_wreg_mmio_rlc(adev, *pos >> 2, value, 0);
276 amdgpu_gfx_select_se_sh(adev, 0xffffffff, 0xffffffff, 0xffffffff);
277 mutex_unlock(&adev->grbm_idx_mutex);
278 } else if (use_ring) {
279 amdgpu_gfx_select_me_pipe_q(adev, 0, 0, 0, 0);
280 mutex_unlock(&adev->srbm_mutex);
284 mutex_unlock(&adev->pm.mutex);
286 pm_runtime_mark_last_busy(adev->ddev->dev);
287 pm_runtime_put_autosuspend(adev->ddev->dev);
289 amdgpu_virt_disable_access_debugfs(adev);
294 * amdgpu_debugfs_regs_read - Callback for reading MMIO registers
296 static ssize_t amdgpu_debugfs_regs_read(struct file *f, char __user *buf,
297 size_t size, loff_t *pos)
299 return amdgpu_debugfs_process_reg_op(true, f, buf, size, pos);
303 * amdgpu_debugfs_regs_write - Callback for writing MMIO registers
305 static ssize_t amdgpu_debugfs_regs_write(struct file *f, const char __user *buf,
306 size_t size, loff_t *pos)
308 return amdgpu_debugfs_process_reg_op(false, f, (char __user *)buf, size, pos);
313 * amdgpu_debugfs_regs_pcie_read - Read from a PCIE register
315 * @f: open file handle
316 * @buf: User buffer to store read data in
317 * @size: Number of bytes to read
318 * @pos: Offset to seek to
320 * The lower bits are the BYTE offset of the register to read. This
321 * allows reading multiple registers in a single call and having
322 * the returned size reflect that.
324 static ssize_t amdgpu_debugfs_regs_pcie_read(struct file *f, char __user *buf,
325 size_t size, loff_t *pos)
327 struct amdgpu_device *adev = file_inode(f)->i_private;
331 if (size & 0x3 || *pos & 0x3)
334 r = pm_runtime_get_sync(adev->ddev->dev);
338 r = amdgpu_virt_enable_access_debugfs(adev);
345 value = RREG32_PCIE(*pos >> 2);
346 r = put_user(value, (uint32_t *)buf);
348 pm_runtime_mark_last_busy(adev->ddev->dev);
349 pm_runtime_put_autosuspend(adev->ddev->dev);
350 amdgpu_virt_disable_access_debugfs(adev);
360 pm_runtime_mark_last_busy(adev->ddev->dev);
361 pm_runtime_put_autosuspend(adev->ddev->dev);
363 amdgpu_virt_disable_access_debugfs(adev);
368 * amdgpu_debugfs_regs_pcie_write - Write to a PCIE register
370 * @f: open file handle
371 * @buf: User buffer to write data from
372 * @size: Number of bytes to write
373 * @pos: Offset to seek to
375 * The lower bits are the BYTE offset of the register to write. This
376 * allows writing multiple registers in a single call and having
377 * the returned size reflect that.
379 static ssize_t amdgpu_debugfs_regs_pcie_write(struct file *f, const char __user *buf,
380 size_t size, loff_t *pos)
382 struct amdgpu_device *adev = file_inode(f)->i_private;
386 if (size & 0x3 || *pos & 0x3)
389 r = pm_runtime_get_sync(adev->ddev->dev);
393 r = amdgpu_virt_enable_access_debugfs(adev);
400 r = get_user(value, (uint32_t *)buf);
402 pm_runtime_mark_last_busy(adev->ddev->dev);
403 pm_runtime_put_autosuspend(adev->ddev->dev);
404 amdgpu_virt_disable_access_debugfs(adev);
408 WREG32_PCIE(*pos >> 2, value);
416 pm_runtime_mark_last_busy(adev->ddev->dev);
417 pm_runtime_put_autosuspend(adev->ddev->dev);
419 amdgpu_virt_disable_access_debugfs(adev);
424 * amdgpu_debugfs_regs_didt_read - Read from a DIDT register
426 * @f: open file handle
427 * @buf: User buffer to store read data in
428 * @size: Number of bytes to read
429 * @pos: Offset to seek to
431 * The lower bits are the BYTE offset of the register to read. This
432 * allows reading multiple registers in a single call and having
433 * the returned size reflect that.
435 static ssize_t amdgpu_debugfs_regs_didt_read(struct file *f, char __user *buf,
436 size_t size, loff_t *pos)
438 struct amdgpu_device *adev = file_inode(f)->i_private;
442 if (size & 0x3 || *pos & 0x3)
445 r = pm_runtime_get_sync(adev->ddev->dev);
449 r = amdgpu_virt_enable_access_debugfs(adev);
456 value = RREG32_DIDT(*pos >> 2);
457 r = put_user(value, (uint32_t *)buf);
459 pm_runtime_mark_last_busy(adev->ddev->dev);
460 pm_runtime_put_autosuspend(adev->ddev->dev);
461 amdgpu_virt_disable_access_debugfs(adev);
471 pm_runtime_mark_last_busy(adev->ddev->dev);
472 pm_runtime_put_autosuspend(adev->ddev->dev);
474 amdgpu_virt_disable_access_debugfs(adev);
479 * amdgpu_debugfs_regs_didt_write - Write to a DIDT register
481 * @f: open file handle
482 * @buf: User buffer to write data from
483 * @size: Number of bytes to write
484 * @pos: Offset to seek to
486 * The lower bits are the BYTE offset of the register to write. This
487 * allows writing multiple registers in a single call and having
488 * the returned size reflect that.
490 static ssize_t amdgpu_debugfs_regs_didt_write(struct file *f, const char __user *buf,
491 size_t size, loff_t *pos)
493 struct amdgpu_device *adev = file_inode(f)->i_private;
497 if (size & 0x3 || *pos & 0x3)
500 r = pm_runtime_get_sync(adev->ddev->dev);
504 r = amdgpu_virt_enable_access_debugfs(adev);
511 r = get_user(value, (uint32_t *)buf);
513 pm_runtime_mark_last_busy(adev->ddev->dev);
514 pm_runtime_put_autosuspend(adev->ddev->dev);
515 amdgpu_virt_disable_access_debugfs(adev);
519 WREG32_DIDT(*pos >> 2, value);
527 pm_runtime_mark_last_busy(adev->ddev->dev);
528 pm_runtime_put_autosuspend(adev->ddev->dev);
530 amdgpu_virt_disable_access_debugfs(adev);
535 * amdgpu_debugfs_regs_smc_read - Read from a SMC register
537 * @f: open file handle
538 * @buf: User buffer to store read data in
539 * @size: Number of bytes to read
540 * @pos: Offset to seek to
542 * The lower bits are the BYTE offset of the register to read. This
543 * allows reading multiple registers in a single call and having
544 * the returned size reflect that.
546 static ssize_t amdgpu_debugfs_regs_smc_read(struct file *f, char __user *buf,
547 size_t size, loff_t *pos)
549 struct amdgpu_device *adev = file_inode(f)->i_private;
553 if (size & 0x3 || *pos & 0x3)
556 r = pm_runtime_get_sync(adev->ddev->dev);
560 r = amdgpu_virt_enable_access_debugfs(adev);
567 value = RREG32_SMC(*pos);
568 r = put_user(value, (uint32_t *)buf);
570 pm_runtime_mark_last_busy(adev->ddev->dev);
571 pm_runtime_put_autosuspend(adev->ddev->dev);
572 amdgpu_virt_disable_access_debugfs(adev);
582 pm_runtime_mark_last_busy(adev->ddev->dev);
583 pm_runtime_put_autosuspend(adev->ddev->dev);
585 amdgpu_virt_disable_access_debugfs(adev);
590 * amdgpu_debugfs_regs_smc_write - Write to a SMC register
592 * @f: open file handle
593 * @buf: User buffer to write data from
594 * @size: Number of bytes to write
595 * @pos: Offset to seek to
597 * The lower bits are the BYTE offset of the register to write. This
598 * allows writing multiple registers in a single call and having
599 * the returned size reflect that.
601 static ssize_t amdgpu_debugfs_regs_smc_write(struct file *f, const char __user *buf,
602 size_t size, loff_t *pos)
604 struct amdgpu_device *adev = file_inode(f)->i_private;
608 if (size & 0x3 || *pos & 0x3)
611 r = pm_runtime_get_sync(adev->ddev->dev);
615 r = amdgpu_virt_enable_access_debugfs(adev);
622 r = get_user(value, (uint32_t *)buf);
624 pm_runtime_mark_last_busy(adev->ddev->dev);
625 pm_runtime_put_autosuspend(adev->ddev->dev);
626 amdgpu_virt_disable_access_debugfs(adev);
630 WREG32_SMC(*pos, value);
638 pm_runtime_mark_last_busy(adev->ddev->dev);
639 pm_runtime_put_autosuspend(adev->ddev->dev);
641 amdgpu_virt_disable_access_debugfs(adev);
646 * amdgpu_debugfs_gca_config_read - Read from gfx config data
648 * @f: open file handle
649 * @buf: User buffer to store read data in
650 * @size: Number of bytes to read
651 * @pos: Offset to seek to
653 * This file is used to access configuration data in a somewhat
654 * stable fashion. The format is a series of DWORDs with the first
655 * indicating which revision it is. New content is appended to the
656 * end so that older software can still read the data.
659 static ssize_t amdgpu_debugfs_gca_config_read(struct file *f, char __user *buf,
660 size_t size, loff_t *pos)
662 struct amdgpu_device *adev = file_inode(f)->i_private;
665 uint32_t *config, no_regs = 0;
667 if (size & 0x3 || *pos & 0x3)
670 config = kmalloc_array(256, sizeof(*config), GFP_KERNEL);
674 /* version, increment each time something is added */
675 config[no_regs++] = 3;
676 config[no_regs++] = adev->gfx.config.max_shader_engines;
677 config[no_regs++] = adev->gfx.config.max_tile_pipes;
678 config[no_regs++] = adev->gfx.config.max_cu_per_sh;
679 config[no_regs++] = adev->gfx.config.max_sh_per_se;
680 config[no_regs++] = adev->gfx.config.max_backends_per_se;
681 config[no_regs++] = adev->gfx.config.max_texture_channel_caches;
682 config[no_regs++] = adev->gfx.config.max_gprs;
683 config[no_regs++] = adev->gfx.config.max_gs_threads;
684 config[no_regs++] = adev->gfx.config.max_hw_contexts;
685 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_frontend;
686 config[no_regs++] = adev->gfx.config.sc_prim_fifo_size_backend;
687 config[no_regs++] = adev->gfx.config.sc_hiz_tile_fifo_size;
688 config[no_regs++] = adev->gfx.config.sc_earlyz_tile_fifo_size;
689 config[no_regs++] = adev->gfx.config.num_tile_pipes;
690 config[no_regs++] = adev->gfx.config.backend_enable_mask;
691 config[no_regs++] = adev->gfx.config.mem_max_burst_length_bytes;
692 config[no_regs++] = adev->gfx.config.mem_row_size_in_kb;
693 config[no_regs++] = adev->gfx.config.shader_engine_tile_size;
694 config[no_regs++] = adev->gfx.config.num_gpus;
695 config[no_regs++] = adev->gfx.config.multi_gpu_tile_size;
696 config[no_regs++] = adev->gfx.config.mc_arb_ramcfg;
697 config[no_regs++] = adev->gfx.config.gb_addr_config;
698 config[no_regs++] = adev->gfx.config.num_rbs;
701 config[no_regs++] = adev->rev_id;
702 config[no_regs++] = adev->pg_flags;
703 config[no_regs++] = adev->cg_flags;
706 config[no_regs++] = adev->family;
707 config[no_regs++] = adev->external_rev_id;
710 config[no_regs++] = adev->pdev->device;
711 config[no_regs++] = adev->pdev->revision;
712 config[no_regs++] = adev->pdev->subsystem_device;
713 config[no_regs++] = adev->pdev->subsystem_vendor;
715 while (size && (*pos < no_regs * 4)) {
718 value = config[*pos >> 2];
719 r = put_user(value, (uint32_t *)buf);
736 * amdgpu_debugfs_sensor_read - Read from the powerplay sensors
738 * @f: open file handle
739 * @buf: User buffer to store read data in
740 * @size: Number of bytes to read
741 * @pos: Offset to seek to
743 * The offset is treated as the BYTE address of one of the sensors
744 * enumerated in amd/include/kgd_pp_interface.h under the
745 * 'amd_pp_sensors' enumeration. For instance to read the UVD VCLK
746 * you would use the offset 3 * 4 = 12.
748 static ssize_t amdgpu_debugfs_sensor_read(struct file *f, char __user *buf,
749 size_t size, loff_t *pos)
751 struct amdgpu_device *adev = file_inode(f)->i_private;
752 int idx, x, outsize, r, valuesize;
755 if (size & 3 || *pos & 0x3)
758 if (!adev->pm.dpm_enabled)
761 /* convert offset to sensor number */
764 valuesize = sizeof(values);
766 r = pm_runtime_get_sync(adev->ddev->dev);
770 r = amdgpu_virt_enable_access_debugfs(adev);
774 r = amdgpu_dpm_read_sensor(adev, idx, &values[0], &valuesize);
776 pm_runtime_mark_last_busy(adev->ddev->dev);
777 pm_runtime_put_autosuspend(adev->ddev->dev);
780 amdgpu_virt_disable_access_debugfs(adev);
784 if (size > valuesize) {
785 amdgpu_virt_disable_access_debugfs(adev);
793 r = put_user(values[x++], (int32_t *)buf);
800 amdgpu_virt_disable_access_debugfs(adev);
801 return !r ? outsize : r;
804 /** amdgpu_debugfs_wave_read - Read WAVE STATUS data
806 * @f: open file handle
807 * @buf: User buffer to store read data in
808 * @size: Number of bytes to read
809 * @pos: Offset to seek to
811 * The offset being sought changes which wave that the status data
812 * will be returned for. The bits are used as follows:
814 * Bits 0..6: Byte offset into data
815 * Bits 7..14: SE selector
816 * Bits 15..22: SH/SA selector
817 * Bits 23..30: CU/{WGP+SIMD} selector
818 * Bits 31..36: WAVE ID selector
819 * Bits 37..44: SIMD ID selector
821 * The returned data begins with one DWORD of version information
822 * Followed by WAVE STATUS registers relevant to the GFX IP version
823 * being used. See gfx_v8_0_read_wave_data() for an example output.
825 static ssize_t amdgpu_debugfs_wave_read(struct file *f, char __user *buf,
826 size_t size, loff_t *pos)
828 struct amdgpu_device *adev = f->f_inode->i_private;
831 uint32_t offset, se, sh, cu, wave, simd, data[32];
833 if (size & 3 || *pos & 3)
837 offset = (*pos & GENMASK_ULL(6, 0));
838 se = (*pos & GENMASK_ULL(14, 7)) >> 7;
839 sh = (*pos & GENMASK_ULL(22, 15)) >> 15;
840 cu = (*pos & GENMASK_ULL(30, 23)) >> 23;
841 wave = (*pos & GENMASK_ULL(36, 31)) >> 31;
842 simd = (*pos & GENMASK_ULL(44, 37)) >> 37;
844 r = pm_runtime_get_sync(adev->ddev->dev);
848 r = amdgpu_virt_enable_access_debugfs(adev);
852 /* switch to the specific se/sh/cu */
853 mutex_lock(&adev->grbm_idx_mutex);
854 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
857 if (adev->gfx.funcs->read_wave_data)
858 adev->gfx.funcs->read_wave_data(adev, simd, wave, data, &x);
860 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
861 mutex_unlock(&adev->grbm_idx_mutex);
863 pm_runtime_mark_last_busy(adev->ddev->dev);
864 pm_runtime_put_autosuspend(adev->ddev->dev);
867 amdgpu_virt_disable_access_debugfs(adev);
871 while (size && (offset < x * 4)) {
874 value = data[offset >> 2];
875 r = put_user(value, (uint32_t *)buf);
877 amdgpu_virt_disable_access_debugfs(adev);
887 amdgpu_virt_disable_access_debugfs(adev);
891 /** amdgpu_debugfs_gpr_read - Read wave gprs
893 * @f: open file handle
894 * @buf: User buffer to store read data in
895 * @size: Number of bytes to read
896 * @pos: Offset to seek to
898 * The offset being sought changes which wave that the status data
899 * will be returned for. The bits are used as follows:
901 * Bits 0..11: Byte offset into data
902 * Bits 12..19: SE selector
903 * Bits 20..27: SH/SA selector
904 * Bits 28..35: CU/{WGP+SIMD} selector
905 * Bits 36..43: WAVE ID selector
906 * Bits 37..44: SIMD ID selector
907 * Bits 52..59: Thread selector
908 * Bits 60..61: Bank selector (VGPR=0,SGPR=1)
910 * The return data comes from the SGPR or VGPR register bank for
911 * the selected operational unit.
913 static ssize_t amdgpu_debugfs_gpr_read(struct file *f, char __user *buf,
914 size_t size, loff_t *pos)
916 struct amdgpu_device *adev = f->f_inode->i_private;
919 uint32_t offset, se, sh, cu, wave, simd, thread, bank, *data;
921 if (size > 4096 || size & 3 || *pos & 3)
925 offset = (*pos & GENMASK_ULL(11, 0)) >> 2;
926 se = (*pos & GENMASK_ULL(19, 12)) >> 12;
927 sh = (*pos & GENMASK_ULL(27, 20)) >> 20;
928 cu = (*pos & GENMASK_ULL(35, 28)) >> 28;
929 wave = (*pos & GENMASK_ULL(43, 36)) >> 36;
930 simd = (*pos & GENMASK_ULL(51, 44)) >> 44;
931 thread = (*pos & GENMASK_ULL(59, 52)) >> 52;
932 bank = (*pos & GENMASK_ULL(61, 60)) >> 60;
934 data = kcalloc(1024, sizeof(*data), GFP_KERNEL);
938 r = pm_runtime_get_sync(adev->ddev->dev);
942 r = amdgpu_virt_enable_access_debugfs(adev);
946 /* switch to the specific se/sh/cu */
947 mutex_lock(&adev->grbm_idx_mutex);
948 amdgpu_gfx_select_se_sh(adev, se, sh, cu);
951 if (adev->gfx.funcs->read_wave_vgprs)
952 adev->gfx.funcs->read_wave_vgprs(adev, simd, wave, thread, offset, size>>2, data);
954 if (adev->gfx.funcs->read_wave_sgprs)
955 adev->gfx.funcs->read_wave_sgprs(adev, simd, wave, offset, size>>2, data);
958 amdgpu_gfx_select_se_sh(adev, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF);
959 mutex_unlock(&adev->grbm_idx_mutex);
961 pm_runtime_mark_last_busy(adev->ddev->dev);
962 pm_runtime_put_autosuspend(adev->ddev->dev);
967 value = data[result >> 2];
968 r = put_user(value, (uint32_t *)buf);
981 amdgpu_virt_disable_access_debugfs(adev);
986 * amdgpu_debugfs_regs_gfxoff_write - Enable/disable GFXOFF
988 * @f: open file handle
989 * @buf: User buffer to write data from
990 * @size: Number of bytes to write
991 * @pos: Offset to seek to
993 * Write a 32-bit zero to disable or a 32-bit non-zero to enable
995 static ssize_t amdgpu_debugfs_gfxoff_write(struct file *f, const char __user *buf,
996 size_t size, loff_t *pos)
998 struct amdgpu_device *adev = file_inode(f)->i_private;
1002 if (size & 0x3 || *pos & 0x3)
1005 r = pm_runtime_get_sync(adev->ddev->dev);
1012 r = get_user(value, (uint32_t *)buf);
1014 pm_runtime_mark_last_busy(adev->ddev->dev);
1015 pm_runtime_put_autosuspend(adev->ddev->dev);
1019 amdgpu_gfx_off_ctrl(adev, value ? true : false);
1027 pm_runtime_mark_last_busy(adev->ddev->dev);
1028 pm_runtime_put_autosuspend(adev->ddev->dev);
1034 static const struct file_operations amdgpu_debugfs_regs_fops = {
1035 .owner = THIS_MODULE,
1036 .read = amdgpu_debugfs_regs_read,
1037 .write = amdgpu_debugfs_regs_write,
1038 .llseek = default_llseek
1040 static const struct file_operations amdgpu_debugfs_regs_didt_fops = {
1041 .owner = THIS_MODULE,
1042 .read = amdgpu_debugfs_regs_didt_read,
1043 .write = amdgpu_debugfs_regs_didt_write,
1044 .llseek = default_llseek
1046 static const struct file_operations amdgpu_debugfs_regs_pcie_fops = {
1047 .owner = THIS_MODULE,
1048 .read = amdgpu_debugfs_regs_pcie_read,
1049 .write = amdgpu_debugfs_regs_pcie_write,
1050 .llseek = default_llseek
1052 static const struct file_operations amdgpu_debugfs_regs_smc_fops = {
1053 .owner = THIS_MODULE,
1054 .read = amdgpu_debugfs_regs_smc_read,
1055 .write = amdgpu_debugfs_regs_smc_write,
1056 .llseek = default_llseek
1059 static const struct file_operations amdgpu_debugfs_gca_config_fops = {
1060 .owner = THIS_MODULE,
1061 .read = amdgpu_debugfs_gca_config_read,
1062 .llseek = default_llseek
1065 static const struct file_operations amdgpu_debugfs_sensors_fops = {
1066 .owner = THIS_MODULE,
1067 .read = amdgpu_debugfs_sensor_read,
1068 .llseek = default_llseek
1071 static const struct file_operations amdgpu_debugfs_wave_fops = {
1072 .owner = THIS_MODULE,
1073 .read = amdgpu_debugfs_wave_read,
1074 .llseek = default_llseek
1076 static const struct file_operations amdgpu_debugfs_gpr_fops = {
1077 .owner = THIS_MODULE,
1078 .read = amdgpu_debugfs_gpr_read,
1079 .llseek = default_llseek
1082 static const struct file_operations amdgpu_debugfs_gfxoff_fops = {
1083 .owner = THIS_MODULE,
1084 .write = amdgpu_debugfs_gfxoff_write,
1087 static const struct file_operations *debugfs_regs[] = {
1088 &amdgpu_debugfs_regs_fops,
1089 &amdgpu_debugfs_regs_didt_fops,
1090 &amdgpu_debugfs_regs_pcie_fops,
1091 &amdgpu_debugfs_regs_smc_fops,
1092 &amdgpu_debugfs_gca_config_fops,
1093 &amdgpu_debugfs_sensors_fops,
1094 &amdgpu_debugfs_wave_fops,
1095 &amdgpu_debugfs_gpr_fops,
1096 &amdgpu_debugfs_gfxoff_fops,
1099 static const char *debugfs_regs_names[] = {
1104 "amdgpu_gca_config",
1112 * amdgpu_debugfs_regs_init - Initialize debugfs entries that provide
1115 * @adev: The device to attach the debugfs entries to
1117 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)
1119 struct drm_minor *minor = adev->ddev->primary;
1120 struct dentry *ent, *root = minor->debugfs_root;
1123 for (i = 0; i < ARRAY_SIZE(debugfs_regs); i++) {
1124 ent = debugfs_create_file(debugfs_regs_names[i],
1125 S_IFREG | S_IRUGO, root,
1126 adev, debugfs_regs[i]);
1127 if (!i && !IS_ERR_OR_NULL(ent))
1128 i_size_write(ent->d_inode, adev->rmmio_size);
1129 adev->debugfs_regs[i] = ent;
1135 static int amdgpu_debugfs_test_ib(struct seq_file *m, void *data)
1137 struct drm_info_node *node = (struct drm_info_node *) m->private;
1138 struct drm_device *dev = node->minor->dev;
1139 struct amdgpu_device *adev = dev->dev_private;
1142 r = pm_runtime_get_sync(dev->dev);
1146 /* Avoid accidently unparking the sched thread during GPU reset */
1147 mutex_lock(&adev->lock_reset);
1149 /* hold on the scheduler */
1150 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1151 struct amdgpu_ring *ring = adev->rings[i];
1153 if (!ring || !ring->sched.thread)
1155 kthread_park(ring->sched.thread);
1158 seq_printf(m, "run ib test:\n");
1159 r = amdgpu_ib_ring_tests(adev);
1161 seq_printf(m, "ib ring tests failed (%d).\n", r);
1163 seq_printf(m, "ib ring tests passed.\n");
1165 /* go on the scheduler */
1166 for (i = 0; i < AMDGPU_MAX_RINGS; i++) {
1167 struct amdgpu_ring *ring = adev->rings[i];
1169 if (!ring || !ring->sched.thread)
1171 kthread_unpark(ring->sched.thread);
1174 mutex_unlock(&adev->lock_reset);
1176 pm_runtime_mark_last_busy(dev->dev);
1177 pm_runtime_put_autosuspend(dev->dev);
1182 static int amdgpu_debugfs_get_vbios_dump(struct seq_file *m, void *data)
1184 struct drm_info_node *node = (struct drm_info_node *) m->private;
1185 struct drm_device *dev = node->minor->dev;
1186 struct amdgpu_device *adev = dev->dev_private;
1188 seq_write(m, adev->bios, adev->bios_size);
1192 static int amdgpu_debugfs_evict_vram(struct seq_file *m, void *data)
1194 struct drm_info_node *node = (struct drm_info_node *)m->private;
1195 struct drm_device *dev = node->minor->dev;
1196 struct amdgpu_device *adev = dev->dev_private;
1199 r = pm_runtime_get_sync(dev->dev);
1203 seq_printf(m, "(%d)\n", amdgpu_bo_evict_vram(adev));
1205 pm_runtime_mark_last_busy(dev->dev);
1206 pm_runtime_put_autosuspend(dev->dev);
1211 static int amdgpu_debugfs_evict_gtt(struct seq_file *m, void *data)
1213 struct drm_info_node *node = (struct drm_info_node *)m->private;
1214 struct drm_device *dev = node->minor->dev;
1215 struct amdgpu_device *adev = dev->dev_private;
1218 r = pm_runtime_get_sync(dev->dev);
1222 seq_printf(m, "(%d)\n", ttm_bo_evict_mm(&adev->mman.bdev, TTM_PL_TT));
1224 pm_runtime_mark_last_busy(dev->dev);
1225 pm_runtime_put_autosuspend(dev->dev);
1230 static const struct drm_info_list amdgpu_debugfs_list[] = {
1231 {"amdgpu_vbios", amdgpu_debugfs_get_vbios_dump},
1232 {"amdgpu_test_ib", &amdgpu_debugfs_test_ib},
1233 {"amdgpu_evict_vram", &amdgpu_debugfs_evict_vram},
1234 {"amdgpu_evict_gtt", &amdgpu_debugfs_evict_gtt},
1237 static void amdgpu_ib_preempt_fences_swap(struct amdgpu_ring *ring,
1238 struct dma_fence **fences)
1240 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1241 uint32_t sync_seq, last_seq;
1243 last_seq = atomic_read(&ring->fence_drv.last_seq);
1244 sync_seq = ring->fence_drv.sync_seq;
1246 last_seq &= drv->num_fences_mask;
1247 sync_seq &= drv->num_fences_mask;
1250 struct dma_fence *fence, **ptr;
1253 last_seq &= drv->num_fences_mask;
1254 ptr = &drv->fences[last_seq];
1256 fence = rcu_dereference_protected(*ptr, 1);
1257 RCU_INIT_POINTER(*ptr, NULL);
1262 fences[last_seq] = fence;
1264 } while (last_seq != sync_seq);
1267 static void amdgpu_ib_preempt_signal_fences(struct dma_fence **fences,
1271 struct dma_fence *fence;
1273 for (i = 0; i < length; i++) {
1277 dma_fence_signal(fence);
1278 dma_fence_put(fence);
1282 static void amdgpu_ib_preempt_job_recovery(struct drm_gpu_scheduler *sched)
1284 struct drm_sched_job *s_job;
1285 struct dma_fence *fence;
1287 spin_lock(&sched->job_list_lock);
1288 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1289 fence = sched->ops->run_job(s_job);
1290 dma_fence_put(fence);
1292 spin_unlock(&sched->job_list_lock);
1295 static void amdgpu_ib_preempt_mark_partial_job(struct amdgpu_ring *ring)
1297 struct amdgpu_job *job;
1298 struct drm_sched_job *s_job;
1299 uint32_t preempt_seq;
1300 struct dma_fence *fence, **ptr;
1301 struct amdgpu_fence_driver *drv = &ring->fence_drv;
1302 struct drm_gpu_scheduler *sched = &ring->sched;
1304 if (ring->funcs->type != AMDGPU_RING_TYPE_GFX)
1307 preempt_seq = le32_to_cpu(*(drv->cpu_addr + 2));
1308 if (preempt_seq <= atomic_read(&drv->last_seq))
1311 preempt_seq &= drv->num_fences_mask;
1312 ptr = &drv->fences[preempt_seq];
1313 fence = rcu_dereference_protected(*ptr, 1);
1315 spin_lock(&sched->job_list_lock);
1316 list_for_each_entry(s_job, &sched->ring_mirror_list, node) {
1317 job = to_amdgpu_job(s_job);
1318 if (job->fence == fence)
1319 /* mark the job as preempted */
1320 job->preemption_status |= AMDGPU_IB_PREEMPTED;
1322 spin_unlock(&sched->job_list_lock);
1325 static int amdgpu_debugfs_ib_preempt(void *data, u64 val)
1327 int r, resched, length;
1328 struct amdgpu_ring *ring;
1329 struct dma_fence **fences = NULL;
1330 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1332 if (val >= AMDGPU_MAX_RINGS)
1335 ring = adev->rings[val];
1337 if (!ring || !ring->funcs->preempt_ib || !ring->sched.thread)
1340 /* the last preemption failed */
1341 if (ring->trail_seq != le32_to_cpu(*ring->trail_fence_cpu_addr))
1344 length = ring->fence_drv.num_fences_mask + 1;
1345 fences = kcalloc(length, sizeof(void *), GFP_KERNEL);
1349 /* Avoid accidently unparking the sched thread during GPU reset */
1350 mutex_lock(&adev->lock_reset);
1352 /* stop the scheduler */
1353 kthread_park(ring->sched.thread);
1355 resched = ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
1357 /* preempt the IB */
1358 r = amdgpu_ring_preempt_ib(ring);
1360 DRM_WARN("failed to preempt ring %d\n", ring->idx);
1364 amdgpu_fence_process(ring);
1366 if (atomic_read(&ring->fence_drv.last_seq) !=
1367 ring->fence_drv.sync_seq) {
1368 DRM_INFO("ring %d was preempted\n", ring->idx);
1370 amdgpu_ib_preempt_mark_partial_job(ring);
1372 /* swap out the old fences */
1373 amdgpu_ib_preempt_fences_swap(ring, fences);
1375 amdgpu_fence_driver_force_completion(ring);
1377 /* resubmit unfinished jobs */
1378 amdgpu_ib_preempt_job_recovery(&ring->sched);
1380 /* wait for jobs finished */
1381 amdgpu_fence_wait_empty(ring);
1383 /* signal the old fences */
1384 amdgpu_ib_preempt_signal_fences(fences, length);
1388 /* restart the scheduler */
1389 kthread_unpark(ring->sched.thread);
1391 mutex_unlock(&adev->lock_reset);
1393 ttm_bo_unlock_delayed_workqueue(&adev->mman.bdev, resched);
1400 static int amdgpu_debugfs_sclk_set(void *data, u64 val)
1403 uint32_t max_freq, min_freq;
1404 struct amdgpu_device *adev = (struct amdgpu_device *)data;
1406 if (amdgpu_sriov_vf(adev) && !amdgpu_sriov_is_pp_one_vf(adev))
1409 ret = pm_runtime_get_sync(adev->ddev->dev);
1413 if (is_support_sw_smu(adev)) {
1414 ret = smu_get_dpm_freq_range(&adev->smu, SMU_SCLK, &min_freq, &max_freq, true);
1415 if (ret || val > max_freq || val < min_freq)
1417 ret = smu_set_soft_freq_range(&adev->smu, SMU_SCLK, (uint32_t)val, (uint32_t)val, true);
1422 pm_runtime_mark_last_busy(adev->ddev->dev);
1423 pm_runtime_put_autosuspend(adev->ddev->dev);
1431 DEFINE_SIMPLE_ATTRIBUTE(fops_ib_preempt, NULL,
1432 amdgpu_debugfs_ib_preempt, "%llu\n");
1434 DEFINE_SIMPLE_ATTRIBUTE(fops_sclk_set, NULL,
1435 amdgpu_debugfs_sclk_set, "%llu\n");
1437 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1441 adev->debugfs_preempt =
1442 debugfs_create_file("amdgpu_preempt_ib", 0600,
1443 adev->ddev->primary->debugfs_root, adev,
1445 if (!(adev->debugfs_preempt)) {
1446 DRM_ERROR("unable to create amdgpu_preempt_ib debugsfs file\n");
1450 adev->smu.debugfs_sclk =
1451 debugfs_create_file("amdgpu_force_sclk", 0200,
1452 adev->ddev->primary->debugfs_root, adev,
1454 if (!(adev->smu.debugfs_sclk)) {
1455 DRM_ERROR("unable to create amdgpu_set_sclk debugsfs file\n");
1459 /* Register debugfs entries for amdgpu_ttm */
1460 r = amdgpu_ttm_debugfs_init(adev);
1462 DRM_ERROR("Failed to init debugfs\n");
1466 r = amdgpu_debugfs_pm_init(adev);
1468 DRM_ERROR("Failed to register debugfs file for dpm!\n");
1472 if (amdgpu_debugfs_sa_init(adev)) {
1473 dev_err(adev->dev, "failed to register debugfs file for SA\n");
1476 if (amdgpu_debugfs_fence_init(adev))
1477 dev_err(adev->dev, "fence debugfs file creation failed\n");
1479 r = amdgpu_debugfs_gem_init(adev);
1481 DRM_ERROR("registering gem debugfs failed (%d).\n", r);
1483 r = amdgpu_debugfs_regs_init(adev);
1485 DRM_ERROR("registering register debugfs failed (%d).\n", r);
1487 r = amdgpu_debugfs_firmware_init(adev);
1489 DRM_ERROR("registering firmware debugfs failed (%d).\n", r);
1491 #if defined(CONFIG_DRM_AMD_DC)
1492 if (amdgpu_device_has_dc_support(adev)) {
1493 if (dtn_debugfs_init(adev))
1494 DRM_ERROR("amdgpu: failed initialize dtn debugfs support.\n");
1498 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
1499 struct amdgpu_ring *ring = adev->rings[i];
1504 if (amdgpu_debugfs_ring_init(adev, ring)) {
1505 DRM_ERROR("Failed to register debugfs file for rings !\n");
1509 amdgpu_ras_debugfs_create_all(adev);
1511 amdgpu_debugfs_autodump_init(adev);
1513 return amdgpu_debugfs_add_files(adev, amdgpu_debugfs_list,
1514 ARRAY_SIZE(amdgpu_debugfs_list));
1518 int amdgpu_debugfs_init(struct amdgpu_device *adev)
1522 int amdgpu_debugfs_regs_init(struct amdgpu_device *adev)