drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34
  35 #include <drm/drm_atomic_helper.h>
  36 #include <drm/drm_probe_helper.h>
  37 #include <drm/amdgpu_drm.h>
  38 #include <linux/vgaarb.h>
  39 #include <linux/vga_switcheroo.h>
  40 #include <linux/efi.h>
  41 #include "amdgpu.h"
  42 #include "amdgpu_trace.h"
  43 #include "amdgpu_i2c.h"
  44 #include "atom.h"
  45 #include "amdgpu_atombios.h"
  46 #include "amdgpu_atomfirmware.h"
  47 #include "amd_pcie.h"
  48 #ifdef CONFIG_DRM_AMDGPU_SI
  49 #include "si.h"
  50 #endif
  51 #ifdef CONFIG_DRM_AMDGPU_CIK
  52 #include "cik.h"
  53 #endif
  54 #include "vi.h"
  55 #include "soc15.h"
  56 #include "nv.h"
  57 #include "bif/bif_4_1_d.h"
  58 #include <linux/pci.h>
  59 #include <linux/firmware.h>
  60 #include "amdgpu_vf_error.h"
  61
  62 #include "amdgpu_amdkfd.h"
  63 #include "amdgpu_pm.h"
  64
  65 #include "amdgpu_xgmi.h"
  66 #include "amdgpu_ras.h"
  67 #include "amdgpu_pmu.h"
  68 #include "amdgpu_fru_eeprom.h"
  69 #include "amdgpu_reset.h"
  70
  71 #include <linux/suspend.h>
  72 #include <drm/task_barrier.h>
  73 #include <linux/pm_runtime.h>
  74
  75 #include <drm/drm_drv.h>
  76
  77 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  78 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  79 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  80 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  81 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  87 MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
  88 MODULE_FIRMWARE("amdgpu/yellow_carp_gpu_info.bin");
  89
  90 #define AMDGPU_RESUME_MS                2000
  91
  92 const char *amdgpu_asic_name[] = {
  93         "TAHITI",
  94         "PITCAIRN",
  95         "VERDE",
  96         "OLAND",
  97         "HAINAN",
  98         "BONAIRE",
  99         "KAVERI",
 100         "KABINI",
 101         "HAWAII",
 102         "MULLINS",
 103         "TOPAZ",
 104         "TONGA",
 105         "FIJI",
 106         "CARRIZO",
 107         "STONEY",
 108         "POLARIS10",
 109         "POLARIS11",
 110         "POLARIS12",
 111         "VEGAM",
 112         "VEGA10",
 113         "VEGA12",
 114         "VEGA20",
 115         "RAVEN",
 116         "ARCTURUS",
 117         "RENOIR",
 118         "ALDEBARAN",
 119         "NAVI10",
 120         "CYAN_SKILLFISH",
 121         "NAVI14",
 122         "NAVI12",
 123         "SIENNA_CICHLID",
 124         "NAVY_FLOUNDER",
 125         "VANGOGH",
 126         "DIMGREY_CAVEFISH",
 127         "BEIGE_GOBY",
 128         "YELLOW_CARP",
 129         "IP DISCOVERY",
 130         "LAST",
 131 };
 132
 133 /**
 134  * DOC: pcie_replay_count
 135  *
 136  * The amdgpu driver provides a sysfs API for reporting the total number
 137  * of PCIe replays (NAKs)
 138  * The file pcie_replay_count is used for this and returns the total
 139  * number of replays as a sum of the NAKs generated and NAKs received
 140  */
 141
 142 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 143                 struct device_attribute *attr, char *buf)
 144 {
 145         struct drm_device *ddev = dev_get_drvdata(dev);
 146         struct amdgpu_device *adev = drm_to_adev(ddev);
 147         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 148
 149         return sysfs_emit(buf, "%llu\n", cnt);
 150 }
 151
 152 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 153                 amdgpu_device_get_pcie_replay_count, NULL);
 154
 155 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 156
 157 /**
 158  * DOC: product_name
 159  *
 160  * The amdgpu driver provides a sysfs API for reporting the product name
 161  * for the device
 162  * The file serial_number is used for this and returns the product name
 163  * as returned from the FRU.
 164  * NOTE: This is only available for certain server cards
 165  */
 166
 167 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 168                 struct device_attribute *attr, char *buf)
 169 {
 170         struct drm_device *ddev = dev_get_drvdata(dev);
 171         struct amdgpu_device *adev = drm_to_adev(ddev);
 172
 173         return sysfs_emit(buf, "%s\n", adev->product_name);
 174 }
 175
 176 static DEVICE_ATTR(product_name, S_IRUGO,
 177                 amdgpu_device_get_product_name, NULL);
 178
 179 /**
 180  * DOC: product_number
 181  *
 182  * The amdgpu driver provides a sysfs API for reporting the part number
 183  * for the device
 184  * The file serial_number is used for this and returns the part number
 185  * as returned from the FRU.
 186  * NOTE: This is only available for certain server cards
 187  */
 188
 189 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 190                 struct device_attribute *attr, char *buf)
 191 {
 192         struct drm_device *ddev = dev_get_drvdata(dev);
 193         struct amdgpu_device *adev = drm_to_adev(ddev);
 194
 195         return sysfs_emit(buf, "%s\n", adev->product_number);
 196 }
 197
 198 static DEVICE_ATTR(product_number, S_IRUGO,
 199                 amdgpu_device_get_product_number, NULL);
 200
 201 /**
 202  * DOC: serial_number
 203  *
 204  * The amdgpu driver provides a sysfs API for reporting the serial number
 205  * for the device
 206  * The file serial_number is used for this and returns the serial number
 207  * as returned from the FRU.
 208  * NOTE: This is only available for certain server cards
 209  */
 210
 211 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 212                 struct device_attribute *attr, char *buf)
 213 {
 214         struct drm_device *ddev = dev_get_drvdata(dev);
 215         struct amdgpu_device *adev = drm_to_adev(ddev);
 216
 217         return sysfs_emit(buf, "%s\n", adev->serial);
 218 }
 219
 220 static DEVICE_ATTR(serial_number, S_IRUGO,
 221                 amdgpu_device_get_serial_number, NULL);
 222
 223 /**
 224  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 225  *
 226  * @dev: drm_device pointer
 227  *
 228  * Returns true if the device is a dGPU with ATPX power control,
 229  * otherwise return false.
 230  */
 231 bool amdgpu_device_supports_px(struct drm_device *dev)
 232 {
 233         struct amdgpu_device *adev = drm_to_adev(dev);
 234
 235         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 236                 return true;
 237         return false;
 238 }
 239
 240 /**
 241  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 242  *
 243  * @dev: drm_device pointer
 244  *
 245  * Returns true if the device is a dGPU with ACPI power control,
 246  * otherwise return false.
 247  */
 248 bool amdgpu_device_supports_boco(struct drm_device *dev)
 249 {
 250         struct amdgpu_device *adev = drm_to_adev(dev);
 251
 252         if (adev->has_pr3 ||
 253             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 254                 return true;
 255         return false;
 256 }
 257
 258 /**
 259  * amdgpu_device_supports_baco - Does the device support BACO
 260  *
 261  * @dev: drm_device pointer
 262  *
 263  * Returns true if the device supporte BACO,
 264  * otherwise return false.
 265  */
 266 bool amdgpu_device_supports_baco(struct drm_device *dev)
 267 {
 268         struct amdgpu_device *adev = drm_to_adev(dev);
 269
 270         return amdgpu_asic_supports_baco(adev);
 271 }
 272
 273 /**
 274  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 275  * smart shift support
 276  *
 277  * @dev: drm_device pointer
 278  *
 279  * Returns true if the device is a dGPU with Smart Shift support,
 280  * otherwise returns false.
 281  */
 282 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 283 {
 284         return (amdgpu_device_supports_boco(dev) &&
 285                 amdgpu_acpi_is_power_shift_control_supported());
 286 }
 287
 288 /*
 289  * VRAM access helper functions
 290  */
 291
 292 /**
 293  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 294  *
 295  * @adev: amdgpu_device pointer
 296  * @pos: offset of the buffer in vram
 297  * @buf: virtual address of the buffer in system memory
 298  * @size: read/write size, sizeof(@buf) must > @size
 299  * @write: true - write to vram, otherwise - read from vram
 300  */
 301 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 302                              void *buf, size_t size, bool write)
 303 {
 304         unsigned long flags;
 305         uint32_t hi = ~0, tmp = 0;
 306         uint32_t *data = buf;
 307         uint64_t last;
 308         int idx;
 309
 310         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 311                 return;
 312
 313         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 314
 315         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 316         for (last = pos + size; pos < last; pos += 4) {
 317                 tmp = pos >> 31;
 318
 319                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 320                 if (tmp != hi) {
 321                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 322                         hi = tmp;
 323                 }
 324                 if (write)
 325                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 326                 else
 327                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 328         }
 329
 330         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 331         drm_dev_exit(idx);
 332 }
 333
 334 /**
 335  * amdgpu_device_aper_access - access vram by vram aperature
 336  *
 337  * @adev: amdgpu_device pointer
 338  * @pos: offset of the buffer in vram
 339  * @buf: virtual address of the buffer in system memory
 340  * @size: read/write size, sizeof(@buf) must > @size
 341  * @write: true - write to vram, otherwise - read from vram
 342  *
 343  * The return value means how many bytes have been transferred.
 344  */
 345 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 346                                  void *buf, size_t size, bool write)
 347 {
 348 #ifdef CONFIG_64BIT
 349         void __iomem *addr;
 350         size_t count = 0;
 351         uint64_t last;
 352
 353         if (!adev->mman.aper_base_kaddr)
 354                 return 0;
 355
 356         last = min(pos + size, adev->gmc.visible_vram_size);
 357         if (last > pos) {
 358                 addr = adev->mman.aper_base_kaddr + pos;
 359                 count = last - pos;
 360
 361                 if (write) {
 362                         memcpy_toio(addr, buf, count);
 363                         mb();
 364                         amdgpu_device_flush_hdp(adev, NULL);
 365                 } else {
 366                         amdgpu_device_invalidate_hdp(adev, NULL);
 367                         mb();
 368                         memcpy_fromio(buf, addr, count);
 369                 }
 370
 371         }
 372
 373         return count;
 374 #else
 375         return 0;
 376 #endif
 377 }
 378
 379 /**
 380  * amdgpu_device_vram_access - read/write a buffer in vram
 381  *
 382  * @adev: amdgpu_device pointer
 383  * @pos: offset of the buffer in vram
 384  * @buf: virtual address of the buffer in system memory
 385  * @size: read/write size, sizeof(@buf) must > @size
 386  * @write: true - write to vram, otherwise - read from vram
 387  */
 388 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 389                                void *buf, size_t size, bool write)
 390 {
 391         size_t count;
 392
 393         /* try to using vram apreature to access vram first */
 394         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 395         size -= count;
 396         if (size) {
 397                 /* using MM to access rest vram */
 398                 pos += count;
 399                 buf += count;
 400                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 401         }
 402 }
 403
 404 /*
 405  * register access helper functions.
 406  */
 407
 408 /* Check if hw access should be skipped because of hotplug or device error */
 409 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 410 {
 411         if (adev->no_hw_access)
 412                 return true;
 413
 414 #ifdef CONFIG_LOCKDEP
 415         /*
 416          * This is a bit complicated to understand, so worth a comment. What we assert
 417          * here is that the GPU reset is not running on another thread in parallel.
 418          *
 419          * For this we trylock the read side of the reset semaphore, if that succeeds
 420          * we know that the reset is not running in paralell.
 421          *
 422          * If the trylock fails we assert that we are either already holding the read
 423          * side of the lock or are the reset thread itself and hold the write side of
 424          * the lock.
 425          */
 426         if (in_task()) {
 427                 if (down_read_trylock(&adev->reset_sem))
 428                         up_read(&adev->reset_sem);
 429                 else
 430                         lockdep_assert_held(&adev->reset_sem);
 431         }
 432 #endif
 433         return false;
 434 }
 435
 436 /**
 437  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 438  *
 439  * @adev: amdgpu_device pointer
 440  * @reg: dword aligned register offset
 441  * @acc_flags: access flags which require special behavior
 442  *
 443  * Returns the 32 bit value from the offset specified.
 444  */
 445 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 446                             uint32_t reg, uint32_t acc_flags)
 447 {
 448         uint32_t ret;
 449
 450         if (amdgpu_device_skip_hw_access(adev))
 451                 return 0;
 452
 453         if ((reg * 4) < adev->rmmio_size) {
 454                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 455                     amdgpu_sriov_runtime(adev) &&
 456                     down_read_trylock(&adev->reset_sem)) {
 457                         ret = amdgpu_kiq_rreg(adev, reg);
 458                         up_read(&adev->reset_sem);
 459                 } else {
 460                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 461                 }
 462         } else {
 463                 ret = adev->pcie_rreg(adev, reg * 4);
 464         }
 465
 466         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 467
 468         return ret;
 469 }
 470
 471 /*
 472  * MMIO register read with bytes helper functions
 473  * @offset:bytes offset from MMIO start
 474  *
 475 */
 476
 477 /**
 478  * amdgpu_mm_rreg8 - read a memory mapped IO register
 479  *
 480  * @adev: amdgpu_device pointer
 481  * @offset: byte aligned register offset
 482  *
 483  * Returns the 8 bit value from the offset specified.
 484  */
 485 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 486 {
 487         if (amdgpu_device_skip_hw_access(adev))
 488                 return 0;
 489
 490         if (offset < adev->rmmio_size)
 491                 return (readb(adev->rmmio + offset));
 492         BUG();
 493 }
 494
 495 /*
 496  * MMIO register write with bytes helper functions
 497  * @offset:bytes offset from MMIO start
 498  * @value: the value want to be written to the register
 499  *
 500 */
 501 /**
 502  * amdgpu_mm_wreg8 - read a memory mapped IO register
 503  *
 504  * @adev: amdgpu_device pointer
 505  * @offset: byte aligned register offset
 506  * @value: 8 bit value to write
 507  *
 508  * Writes the value specified to the offset specified.
 509  */
 510 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 511 {
 512         if (amdgpu_device_skip_hw_access(adev))
 513                 return;
 514
 515         if (offset < adev->rmmio_size)
 516                 writeb(value, adev->rmmio + offset);
 517         else
 518                 BUG();
 519 }
 520
 521 /**
 522  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 523  *
 524  * @adev: amdgpu_device pointer
 525  * @reg: dword aligned register offset
 526  * @v: 32 bit value to write to the register
 527  * @acc_flags: access flags which require special behavior
 528  *
 529  * Writes the value specified to the offset specified.
 530  */
 531 void amdgpu_device_wreg(struct amdgpu_device *adev,
 532                         uint32_t reg, uint32_t v,
 533                         uint32_t acc_flags)
 534 {
 535         if (amdgpu_device_skip_hw_access(adev))
 536                 return;
 537
 538         if ((reg * 4) < adev->rmmio_size) {
 539                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 540                     amdgpu_sriov_runtime(adev) &&
 541                     down_read_trylock(&adev->reset_sem)) {
 542                         amdgpu_kiq_wreg(adev, reg, v);
 543                         up_read(&adev->reset_sem);
 544                 } else {
 545                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 546                 }
 547         } else {
 548                 adev->pcie_wreg(adev, reg * 4, v);
 549         }
 550
 551         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 552 }
 553
 554 /*
 555  * amdgpu_mm_wreg_mmio_rlc -  write register either with mmio or with RLC path if in range
 556  *
 557  * this function is invoked only the debugfs register access
 558  * */
 559 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 560                              uint32_t reg, uint32_t v)
 561 {
 562         if (amdgpu_device_skip_hw_access(adev))
 563                 return;
 564
 565         if (amdgpu_sriov_fullaccess(adev) &&
 566             adev->gfx.rlc.funcs &&
 567             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 568                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 569                         return adev->gfx.rlc.funcs->sriov_wreg(adev, reg, v, 0, 0);
 570         } else {
 571                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 572         }
 573 }
 574
 575 /**
 576  * amdgpu_mm_rdoorbell - read a doorbell dword
 577  *
 578  * @adev: amdgpu_device pointer
 579  * @index: doorbell index
 580  *
 581  * Returns the value in the doorbell aperture at the
 582  * requested doorbell index (CIK).
 583  */
 584 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 585 {
 586         if (amdgpu_device_skip_hw_access(adev))
 587                 return 0;
 588
 589         if (index < adev->doorbell.num_doorbells) {
 590                 return readl(adev->doorbell.ptr + index);
 591         } else {
 592                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 593                 return 0;
 594         }
 595 }
 596
 597 /**
 598  * amdgpu_mm_wdoorbell - write a doorbell dword
 599  *
 600  * @adev: amdgpu_device pointer
 601  * @index: doorbell index
 602  * @v: value to write
 603  *
 604  * Writes @v to the doorbell aperture at the
 605  * requested doorbell index (CIK).
 606  */
 607 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 608 {
 609         if (amdgpu_device_skip_hw_access(adev))
 610                 return;
 611
 612         if (index < adev->doorbell.num_doorbells) {
 613                 writel(v, adev->doorbell.ptr + index);
 614         } else {
 615                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 616         }
 617 }
 618
 619 /**
 620  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 621  *
 622  * @adev: amdgpu_device pointer
 623  * @index: doorbell index
 624  *
 625  * Returns the value in the doorbell aperture at the
 626  * requested doorbell index (VEGA10+).
 627  */
 628 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 629 {
 630         if (amdgpu_device_skip_hw_access(adev))
 631                 return 0;
 632
 633         if (index < adev->doorbell.num_doorbells) {
 634                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 635         } else {
 636                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 637                 return 0;
 638         }
 639 }
 640
 641 /**
 642  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 643  *
 644  * @adev: amdgpu_device pointer
 645  * @index: doorbell index
 646  * @v: value to write
 647  *
 648  * Writes @v to the doorbell aperture at the
 649  * requested doorbell index (VEGA10+).
 650  */
 651 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 652 {
 653         if (amdgpu_device_skip_hw_access(adev))
 654                 return;
 655
 656         if (index < adev->doorbell.num_doorbells) {
 657                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 658         } else {
 659                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 660         }
 661 }
 662
 663 /**
 664  * amdgpu_device_indirect_rreg - read an indirect register
 665  *
 666  * @adev: amdgpu_device pointer
 667  * @pcie_index: mmio register offset
 668  * @pcie_data: mmio register offset
 669  * @reg_addr: indirect register address to read from
 670  *
 671  * Returns the value of indirect register @reg_addr
 672  */
 673 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 674                                 u32 pcie_index, u32 pcie_data,
 675                                 u32 reg_addr)
 676 {
 677         unsigned long flags;
 678         u32 r;
 679         void __iomem *pcie_index_offset;
 680         void __iomem *pcie_data_offset;
 681
 682         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 683         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 684         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 685
 686         writel(reg_addr, pcie_index_offset);
 687         readl(pcie_index_offset);
 688         r = readl(pcie_data_offset);
 689         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 690
 691         return r;
 692 }
 693
 694 /**
 695  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 696  *
 697  * @adev: amdgpu_device pointer
 698  * @pcie_index: mmio register offset
 699  * @pcie_data: mmio register offset
 700  * @reg_addr: indirect register address to read from
 701  *
 702  * Returns the value of indirect register @reg_addr
 703  */
 704 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 705                                   u32 pcie_index, u32 pcie_data,
 706                                   u32 reg_addr)
 707 {
 708         unsigned long flags;
 709         u64 r;
 710         void __iomem *pcie_index_offset;
 711         void __iomem *pcie_data_offset;
 712
 713         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 714         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 715         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 716
 717         /* read low 32 bits */
 718         writel(reg_addr, pcie_index_offset);
 719         readl(pcie_index_offset);
 720         r = readl(pcie_data_offset);
 721         /* read high 32 bits */
 722         writel(reg_addr + 4, pcie_index_offset);
 723         readl(pcie_index_offset);
 724         r |= ((u64)readl(pcie_data_offset) << 32);
 725         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 726
 727         return r;
 728 }
 729
 730 /**
 731  * amdgpu_device_indirect_wreg - write an indirect register address
 732  *
 733  * @adev: amdgpu_device pointer
 734  * @pcie_index: mmio register offset
 735  * @pcie_data: mmio register offset
 736  * @reg_addr: indirect register offset
 737  * @reg_data: indirect register data
 738  *
 739  */
 740 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 741                                  u32 pcie_index, u32 pcie_data,
 742                                  u32 reg_addr, u32 reg_data)
 743 {
 744         unsigned long flags;
 745         void __iomem *pcie_index_offset;
 746         void __iomem *pcie_data_offset;
 747
 748         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 749         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 750         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 751
 752         writel(reg_addr, pcie_index_offset);
 753         readl(pcie_index_offset);
 754         writel(reg_data, pcie_data_offset);
 755         readl(pcie_data_offset);
 756         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 757 }
 758
 759 /**
 760  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 761  *
 762  * @adev: amdgpu_device pointer
 763  * @pcie_index: mmio register offset
 764  * @pcie_data: mmio register offset
 765  * @reg_addr: indirect register offset
 766  * @reg_data: indirect register data
 767  *
 768  */
 769 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 770                                    u32 pcie_index, u32 pcie_data,
 771                                    u32 reg_addr, u64 reg_data)
 772 {
 773         unsigned long flags;
 774         void __iomem *pcie_index_offset;
 775         void __iomem *pcie_data_offset;
 776
 777         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 778         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 779         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 780
 781         /* write low 32 bits */
 782         writel(reg_addr, pcie_index_offset);
 783         readl(pcie_index_offset);
 784         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 785         readl(pcie_data_offset);
 786         /* write high 32 bits */
 787         writel(reg_addr + 4, pcie_index_offset);
 788         readl(pcie_index_offset);
 789         writel((u32)(reg_data >> 32), pcie_data_offset);
 790         readl(pcie_data_offset);
 791         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 792 }
 793
 794 /**
 795  * amdgpu_invalid_rreg - dummy reg read function
 796  *
 797  * @adev: amdgpu_device pointer
 798  * @reg: offset of register
 799  *
 800  * Dummy register read function.  Used for register blocks
 801  * that certain asics don't have (all asics).
 802  * Returns the value in the register.
 803  */
 804 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 805 {
 806         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 807         BUG();
 808         return 0;
 809 }
 810
 811 /**
 812  * amdgpu_invalid_wreg - dummy reg write function
 813  *
 814  * @adev: amdgpu_device pointer
 815  * @reg: offset of register
 816  * @v: value to write to the register
 817  *
 818  * Dummy register read function.  Used for register blocks
 819  * that certain asics don't have (all asics).
 820  */
 821 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 822 {
 823         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 824                   reg, v);
 825         BUG();
 826 }
 827
 828 /**
 829  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 830  *
 831  * @adev: amdgpu_device pointer
 832  * @reg: offset of register
 833  *
 834  * Dummy register read function.  Used for register blocks
 835  * that certain asics don't have (all asics).
 836  * Returns the value in the register.
 837  */
 838 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 839 {
 840         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 841         BUG();
 842         return 0;
 843 }
 844
 845 /**
 846  * amdgpu_invalid_wreg64 - dummy reg write function
 847  *
 848  * @adev: amdgpu_device pointer
 849  * @reg: offset of register
 850  * @v: value to write to the register
 851  *
 852  * Dummy register read function.  Used for register blocks
 853  * that certain asics don't have (all asics).
 854  */
 855 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 856 {
 857         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 858                   reg, v);
 859         BUG();
 860 }
 861
 862 /**
 863  * amdgpu_block_invalid_rreg - dummy reg read function
 864  *
 865  * @adev: amdgpu_device pointer
 866  * @block: offset of instance
 867  * @reg: offset of register
 868  *
 869  * Dummy register read function.  Used for register blocks
 870  * that certain asics don't have (all asics).
 871  * Returns the value in the register.
 872  */
 873 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 874                                           uint32_t block, uint32_t reg)
 875 {
 876         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 877                   reg, block);
 878         BUG();
 879         return 0;
 880 }
 881
 882 /**
 883  * amdgpu_block_invalid_wreg - dummy reg write function
 884  *
 885  * @adev: amdgpu_device pointer
 886  * @block: offset of instance
 887  * @reg: offset of register
 888  * @v: value to write to the register
 889  *
 890  * Dummy register read function.  Used for register blocks
 891  * that certain asics don't have (all asics).
 892  */
 893 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 894                                       uint32_t block,
 895                                       uint32_t reg, uint32_t v)
 896 {
 897         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 898                   reg, block, v);
 899         BUG();
 900 }
 901
 902 /**
 903  * amdgpu_device_asic_init - Wrapper for atom asic_init
 904  *
 905  * @adev: amdgpu_device pointer
 906  *
 907  * Does any asic specific work and then calls atom asic init.
 908  */
 909 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 910 {
 911         amdgpu_asic_pre_asic_init(adev);
 912
 913         return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 914 }
 915
 916 /**
 917  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 918  *
 919  * @adev: amdgpu_device pointer
 920  *
 921  * Allocates a scratch page of VRAM for use by various things in the
 922  * driver.
 923  */
 924 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 925 {
 926         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 927                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 928                                        &adev->vram_scratch.robj,
 929                                        &adev->vram_scratch.gpu_addr,
 930                                        (void **)&adev->vram_scratch.ptr);
 931 }
 932
 933 /**
 934  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 935  *
 936  * @adev: amdgpu_device pointer
 937  *
 938  * Frees the VRAM scratch page.
 939  */
 940 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 941 {
 942         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 943 }
 944
 945 /**
 946  * amdgpu_device_program_register_sequence - program an array of registers.
 947  *
 948  * @adev: amdgpu_device pointer
 949  * @registers: pointer to the register array
 950  * @array_size: size of the register array
 951  *
 952  * Programs an array or registers with and and or masks.
 953  * This is a helper for setting golden registers.
 954  */
 955 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 956                                              const u32 *registers,
 957                                              const u32 array_size)
 958 {
 959         u32 tmp, reg, and_mask, or_mask;
 960         int i;
 961
 962         if (array_size % 3)
 963                 return;
 964
 965         for (i = 0; i < array_size; i +=3) {
 966                 reg = registers[i + 0];
 967                 and_mask = registers[i + 1];
 968                 or_mask = registers[i + 2];
 969
 970                 if (and_mask == 0xffffffff) {
 971                         tmp = or_mask;
 972                 } else {
 973                         tmp = RREG32(reg);
 974                         tmp &= ~and_mask;
 975                         if (adev->family >= AMDGPU_FAMILY_AI)
 976                                 tmp |= (or_mask & and_mask);
 977                         else
 978                                 tmp |= or_mask;
 979                 }
 980                 WREG32(reg, tmp);
 981         }
 982 }
 983
 984 /**
 985  * amdgpu_device_pci_config_reset - reset the GPU
 986  *
 987  * @adev: amdgpu_device pointer
 988  *
 989  * Resets the GPU using the pci config reset sequence.
 990  * Only applicable to asics prior to vega10.
 991  */
 992 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 993 {
 994         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 995 }
 996
 997 /**
 998  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
 999  *
1000  * @adev: amdgpu_device pointer
1001  *
1002  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1003  */
1004 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1005 {
1006         return pci_reset_function(adev->pdev);
1007 }
1008
1009 /*
1010  * GPU doorbell aperture helpers function.
1011  */
1012 /**
1013  * amdgpu_device_doorbell_init - Init doorbell driver information.
1014  *
1015  * @adev: amdgpu_device pointer
1016  *
1017  * Init doorbell driver information (CIK)
1018  * Returns 0 on success, error on failure.
1019  */
1020 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1021 {
1022
1023         /* No doorbell on SI hardware generation */
1024         if (adev->asic_type < CHIP_BONAIRE) {
1025                 adev->doorbell.base = 0;
1026                 adev->doorbell.size = 0;
1027                 adev->doorbell.num_doorbells = 0;
1028                 adev->doorbell.ptr = NULL;
1029                 return 0;
1030         }
1031
1032         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1033                 return -EINVAL;
1034
1035         amdgpu_asic_init_doorbell_index(adev);
1036
1037         /* doorbell bar mapping */
1038         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1039         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1040
1041         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
1042                                              adev->doorbell_index.max_assignment+1);
1043         if (adev->doorbell.num_doorbells == 0)
1044                 return -EINVAL;
1045
1046         /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1047          * paging queue doorbell use the second page. The
1048          * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1049          * doorbells are in the first page. So with paging queue enabled,
1050          * the max num_doorbells should + 1 page (0x400 in dword)
1051          */
1052         if (adev->asic_type >= CHIP_VEGA10)
1053                 adev->doorbell.num_doorbells += 0x400;
1054
1055         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1056                                      adev->doorbell.num_doorbells *
1057                                      sizeof(u32));
1058         if (adev->doorbell.ptr == NULL)
1059                 return -ENOMEM;
1060
1061         return 0;
1062 }
1063
1064 /**
1065  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1066  *
1067  * @adev: amdgpu_device pointer
1068  *
1069  * Tear down doorbell driver information (CIK)
1070  */
1071 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1072 {
1073         iounmap(adev->doorbell.ptr);
1074         adev->doorbell.ptr = NULL;
1075 }
1076
1077
1078
1079 /*
1080  * amdgpu_device_wb_*()
1081  * Writeback is the method by which the GPU updates special pages in memory
1082  * with the status of certain GPU events (fences, ring pointers,etc.).
1083  */
1084
1085 /**
1086  * amdgpu_device_wb_fini - Disable Writeback and free memory
1087  *
1088  * @adev: amdgpu_device pointer
1089  *
1090  * Disables Writeback and frees the Writeback memory (all asics).
1091  * Used at driver shutdown.
1092  */
1093 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1094 {
1095         if (adev->wb.wb_obj) {
1096                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1097                                       &adev->wb.gpu_addr,
1098                                       (void **)&adev->wb.wb);
1099                 adev->wb.wb_obj = NULL;
1100         }
1101 }
1102
1103 /**
1104  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1105  *
1106  * @adev: amdgpu_device pointer
1107  *
1108  * Initializes writeback and allocates writeback memory (all asics).
1109  * Used at driver startup.
1110  * Returns 0 on success or an -error on failure.
1111  */
1112 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1113 {
1114         int r;
1115
1116         if (adev->wb.wb_obj == NULL) {
1117                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1118                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1119                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1120                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1121                                             (void **)&adev->wb.wb);
1122                 if (r) {
1123                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1124                         return r;
1125                 }
1126
1127                 adev->wb.num_wb = AMDGPU_MAX_WB;
1128                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1129
1130                 /* clear wb memory */
1131                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1132         }
1133
1134         return 0;
1135 }
1136
1137 /**
1138  * amdgpu_device_wb_get - Allocate a wb entry
1139  *
1140  * @adev: amdgpu_device pointer
1141  * @wb: wb index
1142  *
1143  * Allocate a wb slot for use by the driver (all asics).
1144  * Returns 0 on success or -EINVAL on failure.
1145  */
1146 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1147 {
1148         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1149
1150         if (offset < adev->wb.num_wb) {
1151                 __set_bit(offset, adev->wb.used);
1152                 *wb = offset << 3; /* convert to dw offset */
1153                 return 0;
1154         } else {
1155                 return -EINVAL;
1156         }
1157 }
1158
1159 /**
1160  * amdgpu_device_wb_free - Free a wb entry
1161  *
1162  * @adev: amdgpu_device pointer
1163  * @wb: wb index
1164  *
1165  * Free a wb slot allocated for use by the driver (all asics)
1166  */
1167 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1168 {
1169         wb >>= 3;
1170         if (wb < adev->wb.num_wb)
1171                 __clear_bit(wb, adev->wb.used);
1172 }
1173
1174 /**
1175  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1176  *
1177  * @adev: amdgpu_device pointer
1178  *
1179  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1180  * to fail, but if any of the BARs is not accessible after the size we abort
1181  * driver loading by returning -ENODEV.
1182  */
1183 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1184 {
1185         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1186         struct pci_bus *root;
1187         struct resource *res;
1188         unsigned i;
1189         u16 cmd;
1190         int r;
1191
1192         /* Bypass for VF */
1193         if (amdgpu_sriov_vf(adev))
1194                 return 0;
1195
1196         /* skip if the bios has already enabled large BAR */
1197         if (adev->gmc.real_vram_size &&
1198             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1199                 return 0;
1200
1201         /* Check if the root BUS has 64bit memory resources */
1202         root = adev->pdev->bus;
1203         while (root->parent)
1204                 root = root->parent;
1205
1206         pci_bus_for_each_resource(root, res, i) {
1207                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1208                     res->start > 0x100000000ull)
1209                         break;
1210         }
1211
1212         /* Trying to resize is pointless without a root hub window above 4GB */
1213         if (!res)
1214                 return 0;
1215
1216         /* Limit the BAR size to what is available */
1217         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1218                         rbar_size);
1219
1220         /* Disable memory decoding while we change the BAR addresses and size */
1221         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1222         pci_write_config_word(adev->pdev, PCI_COMMAND,
1223                               cmd & ~PCI_COMMAND_MEMORY);
1224
1225         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1226         amdgpu_device_doorbell_fini(adev);
1227         if (adev->asic_type >= CHIP_BONAIRE)
1228                 pci_release_resource(adev->pdev, 2);
1229
1230         pci_release_resource(adev->pdev, 0);
1231
1232         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1233         if (r == -ENOSPC)
1234                 DRM_INFO("Not enough PCI address space for a large BAR.");
1235         else if (r && r != -ENOTSUPP)
1236                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1237
1238         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1239
1240         /* When the doorbell or fb BAR isn't available we have no chance of
1241          * using the device.
1242          */
1243         r = amdgpu_device_doorbell_init(adev);
1244         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1245                 return -ENODEV;
1246
1247         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1248
1249         return 0;
1250 }
1251
1252 /*
1253  * GPU helpers function.
1254  */
1255 /**
1256  * amdgpu_device_need_post - check if the hw need post or not
1257  *
1258  * @adev: amdgpu_device pointer
1259  *
1260  * Check if the asic has been initialized (all asics) at driver startup
1261  * or post is needed if  hw reset is performed.
1262  * Returns true if need or false if not.
1263  */
1264 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1265 {
1266         uint32_t reg;
1267
1268         if (amdgpu_sriov_vf(adev))
1269                 return false;
1270
1271         if (amdgpu_passthrough(adev)) {
1272                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1273                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1274                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1275                  * vpost executed for smc version below 22.15
1276                  */
1277                 if (adev->asic_type == CHIP_FIJI) {
1278                         int err;
1279                         uint32_t fw_ver;
1280                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1281                         /* force vPost if error occured */
1282                         if (err)
1283                                 return true;
1284
1285                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1286                         if (fw_ver < 0x00160e00)
1287                                 return true;
1288                 }
1289         }
1290
1291         /* Don't post if we need to reset whole hive on init */
1292         if (adev->gmc.xgmi.pending_reset)
1293                 return false;
1294
1295         if (adev->has_hw_reset) {
1296                 adev->has_hw_reset = false;
1297                 return true;
1298         }
1299
1300         /* bios scratch used on CIK+ */
1301         if (adev->asic_type >= CHIP_BONAIRE)
1302                 return amdgpu_atombios_scratch_need_asic_init(adev);
1303
1304         /* check MEM_SIZE for older asics */
1305         reg = amdgpu_asic_get_config_memsize(adev);
1306
1307         if ((reg != 0) && (reg != 0xffffffff))
1308                 return false;
1309
1310         return true;
1311 }
1312
1313 /* if we get transitioned to only one device, take VGA back */
1314 /**
1315  * amdgpu_device_vga_set_decode - enable/disable vga decode
1316  *
1317  * @pdev: PCI device pointer
1318  * @state: enable/disable vga decode
1319  *
1320  * Enable/disable vga decode (all asics).
1321  * Returns VGA resource flags.
1322  */
1323 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1324                 bool state)
1325 {
1326         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1327         amdgpu_asic_set_vga_state(adev, state);
1328         if (state)
1329                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1330                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1331         else
1332                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1333 }
1334
1335 /**
1336  * amdgpu_device_check_block_size - validate the vm block size
1337  *
1338  * @adev: amdgpu_device pointer
1339  *
1340  * Validates the vm block size specified via module parameter.
1341  * The vm block size defines number of bits in page table versus page directory,
1342  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1343  * page table and the remaining bits are in the page directory.
1344  */
1345 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1346 {
1347         /* defines number of bits in page table versus page directory,
1348          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1349          * page table and the remaining bits are in the page directory */
1350         if (amdgpu_vm_block_size == -1)
1351                 return;
1352
1353         if (amdgpu_vm_block_size < 9) {
1354                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1355                          amdgpu_vm_block_size);
1356                 amdgpu_vm_block_size = -1;
1357         }
1358 }
1359
1360 /**
1361  * amdgpu_device_check_vm_size - validate the vm size
1362  *
1363  * @adev: amdgpu_device pointer
1364  *
1365  * Validates the vm size in GB specified via module parameter.
1366  * The VM size is the size of the GPU virtual memory space in GB.
1367  */
1368 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1369 {
1370         /* no need to check the default value */
1371         if (amdgpu_vm_size == -1)
1372                 return;
1373
1374         if (amdgpu_vm_size < 1) {
1375                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1376                          amdgpu_vm_size);
1377                 amdgpu_vm_size = -1;
1378         }
1379 }
1380
1381 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1382 {
1383         struct sysinfo si;
1384         bool is_os_64 = (sizeof(void *) == 8);
1385         uint64_t total_memory;
1386         uint64_t dram_size_seven_GB = 0x1B8000000;
1387         uint64_t dram_size_three_GB = 0xB8000000;
1388
1389         if (amdgpu_smu_memory_pool_size == 0)
1390                 return;
1391
1392         if (!is_os_64) {
1393                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1394                 goto def_value;
1395         }
1396         si_meminfo(&si);
1397         total_memory = (uint64_t)si.totalram * si.mem_unit;
1398
1399         if ((amdgpu_smu_memory_pool_size == 1) ||
1400                 (amdgpu_smu_memory_pool_size == 2)) {
1401                 if (total_memory < dram_size_three_GB)
1402                         goto def_value1;
1403         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1404                 (amdgpu_smu_memory_pool_size == 8)) {
1405                 if (total_memory < dram_size_seven_GB)
1406                         goto def_value1;
1407         } else {
1408                 DRM_WARN("Smu memory pool size not supported\n");
1409                 goto def_value;
1410         }
1411         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1412
1413         return;
1414
1415 def_value1:
1416         DRM_WARN("No enough system memory\n");
1417 def_value:
1418         adev->pm.smu_prv_buffer_size = 0;
1419 }
1420
1421 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1422 {
1423         if (!(adev->flags & AMD_IS_APU) ||
1424             adev->asic_type < CHIP_RAVEN)
1425                 return 0;
1426
1427         switch (adev->asic_type) {
1428         case CHIP_RAVEN:
1429                 if (adev->pdev->device == 0x15dd)
1430                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1431                 if (adev->pdev->device == 0x15d8)
1432                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1433                 break;
1434         case CHIP_RENOIR:
1435                 if ((adev->pdev->device == 0x1636) ||
1436                     (adev->pdev->device == 0x164c))
1437                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1438                 else
1439                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1440                 break;
1441         case CHIP_VANGOGH:
1442                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1443                 break;
1444         case CHIP_YELLOW_CARP:
1445                 break;
1446         case CHIP_CYAN_SKILLFISH:
1447                 if (adev->pdev->device == 0x13FE)
1448                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1449                 break;
1450         default:
1451                 return -EINVAL;
1452         }
1453
1454         return 0;
1455 }
1456
1457 /**
1458  * amdgpu_device_check_arguments - validate module params
1459  *
1460  * @adev: amdgpu_device pointer
1461  *
1462  * Validates certain module parameters and updates
1463  * the associated values used by the driver (all asics).
1464  */
1465 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1466 {
1467         if (amdgpu_sched_jobs < 4) {
1468                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1469                          amdgpu_sched_jobs);
1470                 amdgpu_sched_jobs = 4;
1471         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1472                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1473                          amdgpu_sched_jobs);
1474                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1475         }
1476
1477         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1478                 /* gart size must be greater or equal to 32M */
1479                 dev_warn(adev->dev, "gart size (%d) too small\n",
1480                          amdgpu_gart_size);
1481                 amdgpu_gart_size = -1;
1482         }
1483
1484         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1485                 /* gtt size must be greater or equal to 32M */
1486                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1487                                  amdgpu_gtt_size);
1488                 amdgpu_gtt_size = -1;
1489         }
1490
1491         /* valid range is between 4 and 9 inclusive */
1492         if (amdgpu_vm_fragment_size != -1 &&
1493             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1494                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1495                 amdgpu_vm_fragment_size = -1;
1496         }
1497
1498         if (amdgpu_sched_hw_submission < 2) {
1499                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1500                          amdgpu_sched_hw_submission);
1501                 amdgpu_sched_hw_submission = 2;
1502         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1503                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1504                          amdgpu_sched_hw_submission);
1505                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1506         }
1507
1508         amdgpu_device_check_smu_prv_buffer_size(adev);
1509
1510         amdgpu_device_check_vm_size(adev);
1511
1512         amdgpu_device_check_block_size(adev);
1513
1514         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1515
1516         amdgpu_gmc_tmz_set(adev);
1517
1518         amdgpu_gmc_noretry_set(adev);
1519
1520         return 0;
1521 }
1522
1523 /**
1524  * amdgpu_switcheroo_set_state - set switcheroo state
1525  *
1526  * @pdev: pci dev pointer
1527  * @state: vga_switcheroo state
1528  *
1529  * Callback for the switcheroo driver.  Suspends or resumes the
1530  * the asics before or after it is powered up using ACPI methods.
1531  */
1532 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1533                                         enum vga_switcheroo_state state)
1534 {
1535         struct drm_device *dev = pci_get_drvdata(pdev);
1536         int r;
1537
1538         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1539                 return;
1540
1541         if (state == VGA_SWITCHEROO_ON) {
1542                 pr_info("switched on\n");
1543                 /* don't suspend or resume card normally */
1544                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1545
1546                 pci_set_power_state(pdev, PCI_D0);
1547                 amdgpu_device_load_pci_state(pdev);
1548                 r = pci_enable_device(pdev);
1549                 if (r)
1550                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1551                 amdgpu_device_resume(dev, true);
1552
1553                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1554         } else {
1555                 pr_info("switched off\n");
1556                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1557                 amdgpu_device_suspend(dev, true);
1558                 amdgpu_device_cache_pci_state(pdev);
1559                 /* Shut down the device */
1560                 pci_disable_device(pdev);
1561                 pci_set_power_state(pdev, PCI_D3cold);
1562                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1563         }
1564 }
1565
1566 /**
1567  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1568  *
1569  * @pdev: pci dev pointer
1570  *
1571  * Callback for the switcheroo driver.  Check of the switcheroo
1572  * state can be changed.
1573  * Returns true if the state can be changed, false if not.
1574  */
1575 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1576 {
1577         struct drm_device *dev = pci_get_drvdata(pdev);
1578
1579         /*
1580         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1581         * locking inversion with the driver load path. And the access here is
1582         * completely racy anyway. So don't bother with locking for now.
1583         */
1584         return atomic_read(&dev->open_count) == 0;
1585 }
1586
1587 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1588         .set_gpu_state = amdgpu_switcheroo_set_state,
1589         .reprobe = NULL,
1590         .can_switch = amdgpu_switcheroo_can_switch,
1591 };
1592
1593 /**
1594  * amdgpu_device_ip_set_clockgating_state - set the CG state
1595  *
1596  * @dev: amdgpu_device pointer
1597  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1598  * @state: clockgating state (gate or ungate)
1599  *
1600  * Sets the requested clockgating state for all instances of
1601  * the hardware IP specified.
1602  * Returns the error code from the last instance.
1603  */
1604 int amdgpu_device_ip_set_clockgating_state(void *dev,
1605                                            enum amd_ip_block_type block_type,
1606                                            enum amd_clockgating_state state)
1607 {
1608         struct amdgpu_device *adev = dev;
1609         int i, r = 0;
1610
1611         for (i = 0; i < adev->num_ip_blocks; i++) {
1612                 if (!adev->ip_blocks[i].status.valid)
1613                         continue;
1614                 if (adev->ip_blocks[i].version->type != block_type)
1615                         continue;
1616                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1617                         continue;
1618                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1619                         (void *)adev, state);
1620                 if (r)
1621                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1622                                   adev->ip_blocks[i].version->funcs->name, r);
1623         }
1624         return r;
1625 }
1626
1627 /**
1628  * amdgpu_device_ip_set_powergating_state - set the PG state
1629  *
1630  * @dev: amdgpu_device pointer
1631  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1632  * @state: powergating state (gate or ungate)
1633  *
1634  * Sets the requested powergating state for all instances of
1635  * the hardware IP specified.
1636  * Returns the error code from the last instance.
1637  */
1638 int amdgpu_device_ip_set_powergating_state(void *dev,
1639                                            enum amd_ip_block_type block_type,
1640                                            enum amd_powergating_state state)
1641 {
1642         struct amdgpu_device *adev = dev;
1643         int i, r = 0;
1644
1645         for (i = 0; i < adev->num_ip_blocks; i++) {
1646                 if (!adev->ip_blocks[i].status.valid)
1647                         continue;
1648                 if (adev->ip_blocks[i].version->type != block_type)
1649                         continue;
1650                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1651                         continue;
1652                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1653                         (void *)adev, state);
1654                 if (r)
1655                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1656                                   adev->ip_blocks[i].version->funcs->name, r);
1657         }
1658         return r;
1659 }
1660
1661 /**
1662  * amdgpu_device_ip_get_clockgating_state - get the CG state
1663  *
1664  * @adev: amdgpu_device pointer
1665  * @flags: clockgating feature flags
1666  *
1667  * Walks the list of IPs on the device and updates the clockgating
1668  * flags for each IP.
1669  * Updates @flags with the feature flags for each hardware IP where
1670  * clockgating is enabled.
1671  */
1672 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1673                                             u32 *flags)
1674 {
1675         int i;
1676
1677         for (i = 0; i < adev->num_ip_blocks; i++) {
1678                 if (!adev->ip_blocks[i].status.valid)
1679                         continue;
1680                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1681                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1682         }
1683 }
1684
1685 /**
1686  * amdgpu_device_ip_wait_for_idle - wait for idle
1687  *
1688  * @adev: amdgpu_device pointer
1689  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1690  *
1691  * Waits for the request hardware IP to be idle.
1692  * Returns 0 for success or a negative error code on failure.
1693  */
1694 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1695                                    enum amd_ip_block_type block_type)
1696 {
1697         int i, r;
1698
1699         for (i = 0; i < adev->num_ip_blocks; i++) {
1700                 if (!adev->ip_blocks[i].status.valid)
1701                         continue;
1702                 if (adev->ip_blocks[i].version->type == block_type) {
1703                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1704                         if (r)
1705                                 return r;
1706                         break;
1707                 }
1708         }
1709         return 0;
1710
1711 }
1712
1713 /**
1714  * amdgpu_device_ip_is_idle - is the hardware IP idle
1715  *
1716  * @adev: amdgpu_device pointer
1717  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1718  *
1719  * Check if the hardware IP is idle or not.
1720  * Returns true if it the IP is idle, false if not.
1721  */
1722 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1723                               enum amd_ip_block_type block_type)
1724 {
1725         int i;
1726
1727         for (i = 0; i < adev->num_ip_blocks; i++) {
1728                 if (!adev->ip_blocks[i].status.valid)
1729                         continue;
1730                 if (adev->ip_blocks[i].version->type == block_type)
1731                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1732         }
1733         return true;
1734
1735 }
1736
1737 /**
1738  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1739  *
1740  * @adev: amdgpu_device pointer
1741  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1742  *
1743  * Returns a pointer to the hardware IP block structure
1744  * if it exists for the asic, otherwise NULL.
1745  */
1746 struct amdgpu_ip_block *
1747 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1748                               enum amd_ip_block_type type)
1749 {
1750         int i;
1751
1752         for (i = 0; i < adev->num_ip_blocks; i++)
1753                 if (adev->ip_blocks[i].version->type == type)
1754                         return &adev->ip_blocks[i];
1755
1756         return NULL;
1757 }
1758
1759 /**
1760  * amdgpu_device_ip_block_version_cmp
1761  *
1762  * @adev: amdgpu_device pointer
1763  * @type: enum amd_ip_block_type
1764  * @major: major version
1765  * @minor: minor version
1766  *
1767  * return 0 if equal or greater
1768  * return 1 if smaller or the ip_block doesn't exist
1769  */
1770 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1771                                        enum amd_ip_block_type type,
1772                                        u32 major, u32 minor)
1773 {
1774         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1775
1776         if (ip_block && ((ip_block->version->major > major) ||
1777                         ((ip_block->version->major == major) &&
1778                         (ip_block->version->minor >= minor))))
1779                 return 0;
1780
1781         return 1;
1782 }
1783
1784 /**
1785  * amdgpu_device_ip_block_add
1786  *
1787  * @adev: amdgpu_device pointer
1788  * @ip_block_version: pointer to the IP to add
1789  *
1790  * Adds the IP block driver information to the collection of IPs
1791  * on the asic.
1792  */
1793 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1794                                const struct amdgpu_ip_block_version *ip_block_version)
1795 {
1796         if (!ip_block_version)
1797                 return -EINVAL;
1798
1799         switch (ip_block_version->type) {
1800         case AMD_IP_BLOCK_TYPE_VCN:
1801                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1802                         return 0;
1803                 break;
1804         case AMD_IP_BLOCK_TYPE_JPEG:
1805                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1806                         return 0;
1807                 break;
1808         default:
1809                 break;
1810         }
1811
1812         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1813                   ip_block_version->funcs->name);
1814
1815         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1816
1817         return 0;
1818 }
1819
1820 /**
1821  * amdgpu_device_enable_virtual_display - enable virtual display feature
1822  *
1823  * @adev: amdgpu_device pointer
1824  *
1825  * Enabled the virtual display feature if the user has enabled it via
1826  * the module parameter virtual_display.  This feature provides a virtual
1827  * display hardware on headless boards or in virtualized environments.
1828  * This function parses and validates the configuration string specified by
1829  * the user and configues the virtual display configuration (number of
1830  * virtual connectors, crtcs, etc.) specified.
1831  */
1832 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1833 {
1834         adev->enable_virtual_display = false;
1835
1836         if (amdgpu_virtual_display) {
1837                 const char *pci_address_name = pci_name(adev->pdev);
1838                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1839
1840                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1841                 pciaddstr_tmp = pciaddstr;
1842                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1843                         pciaddname = strsep(&pciaddname_tmp, ",");
1844                         if (!strcmp("all", pciaddname)
1845                             || !strcmp(pci_address_name, pciaddname)) {
1846                                 long num_crtc;
1847                                 int res = -1;
1848
1849                                 adev->enable_virtual_display = true;
1850
1851                                 if (pciaddname_tmp)
1852                                         res = kstrtol(pciaddname_tmp, 10,
1853                                                       &num_crtc);
1854
1855                                 if (!res) {
1856                                         if (num_crtc < 1)
1857                                                 num_crtc = 1;
1858                                         if (num_crtc > 6)
1859                                                 num_crtc = 6;
1860                                         adev->mode_info.num_crtc = num_crtc;
1861                                 } else {
1862                                         adev->mode_info.num_crtc = 1;
1863                                 }
1864                                 break;
1865                         }
1866                 }
1867
1868                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1869                          amdgpu_virtual_display, pci_address_name,
1870                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1871
1872                 kfree(pciaddstr);
1873         }
1874 }
1875
1876 /**
1877  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1878  *
1879  * @adev: amdgpu_device pointer
1880  *
1881  * Parses the asic configuration parameters specified in the gpu info
1882  * firmware and makes them availale to the driver for use in configuring
1883  * the asic.
1884  * Returns 0 on success, -EINVAL on failure.
1885  */
1886 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1887 {
1888         const char *chip_name;
1889         char fw_name[40];
1890         int err;
1891         const struct gpu_info_firmware_header_v1_0 *hdr;
1892
1893         adev->firmware.gpu_info_fw = NULL;
1894
1895         if (adev->mman.discovery_bin) {
1896                 amdgpu_discovery_get_gfx_info(adev);
1897
1898                 /*
1899                  * FIXME: The bounding box is still needed by Navi12, so
1900                  * temporarily read it from gpu_info firmware. Should be droped
1901                  * when DAL no longer needs it.
1902                  */
1903                 if (adev->asic_type != CHIP_NAVI12)
1904                         return 0;
1905         }
1906
1907         switch (adev->asic_type) {
1908 #ifdef CONFIG_DRM_AMDGPU_SI
1909         case CHIP_VERDE:
1910         case CHIP_TAHITI:
1911         case CHIP_PITCAIRN:
1912         case CHIP_OLAND:
1913         case CHIP_HAINAN:
1914 #endif
1915 #ifdef CONFIG_DRM_AMDGPU_CIK
1916         case CHIP_BONAIRE:
1917         case CHIP_HAWAII:
1918         case CHIP_KAVERI:
1919         case CHIP_KABINI:
1920         case CHIP_MULLINS:
1921 #endif
1922         case CHIP_TOPAZ:
1923         case CHIP_TONGA:
1924         case CHIP_FIJI:
1925         case CHIP_POLARIS10:
1926         case CHIP_POLARIS11:
1927         case CHIP_POLARIS12:
1928         case CHIP_VEGAM:
1929         case CHIP_CARRIZO:
1930         case CHIP_STONEY:
1931         case CHIP_VEGA20:
1932         case CHIP_ALDEBARAN:
1933         case CHIP_SIENNA_CICHLID:
1934         case CHIP_NAVY_FLOUNDER:
1935         case CHIP_DIMGREY_CAVEFISH:
1936         case CHIP_BEIGE_GOBY:
1937         default:
1938                 return 0;
1939         case CHIP_VEGA10:
1940                 chip_name = "vega10";
1941                 break;
1942         case CHIP_VEGA12:
1943                 chip_name = "vega12";
1944                 break;
1945         case CHIP_RAVEN:
1946                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1947                         chip_name = "raven2";
1948                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1949                         chip_name = "picasso";
1950                 else
1951                         chip_name = "raven";
1952                 break;
1953         case CHIP_ARCTURUS:
1954                 chip_name = "arcturus";
1955                 break;
1956         case CHIP_RENOIR:
1957                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1958                         chip_name = "renoir";
1959                 else
1960                         chip_name = "green_sardine";
1961                 break;
1962         case CHIP_NAVI10:
1963                 chip_name = "navi10";
1964                 break;
1965         case CHIP_NAVI14:
1966                 chip_name = "navi14";
1967                 break;
1968         case CHIP_NAVI12:
1969                 chip_name = "navi12";
1970                 break;
1971         case CHIP_VANGOGH:
1972                 chip_name = "vangogh";
1973                 break;
1974         case CHIP_YELLOW_CARP:
1975                 chip_name = "yellow_carp";
1976                 break;
1977         }
1978
1979         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1980         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1981         if (err) {
1982                 dev_err(adev->dev,
1983                         "Failed to load gpu_info firmware \"%s\"\n",
1984                         fw_name);
1985                 goto out;
1986         }
1987         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1988         if (err) {
1989                 dev_err(adev->dev,
1990                         "Failed to validate gpu_info firmware \"%s\"\n",
1991                         fw_name);
1992                 goto out;
1993         }
1994
1995         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1996         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1997
1998         switch (hdr->version_major) {
1999         case 1:
2000         {
2001                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2002                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2003                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2004
2005                 /*
2006                  * Should be droped when DAL no longer needs it.
2007                  */
2008                 if (adev->asic_type == CHIP_NAVI12)
2009                         goto parse_soc_bounding_box;
2010
2011                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2012                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2013                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2014                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2015                 adev->gfx.config.max_texture_channel_caches =
2016                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2017                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2018                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2019                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2020                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2021                 adev->gfx.config.double_offchip_lds_buf =
2022                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2023                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2024                 adev->gfx.cu_info.max_waves_per_simd =
2025                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2026                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2027                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2028                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2029                 if (hdr->version_minor >= 1) {
2030                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2031                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2032                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2033                         adev->gfx.config.num_sc_per_sh =
2034                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2035                         adev->gfx.config.num_packer_per_sc =
2036                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2037                 }
2038
2039 parse_soc_bounding_box:
2040                 /*
2041                  * soc bounding box info is not integrated in disocovery table,
2042                  * we always need to parse it from gpu info firmware if needed.
2043                  */
2044                 if (hdr->version_minor == 2) {
2045                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2046                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2047                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2048                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2049                 }
2050                 break;
2051         }
2052         default:
2053                 dev_err(adev->dev,
2054                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2055                 err = -EINVAL;
2056                 goto out;
2057         }
2058 out:
2059         return err;
2060 }
2061
2062 /**
2063  * amdgpu_device_ip_early_init - run early init for hardware IPs
2064  *
2065  * @adev: amdgpu_device pointer
2066  *
2067  * Early initialization pass for hardware IPs.  The hardware IPs that make
2068  * up each asic are discovered each IP's early_init callback is run.  This
2069  * is the first stage in initializing the asic.
2070  * Returns 0 on success, negative error code on failure.
2071  */
2072 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2073 {
2074         int i, r;
2075
2076         amdgpu_device_enable_virtual_display(adev);
2077
2078         if (amdgpu_sriov_vf(adev)) {
2079                 r = amdgpu_virt_request_full_gpu(adev, true);
2080                 if (r)
2081                         return r;
2082         }
2083
2084         switch (adev->asic_type) {
2085 #ifdef CONFIG_DRM_AMDGPU_SI
2086         case CHIP_VERDE:
2087         case CHIP_TAHITI:
2088         case CHIP_PITCAIRN:
2089         case CHIP_OLAND:
2090         case CHIP_HAINAN:
2091                 adev->family = AMDGPU_FAMILY_SI;
2092                 r = si_set_ip_blocks(adev);
2093                 if (r)
2094                         return r;
2095                 break;
2096 #endif
2097 #ifdef CONFIG_DRM_AMDGPU_CIK
2098         case CHIP_BONAIRE:
2099         case CHIP_HAWAII:
2100         case CHIP_KAVERI:
2101         case CHIP_KABINI:
2102         case CHIP_MULLINS:
2103                 if (adev->flags & AMD_IS_APU)
2104                         adev->family = AMDGPU_FAMILY_KV;
2105                 else
2106                         adev->family = AMDGPU_FAMILY_CI;
2107
2108                 r = cik_set_ip_blocks(adev);
2109                 if (r)
2110                         return r;
2111                 break;
2112 #endif
2113         case CHIP_TOPAZ:
2114         case CHIP_TONGA:
2115         case CHIP_FIJI:
2116         case CHIP_POLARIS10:
2117         case CHIP_POLARIS11:
2118         case CHIP_POLARIS12:
2119         case CHIP_VEGAM:
2120         case CHIP_CARRIZO:
2121         case CHIP_STONEY:
2122                 if (adev->flags & AMD_IS_APU)
2123                         adev->family = AMDGPU_FAMILY_CZ;
2124                 else
2125                         adev->family = AMDGPU_FAMILY_VI;
2126
2127                 r = vi_set_ip_blocks(adev);
2128                 if (r)
2129                         return r;
2130                 break;
2131         default:
2132                 r = amdgpu_discovery_set_ip_blocks(adev);
2133                 if (r)
2134                         return r;
2135                 break;
2136         }
2137
2138         amdgpu_amdkfd_device_probe(adev);
2139
2140         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2141         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2142                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2143         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2144                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2145
2146         for (i = 0; i < adev->num_ip_blocks; i++) {
2147                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2148                         DRM_ERROR("disabled ip block: %d <%s>\n",
2149                                   i, adev->ip_blocks[i].version->funcs->name);
2150                         adev->ip_blocks[i].status.valid = false;
2151                 } else {
2152                         if (adev->ip_blocks[i].version->funcs->early_init) {
2153                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2154                                 if (r == -ENOENT) {
2155                                         adev->ip_blocks[i].status.valid = false;
2156                                 } else if (r) {
2157                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2158                                                   adev->ip_blocks[i].version->funcs->name, r);
2159                                         return r;
2160                                 } else {
2161                                         adev->ip_blocks[i].status.valid = true;
2162                                 }
2163                         } else {
2164                                 adev->ip_blocks[i].status.valid = true;
2165                         }
2166                 }
2167                 /* get the vbios after the asic_funcs are set up */
2168                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2169                         r = amdgpu_device_parse_gpu_info_fw(adev);
2170                         if (r)
2171                                 return r;
2172
2173                         /* Read BIOS */
2174                         if (!amdgpu_get_bios(adev))
2175                                 return -EINVAL;
2176
2177                         r = amdgpu_atombios_init(adev);
2178                         if (r) {
2179                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2180                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2181                                 return r;
2182                         }
2183
2184                         /*get pf2vf msg info at it's earliest time*/
2185                         if (amdgpu_sriov_vf(adev))
2186                                 amdgpu_virt_init_data_exchange(adev);
2187
2188                 }
2189         }
2190
2191         adev->cg_flags &= amdgpu_cg_mask;
2192         adev->pg_flags &= amdgpu_pg_mask;
2193
2194         return 0;
2195 }
2196
2197 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2198 {
2199         int i, r;
2200
2201         for (i = 0; i < adev->num_ip_blocks; i++) {
2202                 if (!adev->ip_blocks[i].status.sw)
2203                         continue;
2204                 if (adev->ip_blocks[i].status.hw)
2205                         continue;
2206                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2207                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2208                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2209                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2210                         if (r) {
2211                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2212                                           adev->ip_blocks[i].version->funcs->name, r);
2213                                 return r;
2214                         }
2215                         adev->ip_blocks[i].status.hw = true;
2216                 }
2217         }
2218
2219         return 0;
2220 }
2221
2222 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2223 {
2224         int i, r;
2225
2226         for (i = 0; i < adev->num_ip_blocks; i++) {
2227                 if (!adev->ip_blocks[i].status.sw)
2228                         continue;
2229                 if (adev->ip_blocks[i].status.hw)
2230                         continue;
2231                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2232                 if (r) {
2233                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2234                                   adev->ip_blocks[i].version->funcs->name, r);
2235                         return r;
2236                 }
2237                 adev->ip_blocks[i].status.hw = true;
2238         }
2239
2240         return 0;
2241 }
2242
2243 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2244 {
2245         int r = 0;
2246         int i;
2247         uint32_t smu_version;
2248
2249         if (adev->asic_type >= CHIP_VEGA10) {
2250                 for (i = 0; i < adev->num_ip_blocks; i++) {
2251                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2252                                 continue;
2253
2254                         if (!adev->ip_blocks[i].status.sw)
2255                                 continue;
2256
2257                         /* no need to do the fw loading again if already done*/
2258                         if (adev->ip_blocks[i].status.hw == true)
2259                                 break;
2260
2261                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2262                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2263                                 if (r) {
2264                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2265                                                           adev->ip_blocks[i].version->funcs->name, r);
2266                                         return r;
2267                                 }
2268                         } else {
2269                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2270                                 if (r) {
2271                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2272                                                           adev->ip_blocks[i].version->funcs->name, r);
2273                                         return r;
2274                                 }
2275                         }
2276
2277                         adev->ip_blocks[i].status.hw = true;
2278                         break;
2279                 }
2280         }
2281
2282         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2283                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2284
2285         return r;
2286 }
2287
2288 /**
2289  * amdgpu_device_ip_init - run init for hardware IPs
2290  *
2291  * @adev: amdgpu_device pointer
2292  *
2293  * Main initialization pass for hardware IPs.  The list of all the hardware
2294  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2295  * are run.  sw_init initializes the software state associated with each IP
2296  * and hw_init initializes the hardware associated with each IP.
2297  * Returns 0 on success, negative error code on failure.
2298  */
2299 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2300 {
2301         int i, r;
2302
2303         r = amdgpu_ras_init(adev);
2304         if (r)
2305                 return r;
2306
2307         for (i = 0; i < adev->num_ip_blocks; i++) {
2308                 if (!adev->ip_blocks[i].status.valid)
2309                         continue;
2310                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2311                 if (r) {
2312                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2313                                   adev->ip_blocks[i].version->funcs->name, r);
2314                         goto init_failed;
2315                 }
2316                 adev->ip_blocks[i].status.sw = true;
2317
2318                 /* need to do gmc hw init early so we can allocate gpu mem */
2319                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2320                         r = amdgpu_device_vram_scratch_init(adev);
2321                         if (r) {
2322                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2323                                 goto init_failed;
2324                         }
2325                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2326                         if (r) {
2327                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2328                                 goto init_failed;
2329                         }
2330                         r = amdgpu_device_wb_init(adev);
2331                         if (r) {
2332                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2333                                 goto init_failed;
2334                         }
2335                         adev->ip_blocks[i].status.hw = true;
2336
2337                         /* right after GMC hw init, we create CSA */
2338                         if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2339                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2340                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2341                                                                 AMDGPU_CSA_SIZE);
2342                                 if (r) {
2343                                         DRM_ERROR("allocate CSA failed %d\n", r);
2344                                         goto init_failed;
2345                                 }
2346                         }
2347                 }
2348         }
2349
2350         if (amdgpu_sriov_vf(adev))
2351                 amdgpu_virt_init_data_exchange(adev);
2352
2353         r = amdgpu_ib_pool_init(adev);
2354         if (r) {
2355                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2356                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2357                 goto init_failed;
2358         }
2359
2360         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2361         if (r)
2362                 goto init_failed;
2363
2364         r = amdgpu_device_ip_hw_init_phase1(adev);
2365         if (r)
2366                 goto init_failed;
2367
2368         r = amdgpu_device_fw_loading(adev);
2369         if (r)
2370                 goto init_failed;
2371
2372         r = amdgpu_device_ip_hw_init_phase2(adev);
2373         if (r)
2374                 goto init_failed;
2375
2376         /*
2377          * retired pages will be loaded from eeprom and reserved here,
2378          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2379          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2380          * for I2C communication which only true at this point.
2381          *
2382          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2383          * failure from bad gpu situation and stop amdgpu init process
2384          * accordingly. For other failed cases, it will still release all
2385          * the resource and print error message, rather than returning one
2386          * negative value to upper level.
2387          *
2388          * Note: theoretically, this should be called before all vram allocations
2389          * to protect retired page from abusing
2390          */
2391         r = amdgpu_ras_recovery_init(adev);
2392         if (r)
2393                 goto init_failed;
2394
2395         if (adev->gmc.xgmi.num_physical_nodes > 1)
2396                 amdgpu_xgmi_add_device(adev);
2397
2398         /* Don't init kfd if whole hive need to be reset during init */
2399         if (!adev->gmc.xgmi.pending_reset)
2400                 amdgpu_amdkfd_device_init(adev);
2401
2402         amdgpu_fru_get_product_info(adev);
2403
2404 init_failed:
2405         if (amdgpu_sriov_vf(adev))
2406                 amdgpu_virt_release_full_gpu(adev, true);
2407
2408         return r;
2409 }
2410
2411 /**
2412  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2413  *
2414  * @adev: amdgpu_device pointer
2415  *
2416  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2417  * this function before a GPU reset.  If the value is retained after a
2418  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2419  */
2420 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2421 {
2422         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2423 }
2424
2425 /**
2426  * amdgpu_device_check_vram_lost - check if vram is valid
2427  *
2428  * @adev: amdgpu_device pointer
2429  *
2430  * Checks the reset magic value written to the gart pointer in VRAM.
2431  * The driver calls this after a GPU reset to see if the contents of
2432  * VRAM is lost or now.
2433  * returns true if vram is lost, false if not.
2434  */
2435 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2436 {
2437         if (memcmp(adev->gart.ptr, adev->reset_magic,
2438                         AMDGPU_RESET_MAGIC_NUM))
2439                 return true;
2440
2441         if (!amdgpu_in_reset(adev))
2442                 return false;
2443
2444         /*
2445          * For all ASICs with baco/mode1 reset, the VRAM is
2446          * always assumed to be lost.
2447          */
2448         switch (amdgpu_asic_reset_method(adev)) {
2449         case AMD_RESET_METHOD_BACO:
2450         case AMD_RESET_METHOD_MODE1:
2451                 return true;
2452         default:
2453                 return false;
2454         }
2455 }
2456
2457 /**
2458  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2459  *
2460  * @adev: amdgpu_device pointer
2461  * @state: clockgating state (gate or ungate)
2462  *
2463  * The list of all the hardware IPs that make up the asic is walked and the
2464  * set_clockgating_state callbacks are run.
2465  * Late initialization pass enabling clockgating for hardware IPs.
2466  * Fini or suspend, pass disabling clockgating for hardware IPs.
2467  * Returns 0 on success, negative error code on failure.
2468  */
2469
2470 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2471                                enum amd_clockgating_state state)
2472 {
2473         int i, j, r;
2474
2475         if (amdgpu_emu_mode == 1)
2476                 return 0;
2477
2478         for (j = 0; j < adev->num_ip_blocks; j++) {
2479                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2480                 if (!adev->ip_blocks[i].status.late_initialized)
2481                         continue;
2482                 /* skip CG for GFX on S0ix */
2483                 if (adev->in_s0ix &&
2484                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2485                         continue;
2486                 /* skip CG for VCE/UVD, it's handled specially */
2487                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2488                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2489                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2490                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2491                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2492                         /* enable clockgating to save power */
2493                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2494                                                                                      state);
2495                         if (r) {
2496                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2497                                           adev->ip_blocks[i].version->funcs->name, r);
2498                                 return r;
2499                         }
2500                 }
2501         }
2502
2503         return 0;
2504 }
2505
2506 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2507                                enum amd_powergating_state state)
2508 {
2509         int i, j, r;
2510
2511         if (amdgpu_emu_mode == 1)
2512                 return 0;
2513
2514         for (j = 0; j < adev->num_ip_blocks; j++) {
2515                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2516                 if (!adev->ip_blocks[i].status.late_initialized)
2517                         continue;
2518                 /* skip PG for GFX on S0ix */
2519                 if (adev->in_s0ix &&
2520                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2521                         continue;
2522                 /* skip CG for VCE/UVD, it's handled specially */
2523                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2524                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2525                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2526                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2527                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2528                         /* enable powergating to save power */
2529                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2530                                                                                         state);
2531                         if (r) {
2532                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2533                                           adev->ip_blocks[i].version->funcs->name, r);
2534                                 return r;
2535                         }
2536                 }
2537         }
2538         return 0;
2539 }
2540
2541 static int amdgpu_device_enable_mgpu_fan_boost(void)
2542 {
2543         struct amdgpu_gpu_instance *gpu_ins;
2544         struct amdgpu_device *adev;
2545         int i, ret = 0;
2546
2547         mutex_lock(&mgpu_info.mutex);
2548
2549         /*
2550          * MGPU fan boost feature should be enabled
2551          * only when there are two or more dGPUs in
2552          * the system
2553          */
2554         if (mgpu_info.num_dgpu < 2)
2555                 goto out;
2556
2557         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2558                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2559                 adev = gpu_ins->adev;
2560                 if (!(adev->flags & AMD_IS_APU) &&
2561                     !gpu_ins->mgpu_fan_enabled) {
2562                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2563                         if (ret)
2564                                 break;
2565
2566                         gpu_ins->mgpu_fan_enabled = 1;
2567                 }
2568         }
2569
2570 out:
2571         mutex_unlock(&mgpu_info.mutex);
2572
2573         return ret;
2574 }
2575
2576 /**
2577  * amdgpu_device_ip_late_init - run late init for hardware IPs
2578  *
2579  * @adev: amdgpu_device pointer
2580  *
2581  * Late initialization pass for hardware IPs.  The list of all the hardware
2582  * IPs that make up the asic is walked and the late_init callbacks are run.
2583  * late_init covers any special initialization that an IP requires
2584  * after all of the have been initialized or something that needs to happen
2585  * late in the init process.
2586  * Returns 0 on success, negative error code on failure.
2587  */
2588 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2589 {
2590         struct amdgpu_gpu_instance *gpu_instance;
2591         int i = 0, r;
2592
2593         for (i = 0; i < adev->num_ip_blocks; i++) {
2594                 if (!adev->ip_blocks[i].status.hw)
2595                         continue;
2596                 if (adev->ip_blocks[i].version->funcs->late_init) {
2597                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2598                         if (r) {
2599                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2600                                           adev->ip_blocks[i].version->funcs->name, r);
2601                                 return r;
2602                         }
2603                 }
2604                 adev->ip_blocks[i].status.late_initialized = true;
2605         }
2606
2607         amdgpu_ras_set_error_query_ready(adev, true);
2608
2609         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2610         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2611
2612         amdgpu_device_fill_reset_magic(adev);
2613
2614         r = amdgpu_device_enable_mgpu_fan_boost();
2615         if (r)
2616                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2617
2618         /* For XGMI + passthrough configuration on arcturus, enable light SBR */
2619         if (adev->asic_type == CHIP_ARCTURUS &&
2620             amdgpu_passthrough(adev) &&
2621             adev->gmc.xgmi.num_physical_nodes > 1)
2622                 smu_set_light_sbr(&adev->smu, true);
2623
2624         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2625                 mutex_lock(&mgpu_info.mutex);
2626
2627                 /*
2628                  * Reset device p-state to low as this was booted with high.
2629                  *
2630                  * This should be performed only after all devices from the same
2631                  * hive get initialized.
2632                  *
2633                  * However, it's unknown how many device in the hive in advance.
2634                  * As this is counted one by one during devices initializations.
2635                  *
2636                  * So, we wait for all XGMI interlinked devices initialized.
2637                  * This may bring some delays as those devices may come from
2638                  * different hives. But that should be OK.
2639                  */
2640                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2641                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2642                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2643                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2644                                         continue;
2645
2646                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2647                                                 AMDGPU_XGMI_PSTATE_MIN);
2648                                 if (r) {
2649                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2650                                         break;
2651                                 }
2652                         }
2653                 }
2654
2655                 mutex_unlock(&mgpu_info.mutex);
2656         }
2657
2658         return 0;
2659 }
2660
2661 /**
2662  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2663  *
2664  * @adev: amdgpu_device pointer
2665  *
2666  * For ASICs need to disable SMC first
2667  */
2668 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2669 {
2670         int i, r;
2671
2672         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2673                 return;
2674
2675         for (i = 0; i < adev->num_ip_blocks; i++) {
2676                 if (!adev->ip_blocks[i].status.hw)
2677                         continue;
2678                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2679                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2680                         /* XXX handle errors */
2681                         if (r) {
2682                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2683                                           adev->ip_blocks[i].version->funcs->name, r);
2684                         }
2685                         adev->ip_blocks[i].status.hw = false;
2686                         break;
2687                 }
2688         }
2689 }
2690
2691 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2692 {
2693         int i, r;
2694
2695         for (i = 0; i < adev->num_ip_blocks; i++) {
2696                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2697                         continue;
2698
2699                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2700                 if (r) {
2701                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2702                                   adev->ip_blocks[i].version->funcs->name, r);
2703                 }
2704         }
2705
2706         amdgpu_amdkfd_suspend(adev, false);
2707
2708         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2709         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2710
2711         /* Workaroud for ASICs need to disable SMC first */
2712         amdgpu_device_smu_fini_early(adev);
2713
2714         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2715                 if (!adev->ip_blocks[i].status.hw)
2716                         continue;
2717
2718                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2719                 /* XXX handle errors */
2720                 if (r) {
2721                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2722                                   adev->ip_blocks[i].version->funcs->name, r);
2723                 }
2724
2725                 adev->ip_blocks[i].status.hw = false;
2726         }
2727
2728         if (amdgpu_sriov_vf(adev)) {
2729                 if (amdgpu_virt_release_full_gpu(adev, false))
2730                         DRM_ERROR("failed to release exclusive mode on fini\n");
2731         }
2732
2733         return 0;
2734 }
2735
2736 /**
2737  * amdgpu_device_ip_fini - run fini for hardware IPs
2738  *
2739  * @adev: amdgpu_device pointer
2740  *
2741  * Main teardown pass for hardware IPs.  The list of all the hardware
2742  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2743  * are run.  hw_fini tears down the hardware associated with each IP
2744  * and sw_fini tears down any software state associated with each IP.
2745  * Returns 0 on success, negative error code on failure.
2746  */
2747 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2748 {
2749         int i, r;
2750
2751         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2752                 amdgpu_virt_release_ras_err_handler_data(adev);
2753
2754         if (adev->gmc.xgmi.num_physical_nodes > 1)
2755                 amdgpu_xgmi_remove_device(adev);
2756
2757         amdgpu_amdkfd_device_fini_sw(adev);
2758
2759         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2760                 if (!adev->ip_blocks[i].status.sw)
2761                         continue;
2762
2763                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2764                         amdgpu_ucode_free_bo(adev);
2765                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2766                         amdgpu_device_wb_fini(adev);
2767                         amdgpu_device_vram_scratch_fini(adev);
2768                         amdgpu_ib_pool_fini(adev);
2769                 }
2770
2771                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2772                 /* XXX handle errors */
2773                 if (r) {
2774                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2775                                   adev->ip_blocks[i].version->funcs->name, r);
2776                 }
2777                 adev->ip_blocks[i].status.sw = false;
2778                 adev->ip_blocks[i].status.valid = false;
2779         }
2780
2781         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2782                 if (!adev->ip_blocks[i].status.late_initialized)
2783                         continue;
2784                 if (adev->ip_blocks[i].version->funcs->late_fini)
2785                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2786                 adev->ip_blocks[i].status.late_initialized = false;
2787         }
2788
2789         amdgpu_ras_fini(adev);
2790
2791         return 0;
2792 }
2793
2794 /**
2795  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2796  *
2797  * @work: work_struct.
2798  */
2799 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2800 {
2801         struct amdgpu_device *adev =
2802                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2803         int r;
2804
2805         r = amdgpu_ib_ring_tests(adev);
2806         if (r)
2807                 DRM_ERROR("ib ring test failed (%d).\n", r);
2808 }
2809
2810 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2811 {
2812         struct amdgpu_device *adev =
2813                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2814
2815         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2816         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2817
2818         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2819                 adev->gfx.gfx_off_state = true;
2820 }
2821
2822 /**
2823  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2824  *
2825  * @adev: amdgpu_device pointer
2826  *
2827  * Main suspend function for hardware IPs.  The list of all the hardware
2828  * IPs that make up the asic is walked, clockgating is disabled and the
2829  * suspend callbacks are run.  suspend puts the hardware and software state
2830  * in each IP into a state suitable for suspend.
2831  * Returns 0 on success, negative error code on failure.
2832  */
2833 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2834 {
2835         int i, r;
2836
2837         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2838         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2839
2840         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2841                 if (!adev->ip_blocks[i].status.valid)
2842                         continue;
2843
2844                 /* displays are handled separately */
2845                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2846                         continue;
2847
2848                 /* XXX handle errors */
2849                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2850                 /* XXX handle errors */
2851                 if (r) {
2852                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2853                                   adev->ip_blocks[i].version->funcs->name, r);
2854                         return r;
2855                 }
2856
2857                 adev->ip_blocks[i].status.hw = false;
2858         }
2859
2860         return 0;
2861 }
2862
2863 /**
2864  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2865  *
2866  * @adev: amdgpu_device pointer
2867  *
2868  * Main suspend function for hardware IPs.  The list of all the hardware
2869  * IPs that make up the asic is walked, clockgating is disabled and the
2870  * suspend callbacks are run.  suspend puts the hardware and software state
2871  * in each IP into a state suitable for suspend.
2872  * Returns 0 on success, negative error code on failure.
2873  */
2874 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2875 {
2876         int i, r;
2877
2878         if (adev->in_s0ix)
2879                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
2880
2881         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2882                 if (!adev->ip_blocks[i].status.valid)
2883                         continue;
2884                 /* displays are handled in phase1 */
2885                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2886                         continue;
2887                 /* PSP lost connection when err_event_athub occurs */
2888                 if (amdgpu_ras_intr_triggered() &&
2889                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2890                         adev->ip_blocks[i].status.hw = false;
2891                         continue;
2892                 }
2893
2894                 /* skip unnecessary suspend if we do not initialize them yet */
2895                 if (adev->gmc.xgmi.pending_reset &&
2896                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2897                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2898                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2899                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2900                         adev->ip_blocks[i].status.hw = false;
2901                         continue;
2902                 }
2903
2904                 /* skip suspend of gfx and psp for S0ix
2905                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
2906                  * like at runtime. PSP is also part of the always on hardware
2907                  * so no need to suspend it.
2908                  */
2909                 if (adev->in_s0ix &&
2910                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
2911                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
2912                         continue;
2913
2914                 /* XXX handle errors */
2915                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2916                 /* XXX handle errors */
2917                 if (r) {
2918                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2919                                   adev->ip_blocks[i].version->funcs->name, r);
2920                 }
2921                 adev->ip_blocks[i].status.hw = false;
2922                 /* handle putting the SMC in the appropriate state */
2923                 if(!amdgpu_sriov_vf(adev)){
2924                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2925                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2926                                 if (r) {
2927                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2928                                                         adev->mp1_state, r);
2929                                         return r;
2930                                 }
2931                         }
2932                 }
2933         }
2934
2935         return 0;
2936 }
2937
2938 /**
2939  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2940  *
2941  * @adev: amdgpu_device pointer
2942  *
2943  * Main suspend function for hardware IPs.  The list of all the hardware
2944  * IPs that make up the asic is walked, clockgating is disabled and the
2945  * suspend callbacks are run.  suspend puts the hardware and software state
2946  * in each IP into a state suitable for suspend.
2947  * Returns 0 on success, negative error code on failure.
2948  */
2949 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2950 {
2951         int r;
2952
2953         if (amdgpu_sriov_vf(adev)) {
2954                 amdgpu_virt_fini_data_exchange(adev);
2955                 amdgpu_virt_request_full_gpu(adev, false);
2956         }
2957
2958         r = amdgpu_device_ip_suspend_phase1(adev);
2959         if (r)
2960                 return r;
2961         r = amdgpu_device_ip_suspend_phase2(adev);
2962
2963         if (amdgpu_sriov_vf(adev))
2964                 amdgpu_virt_release_full_gpu(adev, false);
2965
2966         return r;
2967 }
2968
2969 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2970 {
2971         int i, r;
2972
2973         static enum amd_ip_block_type ip_order[] = {
2974                 AMD_IP_BLOCK_TYPE_GMC,
2975                 AMD_IP_BLOCK_TYPE_COMMON,
2976                 AMD_IP_BLOCK_TYPE_PSP,
2977                 AMD_IP_BLOCK_TYPE_IH,
2978         };
2979
2980         for (i = 0; i < adev->num_ip_blocks; i++) {
2981                 int j;
2982                 struct amdgpu_ip_block *block;
2983
2984                 block = &adev->ip_blocks[i];
2985                 block->status.hw = false;
2986
2987                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2988
2989                         if (block->version->type != ip_order[j] ||
2990                                 !block->status.valid)
2991                                 continue;
2992
2993                         r = block->version->funcs->hw_init(adev);
2994                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2995                         if (r)
2996                                 return r;
2997                         block->status.hw = true;
2998                 }
2999         }
3000
3001         return 0;
3002 }
3003
3004 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3005 {
3006         int i, r;
3007
3008         static enum amd_ip_block_type ip_order[] = {
3009                 AMD_IP_BLOCK_TYPE_SMC,
3010                 AMD_IP_BLOCK_TYPE_DCE,
3011                 AMD_IP_BLOCK_TYPE_GFX,
3012                 AMD_IP_BLOCK_TYPE_SDMA,
3013                 AMD_IP_BLOCK_TYPE_UVD,
3014                 AMD_IP_BLOCK_TYPE_VCE,
3015                 AMD_IP_BLOCK_TYPE_VCN
3016         };
3017
3018         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3019                 int j;
3020                 struct amdgpu_ip_block *block;
3021
3022                 for (j = 0; j < adev->num_ip_blocks; j++) {
3023                         block = &adev->ip_blocks[j];
3024
3025                         if (block->version->type != ip_order[i] ||
3026                                 !block->status.valid ||
3027                                 block->status.hw)
3028                                 continue;
3029
3030                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3031                                 r = block->version->funcs->resume(adev);
3032                         else
3033                                 r = block->version->funcs->hw_init(adev);
3034
3035                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3036                         if (r)
3037                                 return r;
3038                         block->status.hw = true;
3039                 }
3040         }
3041
3042         return 0;
3043 }
3044
3045 /**
3046  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3047  *
3048  * @adev: amdgpu_device pointer
3049  *
3050  * First resume function for hardware IPs.  The list of all the hardware
3051  * IPs that make up the asic is walked and the resume callbacks are run for
3052  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3053  * after a suspend and updates the software state as necessary.  This
3054  * function is also used for restoring the GPU after a GPU reset.
3055  * Returns 0 on success, negative error code on failure.
3056  */
3057 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3058 {
3059         int i, r;
3060
3061         for (i = 0; i < adev->num_ip_blocks; i++) {
3062                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3063                         continue;
3064                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3065                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3066                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
3067
3068                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3069                         if (r) {
3070                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3071                                           adev->ip_blocks[i].version->funcs->name, r);
3072                                 return r;
3073                         }
3074                         adev->ip_blocks[i].status.hw = true;
3075                 }
3076         }
3077
3078         return 0;
3079 }
3080
3081 /**
3082  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3083  *
3084  * @adev: amdgpu_device pointer
3085  *
3086  * First resume function for hardware IPs.  The list of all the hardware
3087  * IPs that make up the asic is walked and the resume callbacks are run for
3088  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3089  * functional state after a suspend and updates the software state as
3090  * necessary.  This function is also used for restoring the GPU after a GPU
3091  * reset.
3092  * Returns 0 on success, negative error code on failure.
3093  */
3094 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3095 {
3096         int i, r;
3097
3098         for (i = 0; i < adev->num_ip_blocks; i++) {
3099                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3100                         continue;
3101                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3102                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3103                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3104                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3105                         continue;
3106                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3107                 if (r) {
3108                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3109                                   adev->ip_blocks[i].version->funcs->name, r);
3110                         return r;
3111                 }
3112                 adev->ip_blocks[i].status.hw = true;
3113         }
3114
3115         return 0;
3116 }
3117
3118 /**
3119  * amdgpu_device_ip_resume - run resume for hardware IPs
3120  *
3121  * @adev: amdgpu_device pointer
3122  *
3123  * Main resume function for hardware IPs.  The hardware IPs
3124  * are split into two resume functions because they are
3125  * are also used in in recovering from a GPU reset and some additional
3126  * steps need to be take between them.  In this case (S3/S4) they are
3127  * run sequentially.
3128  * Returns 0 on success, negative error code on failure.
3129  */
3130 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3131 {
3132         int r;
3133
3134         r = amdgpu_amdkfd_resume_iommu(adev);
3135         if (r)
3136                 return r;
3137
3138         r = amdgpu_device_ip_resume_phase1(adev);
3139         if (r)
3140                 return r;
3141
3142         r = amdgpu_device_fw_loading(adev);
3143         if (r)
3144                 return r;
3145
3146         r = amdgpu_device_ip_resume_phase2(adev);
3147
3148         return r;
3149 }
3150
3151 /**
3152  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3153  *
3154  * @adev: amdgpu_device pointer
3155  *
3156  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3157  */
3158 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3159 {
3160         if (amdgpu_sriov_vf(adev)) {
3161                 if (adev->is_atom_fw) {
3162                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3163                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3164                 } else {
3165                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3166                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3167                 }
3168
3169                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3170                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3171         }
3172 }
3173
3174 /**
3175  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3176  *
3177  * @asic_type: AMD asic type
3178  *
3179  * Check if there is DC (new modesetting infrastructre) support for an asic.
3180  * returns true if DC has support, false if not.
3181  */
3182 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3183 {
3184         switch (asic_type) {
3185 #if defined(CONFIG_DRM_AMD_DC)
3186         case CHIP_TAHITI:
3187         case CHIP_PITCAIRN:
3188         case CHIP_VERDE:
3189         case CHIP_OLAND:
3190                 /*
3191                  * We have systems in the wild with these ASICs that require
3192                  * LVDS and VGA support which is not supported with DC.
3193                  *
3194                  * Fallback to the non-DC driver here by default so as not to
3195                  * cause regressions.
3196                  */
3197 #if defined(CONFIG_DRM_AMD_DC_SI)
3198                 return amdgpu_dc > 0;
3199 #else
3200                 return false;
3201 #endif
3202         case CHIP_BONAIRE:
3203         case CHIP_KAVERI:
3204         case CHIP_KABINI:
3205         case CHIP_MULLINS:
3206                 /*
3207                  * We have systems in the wild with these ASICs that require
3208                  * LVDS and VGA support which is not supported with DC.
3209                  *
3210                  * Fallback to the non-DC driver here by default so as not to
3211                  * cause regressions.
3212                  */
3213                 return amdgpu_dc > 0;
3214         case CHIP_HAWAII:
3215         case CHIP_CARRIZO:
3216         case CHIP_STONEY:
3217         case CHIP_POLARIS10:
3218         case CHIP_POLARIS11:
3219         case CHIP_POLARIS12:
3220         case CHIP_VEGAM:
3221         case CHIP_TONGA:
3222         case CHIP_FIJI:
3223         case CHIP_VEGA10:
3224         case CHIP_VEGA12:
3225         case CHIP_VEGA20:
3226 #if defined(CONFIG_DRM_AMD_DC_DCN)
3227         case CHIP_RAVEN:
3228         case CHIP_NAVI10:
3229         case CHIP_NAVI14:
3230         case CHIP_NAVI12:
3231         case CHIP_RENOIR:
3232         case CHIP_CYAN_SKILLFISH:
3233         case CHIP_SIENNA_CICHLID:
3234         case CHIP_NAVY_FLOUNDER:
3235         case CHIP_DIMGREY_CAVEFISH:
3236         case CHIP_BEIGE_GOBY:
3237         case CHIP_VANGOGH:
3238         case CHIP_YELLOW_CARP:
3239 #endif
3240         default:
3241                 return amdgpu_dc != 0;
3242 #else
3243         default:
3244                 if (amdgpu_dc > 0)
3245                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3246                                          "but isn't supported by ASIC, ignoring\n");
3247                 return false;
3248 #endif
3249         }
3250 }
3251
3252 /**
3253  * amdgpu_device_has_dc_support - check if dc is supported
3254  *
3255  * @adev: amdgpu_device pointer
3256  *
3257  * Returns true for supported, false for not supported
3258  */
3259 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3260 {
3261         if (amdgpu_sriov_vf(adev) ||
3262             adev->enable_virtual_display ||
3263             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3264                 return false;
3265
3266         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3267 }
3268
3269 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3270 {
3271         struct amdgpu_device *adev =
3272                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3273         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3274
3275         /* It's a bug to not have a hive within this function */
3276         if (WARN_ON(!hive))
3277                 return;
3278
3279         /*
3280          * Use task barrier to synchronize all xgmi reset works across the
3281          * hive. task_barrier_enter and task_barrier_exit will block
3282          * until all the threads running the xgmi reset works reach
3283          * those points. task_barrier_full will do both blocks.
3284          */
3285         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3286
3287                 task_barrier_enter(&hive->tb);
3288                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3289
3290                 if (adev->asic_reset_res)
3291                         goto fail;
3292
3293                 task_barrier_exit(&hive->tb);
3294                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3295
3296                 if (adev->asic_reset_res)
3297                         goto fail;
3298
3299                 if (adev->mmhub.ras_funcs &&
3300                     adev->mmhub.ras_funcs->reset_ras_error_count)
3301                         adev->mmhub.ras_funcs->reset_ras_error_count(adev);
3302         } else {
3303
3304                 task_barrier_full(&hive->tb);
3305                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3306         }
3307
3308 fail:
3309         if (adev->asic_reset_res)
3310                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3311                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3312         amdgpu_put_xgmi_hive(hive);
3313 }
3314
3315 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3316 {
3317         char *input = amdgpu_lockup_timeout;
3318         char *timeout_setting = NULL;
3319         int index = 0;
3320         long timeout;
3321         int ret = 0;
3322
3323         /*
3324          * By default timeout for non compute jobs is 10000
3325          * and 60000 for compute jobs.
3326          * In SR-IOV or passthrough mode, timeout for compute
3327          * jobs are 60000 by default.
3328          */
3329         adev->gfx_timeout = msecs_to_jiffies(10000);
3330         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3331         if (amdgpu_sriov_vf(adev))
3332                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3333                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3334         else
3335                 adev->compute_timeout =  msecs_to_jiffies(60000);
3336
3337         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3338                 while ((timeout_setting = strsep(&input, ",")) &&
3339                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3340                         ret = kstrtol(timeout_setting, 0, &timeout);
3341                         if (ret)
3342                                 return ret;
3343
3344                         if (timeout == 0) {
3345                                 index++;
3346                                 continue;
3347                         } else if (timeout < 0) {
3348                                 timeout = MAX_SCHEDULE_TIMEOUT;
3349                                 dev_warn(adev->dev, "lockup timeout disabled");
3350                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3351                         } else {
3352                                 timeout = msecs_to_jiffies(timeout);
3353                         }
3354
3355                         switch (index++) {
3356                         case 0:
3357                                 adev->gfx_timeout = timeout;
3358                                 break;
3359                         case 1:
3360                                 adev->compute_timeout = timeout;
3361                                 break;
3362                         case 2:
3363                                 adev->sdma_timeout = timeout;
3364                                 break;
3365                         case 3:
3366                                 adev->video_timeout = timeout;
3367                                 break;
3368                         default:
3369                                 break;
3370                         }
3371                 }
3372                 /*
3373                  * There is only one value specified and
3374                  * it should apply to all non-compute jobs.
3375                  */
3376                 if (index == 1) {
3377                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3378                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3379                                 adev->compute_timeout = adev->gfx_timeout;
3380                 }
3381         }
3382
3383         return ret;
3384 }
3385
3386 /**
3387  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3388  *
3389  * @adev: amdgpu_device pointer
3390  *
3391  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3392  */
3393 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3394 {
3395         struct iommu_domain *domain;
3396
3397         domain = iommu_get_domain_for_dev(adev->dev);
3398         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3399                 adev->ram_is_direct_mapped = true;
3400 }
3401
3402 static const struct attribute *amdgpu_dev_attributes[] = {
3403         &dev_attr_product_name.attr,
3404         &dev_attr_product_number.attr,
3405         &dev_attr_serial_number.attr,
3406         &dev_attr_pcie_replay_count.attr,
3407         NULL
3408 };
3409
3410 /**
3411  * amdgpu_device_init - initialize the driver
3412  *
3413  * @adev: amdgpu_device pointer
3414  * @flags: driver flags
3415  *
3416  * Initializes the driver info and hw (all asics).
3417  * Returns 0 for success or an error on failure.
3418  * Called at driver startup.
3419  */
3420 int amdgpu_device_init(struct amdgpu_device *adev,
3421                        uint32_t flags)
3422 {
3423         struct drm_device *ddev = adev_to_drm(adev);
3424         struct pci_dev *pdev = adev->pdev;
3425         int r, i;
3426         bool px = false;
3427         u32 max_MBps;
3428
3429         adev->shutdown = false;
3430         adev->flags = flags;
3431
3432         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3433                 adev->asic_type = amdgpu_force_asic_type;
3434         else
3435                 adev->asic_type = flags & AMD_ASIC_MASK;
3436
3437         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3438         if (amdgpu_emu_mode == 1)
3439                 adev->usec_timeout *= 10;
3440         adev->gmc.gart_size = 512 * 1024 * 1024;
3441         adev->accel_working = false;
3442         adev->num_rings = 0;
3443         adev->mman.buffer_funcs = NULL;
3444         adev->mman.buffer_funcs_ring = NULL;
3445         adev->vm_manager.vm_pte_funcs = NULL;
3446         adev->vm_manager.vm_pte_num_scheds = 0;
3447         adev->gmc.gmc_funcs = NULL;
3448         adev->harvest_ip_mask = 0x0;
3449         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3450         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3451
3452         adev->smc_rreg = &amdgpu_invalid_rreg;
3453         adev->smc_wreg = &amdgpu_invalid_wreg;
3454         adev->pcie_rreg = &amdgpu_invalid_rreg;
3455         adev->pcie_wreg = &amdgpu_invalid_wreg;
3456         adev->pciep_rreg = &amdgpu_invalid_rreg;
3457         adev->pciep_wreg = &amdgpu_invalid_wreg;
3458         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3459         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3460         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3461         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3462         adev->didt_rreg = &amdgpu_invalid_rreg;
3463         adev->didt_wreg = &amdgpu_invalid_wreg;
3464         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3465         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3466         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3467         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3468
3469         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3470                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3471                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3472
3473         /* mutex initialization are all done here so we
3474          * can recall function without having locking issues */
3475         mutex_init(&adev->firmware.mutex);
3476         mutex_init(&adev->pm.mutex);
3477         mutex_init(&adev->gfx.gpu_clock_mutex);
3478         mutex_init(&adev->srbm_mutex);
3479         mutex_init(&adev->gfx.pipe_reserve_mutex);
3480         mutex_init(&adev->gfx.gfx_off_mutex);
3481         mutex_init(&adev->grbm_idx_mutex);
3482         mutex_init(&adev->mn_lock);
3483         mutex_init(&adev->virt.vf_errors.lock);
3484         hash_init(adev->mn_hash);
3485         atomic_set(&adev->in_gpu_reset, 0);
3486         init_rwsem(&adev->reset_sem);
3487         mutex_init(&adev->psp.mutex);
3488         mutex_init(&adev->notifier_lock);
3489
3490         r = amdgpu_device_init_apu_flags(adev);
3491         if (r)
3492                 return r;
3493
3494         r = amdgpu_device_check_arguments(adev);
3495         if (r)
3496                 return r;
3497
3498         spin_lock_init(&adev->mmio_idx_lock);
3499         spin_lock_init(&adev->smc_idx_lock);
3500         spin_lock_init(&adev->pcie_idx_lock);
3501         spin_lock_init(&adev->uvd_ctx_idx_lock);
3502         spin_lock_init(&adev->didt_idx_lock);
3503         spin_lock_init(&adev->gc_cac_idx_lock);
3504         spin_lock_init(&adev->se_cac_idx_lock);
3505         spin_lock_init(&adev->audio_endpt_idx_lock);
3506         spin_lock_init(&adev->mm_stats.lock);
3507
3508         INIT_LIST_HEAD(&adev->shadow_list);
3509         mutex_init(&adev->shadow_list_lock);
3510
3511         INIT_LIST_HEAD(&adev->reset_list);
3512
3513         INIT_DELAYED_WORK(&adev->delayed_init_work,
3514                           amdgpu_device_delayed_init_work_handler);
3515         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3516                           amdgpu_device_delay_enable_gfx_off);
3517
3518         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3519
3520         adev->gfx.gfx_off_req_count = 1;
3521         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3522
3523         atomic_set(&adev->throttling_logging_enabled, 1);
3524         /*
3525          * If throttling continues, logging will be performed every minute
3526          * to avoid log flooding. "-1" is subtracted since the thermal
3527          * throttling interrupt comes every second. Thus, the total logging
3528          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3529          * for throttling interrupt) = 60 seconds.
3530          */
3531         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3532         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3533
3534         /* Registers mapping */
3535         /* TODO: block userspace mapping of io register */
3536         if (adev->asic_type >= CHIP_BONAIRE) {
3537                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3538                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3539         } else {
3540                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3541                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3542         }
3543
3544         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3545                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3546
3547         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3548         if (adev->rmmio == NULL) {
3549                 return -ENOMEM;
3550         }
3551         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3552         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3553
3554         amdgpu_device_get_pcie_info(adev);
3555
3556         if (amdgpu_mcbp)
3557                 DRM_INFO("MCBP is enabled\n");
3558
3559         if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3560                 adev->enable_mes = true;
3561
3562         /* detect hw virtualization here */
3563         amdgpu_detect_virtualization(adev);
3564
3565         r = amdgpu_device_get_job_timeout_settings(adev);
3566         if (r) {
3567                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3568                 return r;
3569         }
3570
3571         /* early init functions */
3572         r = amdgpu_device_ip_early_init(adev);
3573         if (r)
3574                 return r;
3575
3576         /* enable PCIE atomic ops */
3577         if (amdgpu_sriov_vf(adev))
3578                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3579                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_enabled_flags ==
3580                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3581         else
3582                 adev->have_atomics_support =
3583                         !pci_enable_atomic_ops_to_root(adev->pdev,
3584                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3585                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3586         if (!adev->have_atomics_support)
3587                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3588
3589         /* doorbell bar mapping and doorbell index init*/
3590         amdgpu_device_doorbell_init(adev);
3591
3592         if (amdgpu_emu_mode == 1) {
3593                 /* post the asic on emulation mode */
3594                 emu_soc_asic_init(adev);
3595                 goto fence_driver_init;
3596         }
3597
3598         amdgpu_reset_init(adev);
3599
3600         /* detect if we are with an SRIOV vbios */
3601         amdgpu_device_detect_sriov_bios(adev);
3602
3603         /* check if we need to reset the asic
3604          *  E.g., driver was not cleanly unloaded previously, etc.
3605          */
3606         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3607                 if (adev->gmc.xgmi.num_physical_nodes) {
3608                         dev_info(adev->dev, "Pending hive reset.\n");
3609                         adev->gmc.xgmi.pending_reset = true;
3610                         /* Only need to init necessary block for SMU to handle the reset */
3611                         for (i = 0; i < adev->num_ip_blocks; i++) {
3612                                 if (!adev->ip_blocks[i].status.valid)
3613                                         continue;
3614                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3615                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3616                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3617                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3618                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3619                                                 adev->ip_blocks[i].version->funcs->name);
3620                                         adev->ip_blocks[i].status.hw = true;
3621                                 }
3622                         }
3623                 } else {
3624                         r = amdgpu_asic_reset(adev);
3625                         if (r) {
3626                                 dev_err(adev->dev, "asic reset on init failed\n");
3627                                 goto failed;
3628                         }
3629                 }
3630         }
3631
3632         pci_enable_pcie_error_reporting(adev->pdev);
3633
3634         /* Post card if necessary */
3635         if (amdgpu_device_need_post(adev)) {
3636                 if (!adev->bios) {
3637                         dev_err(adev->dev, "no vBIOS found\n");
3638                         r = -EINVAL;
3639                         goto failed;
3640                 }
3641                 DRM_INFO("GPU posting now...\n");
3642                 r = amdgpu_device_asic_init(adev);
3643                 if (r) {
3644                         dev_err(adev->dev, "gpu post error!\n");
3645                         goto failed;
3646                 }
3647         }
3648
3649         if (adev->is_atom_fw) {
3650                 /* Initialize clocks */
3651                 r = amdgpu_atomfirmware_get_clock_info(adev);
3652                 if (r) {
3653                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3654                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3655                         goto failed;
3656                 }
3657         } else {
3658                 /* Initialize clocks */
3659                 r = amdgpu_atombios_get_clock_info(adev);
3660                 if (r) {
3661                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3662                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3663                         goto failed;
3664                 }
3665                 /* init i2c buses */
3666                 if (!amdgpu_device_has_dc_support(adev))
3667                         amdgpu_atombios_i2c_init(adev);
3668         }
3669
3670 fence_driver_init:
3671         /* Fence driver */
3672         r = amdgpu_fence_driver_sw_init(adev);
3673         if (r) {
3674                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3675                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3676                 goto failed;
3677         }
3678
3679         /* init the mode config */
3680         drm_mode_config_init(adev_to_drm(adev));
3681
3682         r = amdgpu_device_ip_init(adev);
3683         if (r) {
3684                 /* failed in exclusive mode due to timeout */
3685                 if (amdgpu_sriov_vf(adev) &&
3686                     !amdgpu_sriov_runtime(adev) &&
3687                     amdgpu_virt_mmio_blocked(adev) &&
3688                     !amdgpu_virt_wait_reset(adev)) {
3689                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3690                         /* Don't send request since VF is inactive. */
3691                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3692                         adev->virt.ops = NULL;
3693                         r = -EAGAIN;
3694                         goto release_ras_con;
3695                 }
3696                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3697                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3698                 goto release_ras_con;
3699         }
3700
3701         amdgpu_fence_driver_hw_init(adev);
3702
3703         dev_info(adev->dev,
3704                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3705                         adev->gfx.config.max_shader_engines,
3706                         adev->gfx.config.max_sh_per_se,
3707                         adev->gfx.config.max_cu_per_sh,
3708                         adev->gfx.cu_info.number);
3709
3710         adev->accel_working = true;
3711
3712         amdgpu_vm_check_compute_bug(adev);
3713
3714         /* Initialize the buffer migration limit. */
3715         if (amdgpu_moverate >= 0)
3716                 max_MBps = amdgpu_moverate;
3717         else
3718                 max_MBps = 8; /* Allow 8 MB/s. */
3719         /* Get a log2 for easy divisions. */
3720         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3721
3722         r = amdgpu_pm_sysfs_init(adev);
3723         if (r) {
3724                 adev->pm_sysfs_en = false;
3725                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3726         } else
3727                 adev->pm_sysfs_en = true;
3728
3729         r = amdgpu_ucode_sysfs_init(adev);
3730         if (r) {
3731                 adev->ucode_sysfs_en = false;
3732                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3733         } else
3734                 adev->ucode_sysfs_en = true;
3735
3736         if ((amdgpu_testing & 1)) {
3737                 if (adev->accel_working)
3738                         amdgpu_test_moves(adev);
3739                 else
3740                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3741         }
3742         if (amdgpu_benchmarking) {
3743                 if (adev->accel_working)
3744                         amdgpu_benchmark(adev, amdgpu_benchmarking);
3745                 else
3746                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3747         }
3748
3749         /*
3750          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3751          * Otherwise the mgpu fan boost feature will be skipped due to the
3752          * gpu instance is counted less.
3753          */
3754         amdgpu_register_gpu_instance(adev);
3755
3756         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3757          * explicit gating rather than handling it automatically.
3758          */
3759         if (!adev->gmc.xgmi.pending_reset) {
3760                 r = amdgpu_device_ip_late_init(adev);
3761                 if (r) {
3762                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3763                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3764                         goto release_ras_con;
3765                 }
3766                 /* must succeed. */
3767                 amdgpu_ras_resume(adev);
3768                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3769                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3770         }
3771
3772         if (amdgpu_sriov_vf(adev))
3773                 flush_delayed_work(&adev->delayed_init_work);
3774
3775         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3776         if (r)
3777                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3778
3779         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3780                 r = amdgpu_pmu_init(adev);
3781         if (r)
3782                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3783
3784         /* Have stored pci confspace at hand for restore in sudden PCI error */
3785         if (amdgpu_device_cache_pci_state(adev->pdev))
3786                 pci_restore_state(pdev);
3787
3788         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3789         /* this will fail for cards that aren't VGA class devices, just
3790          * ignore it */
3791         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3792                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3793
3794         if (amdgpu_device_supports_px(ddev)) {
3795                 px = true;
3796                 vga_switcheroo_register_client(adev->pdev,
3797                                                &amdgpu_switcheroo_ops, px);
3798                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3799         }
3800
3801         if (adev->gmc.xgmi.pending_reset)
3802                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3803                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3804
3805         amdgpu_device_check_iommu_direct_map(adev);
3806
3807         return 0;
3808
3809 release_ras_con:
3810         amdgpu_release_ras_context(adev);
3811
3812 failed:
3813         amdgpu_vf_error_trans_all(adev);
3814
3815         return r;
3816 }
3817
3818 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3819 {
3820         /* Clear all CPU mappings pointing to this device */
3821         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3822
3823         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3824         amdgpu_device_doorbell_fini(adev);
3825
3826         iounmap(adev->rmmio);
3827         adev->rmmio = NULL;
3828         if (adev->mman.aper_base_kaddr)
3829                 iounmap(adev->mman.aper_base_kaddr);
3830         adev->mman.aper_base_kaddr = NULL;
3831
3832         /* Memory manager related */
3833         if (!adev->gmc.xgmi.connected_to_cpu) {
3834                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3835                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3836         }
3837 }
3838
3839 /**
3840  * amdgpu_device_fini_hw - tear down the driver
3841  *
3842  * @adev: amdgpu_device pointer
3843  *
3844  * Tear down the driver info (all asics).
3845  * Called at driver shutdown.
3846  */
3847 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3848 {
3849         dev_info(adev->dev, "amdgpu: finishing device.\n");
3850         flush_delayed_work(&adev->delayed_init_work);
3851         if (adev->mman.initialized) {
3852                 flush_delayed_work(&adev->mman.bdev.wq);
3853                 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3854         }
3855         adev->shutdown = true;
3856
3857         /* make sure IB test finished before entering exclusive mode
3858          * to avoid preemption on IB test
3859          * */
3860         if (amdgpu_sriov_vf(adev)) {
3861                 amdgpu_virt_request_full_gpu(adev, false);
3862                 amdgpu_virt_fini_data_exchange(adev);
3863         }
3864
3865         /* disable all interrupts */
3866         amdgpu_irq_disable_all(adev);
3867         if (adev->mode_info.mode_config_initialized){
3868                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
3869                         drm_helper_force_disable_all(adev_to_drm(adev));
3870                 else
3871                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3872         }
3873         amdgpu_fence_driver_hw_fini(adev);
3874
3875         if (adev->pm_sysfs_en)
3876                 amdgpu_pm_sysfs_fini(adev);
3877         if (adev->ucode_sysfs_en)
3878                 amdgpu_ucode_sysfs_fini(adev);
3879         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3880
3881         /* disable ras feature must before hw fini */
3882         amdgpu_ras_pre_fini(adev);
3883
3884         amdgpu_device_ip_fini_early(adev);
3885
3886         amdgpu_irq_fini_hw(adev);
3887
3888         ttm_device_clear_dma_mappings(&adev->mman.bdev);
3889
3890         amdgpu_gart_dummy_page_fini(adev);
3891
3892         amdgpu_device_unmap_mmio(adev);
3893 }
3894
3895 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
3896 {
3897         amdgpu_fence_driver_sw_fini(adev);
3898         amdgpu_device_ip_fini(adev);
3899         release_firmware(adev->firmware.gpu_info_fw);
3900         adev->firmware.gpu_info_fw = NULL;
3901         adev->accel_working = false;
3902
3903         amdgpu_reset_fini(adev);
3904
3905         /* free i2c buses */
3906         if (!amdgpu_device_has_dc_support(adev))
3907                 amdgpu_i2c_fini(adev);
3908
3909         if (amdgpu_emu_mode != 1)
3910                 amdgpu_atombios_fini(adev);
3911
3912         kfree(adev->bios);
3913         adev->bios = NULL;
3914         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3915                 vga_switcheroo_unregister_client(adev->pdev);
3916                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3917         }
3918         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3919                 vga_client_unregister(adev->pdev);
3920
3921         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3922                 amdgpu_pmu_fini(adev);
3923         if (adev->mman.discovery_bin)
3924                 amdgpu_discovery_fini(adev);
3925
3926         kfree(adev->pci_state);
3927
3928 }
3929
3930 /**
3931  * amdgpu_device_evict_resources - evict device resources
3932  * @adev: amdgpu device object
3933  *
3934  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
3935  * of the vram memory type. Mainly used for evicting device resources
3936  * at suspend time.
3937  *
3938  */
3939 static void amdgpu_device_evict_resources(struct amdgpu_device *adev)
3940 {
3941         /* No need to evict vram on APUs for suspend to ram */
3942         if (adev->in_s3 && (adev->flags & AMD_IS_APU))
3943                 return;
3944
3945         if (amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM))
3946                 DRM_WARN("evicting device resources failed\n");
3947
3948 }
3949
3950 /*
3951  * Suspend & resume.
3952  */
3953 /**
3954  * amdgpu_device_suspend - initiate device suspend
3955  *
3956  * @dev: drm dev pointer
3957  * @fbcon : notify the fbdev of suspend
3958  *
3959  * Puts the hw in the suspend state (all asics).
3960  * Returns 0 for success or an error on failure.
3961  * Called at driver suspend.
3962  */
3963 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3964 {
3965         struct amdgpu_device *adev = drm_to_adev(dev);
3966
3967         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3968                 return 0;
3969
3970         adev->in_suspend = true;
3971
3972         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
3973                 DRM_WARN("smart shift update failed\n");
3974
3975         drm_kms_helper_poll_disable(dev);
3976
3977         if (fbcon)
3978                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
3979
3980         cancel_delayed_work_sync(&adev->delayed_init_work);
3981
3982         amdgpu_ras_suspend(adev);
3983
3984         amdgpu_device_ip_suspend_phase1(adev);
3985
3986         if (!adev->in_s0ix)
3987                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
3988
3989         /* First evict vram memory */
3990         amdgpu_device_evict_resources(adev);
3991
3992         amdgpu_fence_driver_hw_fini(adev);
3993
3994         amdgpu_device_ip_suspend_phase2(adev);
3995         /* This second call to evict device resources is to evict
3996          * the gart page table using the CPU.
3997          */
3998         amdgpu_device_evict_resources(adev);
3999
4000         return 0;
4001 }
4002
4003 /**
4004  * amdgpu_device_resume - initiate device resume
4005  *
4006  * @dev: drm dev pointer
4007  * @fbcon : notify the fbdev of resume
4008  *
4009  * Bring the hw back to operating state (all asics).
4010  * Returns 0 for success or an error on failure.
4011  * Called at driver resume.
4012  */
4013 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4014 {
4015         struct amdgpu_device *adev = drm_to_adev(dev);
4016         int r = 0;
4017
4018         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4019                 return 0;
4020
4021         if (adev->in_s0ix)
4022                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
4023
4024         /* post card */
4025         if (amdgpu_device_need_post(adev)) {
4026                 r = amdgpu_device_asic_init(adev);
4027                 if (r)
4028                         dev_err(adev->dev, "amdgpu asic init failed\n");
4029         }
4030
4031         r = amdgpu_device_ip_resume(adev);
4032         if (r) {
4033                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4034                 return r;
4035         }
4036         amdgpu_fence_driver_hw_init(adev);
4037
4038         r = amdgpu_device_ip_late_init(adev);
4039         if (r)
4040                 return r;
4041
4042         queue_delayed_work(system_wq, &adev->delayed_init_work,
4043                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4044
4045         if (!adev->in_s0ix) {
4046                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4047                 if (r)
4048                         return r;
4049         }
4050
4051         /* Make sure IB tests flushed */
4052         flush_delayed_work(&adev->delayed_init_work);
4053
4054         if (fbcon)
4055                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4056
4057         drm_kms_helper_poll_enable(dev);
4058
4059         amdgpu_ras_resume(adev);
4060
4061         /*
4062          * Most of the connector probing functions try to acquire runtime pm
4063          * refs to ensure that the GPU is powered on when connector polling is
4064          * performed. Since we're calling this from a runtime PM callback,
4065          * trying to acquire rpm refs will cause us to deadlock.
4066          *
4067          * Since we're guaranteed to be holding the rpm lock, it's safe to
4068          * temporarily disable the rpm helpers so this doesn't deadlock us.
4069          */
4070 #ifdef CONFIG_PM
4071         dev->dev->power.disable_depth++;
4072 #endif
4073         if (!amdgpu_device_has_dc_support(adev))
4074                 drm_helper_hpd_irq_event(dev);
4075         else
4076                 drm_kms_helper_hotplug_event(dev);
4077 #ifdef CONFIG_PM
4078         dev->dev->power.disable_depth--;
4079 #endif
4080         adev->in_suspend = false;
4081
4082         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4083                 DRM_WARN("smart shift update failed\n");
4084
4085         return 0;
4086 }
4087
4088 /**
4089  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4090  *
4091  * @adev: amdgpu_device pointer
4092  *
4093  * The list of all the hardware IPs that make up the asic is walked and
4094  * the check_soft_reset callbacks are run.  check_soft_reset determines
4095  * if the asic is still hung or not.
4096  * Returns true if any of the IPs are still in a hung state, false if not.
4097  */
4098 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4099 {
4100         int i;
4101         bool asic_hang = false;
4102
4103         if (amdgpu_sriov_vf(adev))
4104                 return true;
4105
4106         if (amdgpu_asic_need_full_reset(adev))
4107                 return true;
4108
4109         for (i = 0; i < adev->num_ip_blocks; i++) {
4110                 if (!adev->ip_blocks[i].status.valid)
4111                         continue;
4112                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4113                         adev->ip_blocks[i].status.hang =
4114                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4115                 if (adev->ip_blocks[i].status.hang) {
4116                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4117                         asic_hang = true;
4118                 }
4119         }
4120         return asic_hang;
4121 }
4122
4123 /**
4124  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4125  *
4126  * @adev: amdgpu_device pointer
4127  *
4128  * The list of all the hardware IPs that make up the asic is walked and the
4129  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4130  * handles any IP specific hardware or software state changes that are
4131  * necessary for a soft reset to succeed.
4132  * Returns 0 on success, negative error code on failure.
4133  */
4134 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4135 {
4136         int i, r = 0;
4137
4138         for (i = 0; i < adev->num_ip_blocks; i++) {
4139                 if (!adev->ip_blocks[i].status.valid)
4140                         continue;
4141                 if (adev->ip_blocks[i].status.hang &&
4142                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4143                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4144                         if (r)
4145                                 return r;
4146                 }
4147         }
4148
4149         return 0;
4150 }
4151
4152 /**
4153  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4154  *
4155  * @adev: amdgpu_device pointer
4156  *
4157  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4158  * reset is necessary to recover.
4159  * Returns true if a full asic reset is required, false if not.
4160  */
4161 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4162 {
4163         int i;
4164
4165         if (amdgpu_asic_need_full_reset(adev))
4166                 return true;
4167
4168         for (i = 0; i < adev->num_ip_blocks; i++) {
4169                 if (!adev->ip_blocks[i].status.valid)
4170                         continue;
4171                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4172                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4173                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4174                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4175                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4176                         if (adev->ip_blocks[i].status.hang) {
4177                                 dev_info(adev->dev, "Some block need full reset!\n");
4178                                 return true;
4179                         }
4180                 }
4181         }
4182         return false;
4183 }
4184
4185 /**
4186  * amdgpu_device_ip_soft_reset - do a soft reset
4187  *
4188  * @adev: amdgpu_device pointer
4189  *
4190  * The list of all the hardware IPs that make up the asic is walked and the
4191  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4192  * IP specific hardware or software state changes that are necessary to soft
4193  * reset the IP.
4194  * Returns 0 on success, negative error code on failure.
4195  */
4196 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4197 {
4198         int i, r = 0;
4199
4200         for (i = 0; i < adev->num_ip_blocks; i++) {
4201                 if (!adev->ip_blocks[i].status.valid)
4202                         continue;
4203                 if (adev->ip_blocks[i].status.hang &&
4204                     adev->ip_blocks[i].version->funcs->soft_reset) {
4205                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4206                         if (r)
4207                                 return r;
4208                 }
4209         }
4210
4211         return 0;
4212 }
4213
4214 /**
4215  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4216  *
4217  * @adev: amdgpu_device pointer
4218  *
4219  * The list of all the hardware IPs that make up the asic is walked and the
4220  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4221  * handles any IP specific hardware or software state changes that are
4222  * necessary after the IP has been soft reset.
4223  * Returns 0 on success, negative error code on failure.
4224  */
4225 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4226 {
4227         int i, r = 0;
4228
4229         for (i = 0; i < adev->num_ip_blocks; i++) {
4230                 if (!adev->ip_blocks[i].status.valid)
4231                         continue;
4232                 if (adev->ip_blocks[i].status.hang &&
4233                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4234                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4235                 if (r)
4236                         return r;
4237         }
4238
4239         return 0;
4240 }
4241
4242 /**
4243  * amdgpu_device_recover_vram - Recover some VRAM contents
4244  *
4245  * @adev: amdgpu_device pointer
4246  *
4247  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4248  * restore things like GPUVM page tables after a GPU reset where
4249  * the contents of VRAM might be lost.
4250  *
4251  * Returns:
4252  * 0 on success, negative error code on failure.
4253  */
4254 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4255 {
4256         struct dma_fence *fence = NULL, *next = NULL;
4257         struct amdgpu_bo *shadow;
4258         struct amdgpu_bo_vm *vmbo;
4259         long r = 1, tmo;
4260
4261         if (amdgpu_sriov_runtime(adev))
4262                 tmo = msecs_to_jiffies(8000);
4263         else
4264                 tmo = msecs_to_jiffies(100);
4265
4266         dev_info(adev->dev, "recover vram bo from shadow start\n");
4267         mutex_lock(&adev->shadow_list_lock);
4268         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4269                 shadow = &vmbo->bo;
4270                 /* No need to recover an evicted BO */
4271                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4272                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4273                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4274                         continue;
4275
4276                 r = amdgpu_bo_restore_shadow(shadow, &next);
4277                 if (r)
4278                         break;
4279
4280                 if (fence) {
4281                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4282                         dma_fence_put(fence);
4283                         fence = next;
4284                         if (tmo == 0) {
4285                                 r = -ETIMEDOUT;
4286                                 break;
4287                         } else if (tmo < 0) {
4288                                 r = tmo;
4289                                 break;
4290                         }
4291                 } else {
4292                         fence = next;
4293                 }
4294         }
4295         mutex_unlock(&adev->shadow_list_lock);
4296
4297         if (fence)
4298                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4299         dma_fence_put(fence);
4300
4301         if (r < 0 || tmo <= 0) {
4302                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4303                 return -EIO;
4304         }
4305
4306         dev_info(adev->dev, "recover vram bo from shadow done\n");
4307         return 0;
4308 }
4309
4310
4311 /**
4312  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4313  *
4314  * @adev: amdgpu_device pointer
4315  * @from_hypervisor: request from hypervisor
4316  *
4317  * do VF FLR and reinitialize Asic
4318  * return 0 means succeeded otherwise failed
4319  */
4320 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4321                                      bool from_hypervisor)
4322 {
4323         int r;
4324         struct amdgpu_hive_info *hive = NULL;
4325
4326         amdgpu_amdkfd_pre_reset(adev);
4327
4328         if (from_hypervisor)
4329                 r = amdgpu_virt_request_full_gpu(adev, true);
4330         else
4331                 r = amdgpu_virt_reset_gpu(adev);
4332         if (r)
4333                 return r;
4334
4335         /* Resume IP prior to SMC */
4336         r = amdgpu_device_ip_reinit_early_sriov(adev);
4337         if (r)
4338                 goto error;
4339
4340         amdgpu_virt_init_data_exchange(adev);
4341         /* we need recover gart prior to run SMC/CP/SDMA resume */
4342         amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));
4343
4344         r = amdgpu_device_fw_loading(adev);
4345         if (r)
4346                 return r;
4347
4348         /* now we are okay to resume SMC/CP/SDMA */
4349         r = amdgpu_device_ip_reinit_late_sriov(adev);
4350         if (r)
4351                 goto error;
4352
4353         hive = amdgpu_get_xgmi_hive(adev);
4354         /* Update PSP FW topology after reset */
4355         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4356                 r = amdgpu_xgmi_update_topology(hive, adev);
4357
4358         if (hive)
4359                 amdgpu_put_xgmi_hive(hive);
4360
4361         if (!r) {
4362                 amdgpu_irq_gpu_reset_resume_helper(adev);
4363                 r = amdgpu_ib_ring_tests(adev);
4364                 amdgpu_amdkfd_post_reset(adev);
4365         }
4366
4367 error:
4368         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4369                 amdgpu_inc_vram_lost(adev);
4370                 r = amdgpu_device_recover_vram(adev);
4371         }
4372         amdgpu_virt_release_full_gpu(adev, true);
4373
4374         return r;
4375 }
4376
4377 /**
4378  * amdgpu_device_has_job_running - check if there is any job in mirror list
4379  *
4380  * @adev: amdgpu_device pointer
4381  *
4382  * check if there is any job in mirror list
4383  */
4384 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4385 {
4386         int i;
4387         struct drm_sched_job *job;
4388
4389         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4390                 struct amdgpu_ring *ring = adev->rings[i];
4391
4392                 if (!ring || !ring->sched.thread)
4393                         continue;
4394
4395                 spin_lock(&ring->sched.job_list_lock);
4396                 job = list_first_entry_or_null(&ring->sched.pending_list,
4397                                                struct drm_sched_job, list);
4398                 spin_unlock(&ring->sched.job_list_lock);
4399                 if (job)
4400                         return true;
4401         }
4402         return false;
4403 }
4404
4405 /**
4406  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4407  *
4408  * @adev: amdgpu_device pointer
4409  *
4410  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4411  * a hung GPU.
4412  */
4413 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4414 {
4415         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4416                 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4417                 return false;
4418         }
4419
4420         if (amdgpu_gpu_recovery == 0)
4421                 goto disabled;
4422
4423         if (amdgpu_sriov_vf(adev))
4424                 return true;
4425
4426         if (amdgpu_gpu_recovery == -1) {
4427                 switch (adev->asic_type) {
4428                 case CHIP_BONAIRE:
4429                 case CHIP_HAWAII:
4430                 case CHIP_TOPAZ:
4431                 case CHIP_TONGA:
4432                 case CHIP_FIJI:
4433                 case CHIP_POLARIS10:
4434                 case CHIP_POLARIS11:
4435                 case CHIP_POLARIS12:
4436                 case CHIP_VEGAM:
4437                 case CHIP_VEGA20:
4438                 case CHIP_VEGA10:
4439                 case CHIP_VEGA12:
4440                 case CHIP_RAVEN:
4441                 case CHIP_ARCTURUS:
4442                 case CHIP_RENOIR:
4443                 case CHIP_NAVI10:
4444                 case CHIP_NAVI14:
4445                 case CHIP_NAVI12:
4446                 case CHIP_SIENNA_CICHLID:
4447                 case CHIP_NAVY_FLOUNDER:
4448                 case CHIP_DIMGREY_CAVEFISH:
4449                 case CHIP_BEIGE_GOBY:
4450                 case CHIP_VANGOGH:
4451                 case CHIP_ALDEBARAN:
4452                         break;
4453                 default:
4454                         goto disabled;
4455                 }
4456         }
4457
4458         return true;
4459
4460 disabled:
4461                 dev_info(adev->dev, "GPU recovery disabled.\n");
4462                 return false;
4463 }
4464
4465 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4466 {
4467         u32 i;
4468         int ret = 0;
4469
4470         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4471
4472         dev_info(adev->dev, "GPU mode1 reset\n");
4473
4474         /* disable BM */
4475         pci_clear_master(adev->pdev);
4476
4477         amdgpu_device_cache_pci_state(adev->pdev);
4478
4479         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4480                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4481                 ret = amdgpu_dpm_mode1_reset(adev);
4482         } else {
4483                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4484                 ret = psp_gpu_reset(adev);
4485         }
4486
4487         if (ret)
4488                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4489
4490         amdgpu_device_load_pci_state(adev->pdev);
4491
4492         /* wait for asic to come out of reset */
4493         for (i = 0; i < adev->usec_timeout; i++) {
4494                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4495
4496                 if (memsize != 0xffffffff)
4497                         break;
4498                 udelay(1);
4499         }
4500
4501         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4502         return ret;
4503 }
4504
4505 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4506                                  struct amdgpu_reset_context *reset_context)
4507 {
4508         int i, j, r = 0;
4509         struct amdgpu_job *job = NULL;
4510         bool need_full_reset =
4511                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4512
4513         if (reset_context->reset_req_dev == adev)
4514                 job = reset_context->job;
4515
4516         if (amdgpu_sriov_vf(adev)) {
4517                 /* stop the data exchange thread */
4518                 amdgpu_virt_fini_data_exchange(adev);
4519         }
4520
4521         /* block all schedulers and reset given job's ring */
4522         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4523                 struct amdgpu_ring *ring = adev->rings[i];
4524
4525                 if (!ring || !ring->sched.thread)
4526                         continue;
4527
4528                 /*clear job fence from fence drv to avoid force_completion
4529                  *leave NULL and vm flush fence in fence drv */
4530                 for (j = 0; j <= ring->fence_drv.num_fences_mask; j++) {
4531                         struct dma_fence *old, **ptr;
4532
4533                         ptr = &ring->fence_drv.fences[j];
4534                         old = rcu_dereference_protected(*ptr, 1);
4535                         if (old && test_bit(AMDGPU_FENCE_FLAG_EMBED_IN_JOB_BIT, &old->flags)) {
4536                                 RCU_INIT_POINTER(*ptr, NULL);
4537                         }
4538                 }
4539                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4540                 amdgpu_fence_driver_force_completion(ring);
4541         }
4542
4543         if (job && job->vm)
4544                 drm_sched_increase_karma(&job->base);
4545
4546         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4547         /* If reset handler not implemented, continue; otherwise return */
4548         if (r == -ENOSYS)
4549                 r = 0;
4550         else
4551                 return r;
4552
4553         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4554         if (!amdgpu_sriov_vf(adev)) {
4555
4556                 if (!need_full_reset)
4557                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4558
4559                 if (!need_full_reset) {
4560                         amdgpu_device_ip_pre_soft_reset(adev);
4561                         r = amdgpu_device_ip_soft_reset(adev);
4562                         amdgpu_device_ip_post_soft_reset(adev);
4563                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4564                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4565                                 need_full_reset = true;
4566                         }
4567                 }
4568
4569                 if (need_full_reset)
4570                         r = amdgpu_device_ip_suspend(adev);
4571                 if (need_full_reset)
4572                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4573                 else
4574                         clear_bit(AMDGPU_NEED_FULL_RESET,
4575                                   &reset_context->flags);
4576         }
4577
4578         return r;
4579 }
4580
4581 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4582                          struct amdgpu_reset_context *reset_context)
4583 {
4584         struct amdgpu_device *tmp_adev = NULL;
4585         bool need_full_reset, skip_hw_reset, vram_lost = false;
4586         int r = 0;
4587
4588         /* Try reset handler method first */
4589         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4590                                     reset_list);
4591         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4592         /* If reset handler not implemented, continue; otherwise return */
4593         if (r == -ENOSYS)
4594                 r = 0;
4595         else
4596                 return r;
4597
4598         /* Reset handler not implemented, use the default method */
4599         need_full_reset =
4600                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4601         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4602
4603         /*
4604          * ASIC reset has to be done on all XGMI hive nodes ASAP
4605          * to allow proper links negotiation in FW (within 1 sec)
4606          */
4607         if (!skip_hw_reset && need_full_reset) {
4608                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4609                         /* For XGMI run all resets in parallel to speed up the process */
4610                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4611                                 tmp_adev->gmc.xgmi.pending_reset = false;
4612                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4613                                         r = -EALREADY;
4614                         } else
4615                                 r = amdgpu_asic_reset(tmp_adev);
4616
4617                         if (r) {
4618                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4619                                          r, adev_to_drm(tmp_adev)->unique);
4620                                 break;
4621                         }
4622                 }
4623
4624                 /* For XGMI wait for all resets to complete before proceed */
4625                 if (!r) {
4626                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4627                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4628                                         flush_work(&tmp_adev->xgmi_reset_work);
4629                                         r = tmp_adev->asic_reset_res;
4630                                         if (r)
4631                                                 break;
4632                                 }
4633                         }
4634                 }
4635         }
4636
4637         if (!r && amdgpu_ras_intr_triggered()) {
4638                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4639                         if (tmp_adev->mmhub.ras_funcs &&
4640                             tmp_adev->mmhub.ras_funcs->reset_ras_error_count)
4641                                 tmp_adev->mmhub.ras_funcs->reset_ras_error_count(tmp_adev);
4642                 }
4643
4644                 amdgpu_ras_intr_cleared();
4645         }
4646
4647         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4648                 if (need_full_reset) {
4649                         /* post card */
4650                         r = amdgpu_device_asic_init(tmp_adev);
4651                         if (r) {
4652                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4653                         } else {
4654                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4655                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4656                                 if (r)
4657                                         goto out;
4658
4659                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4660                                 if (r)
4661                                         goto out;
4662
4663                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4664                                 if (vram_lost) {
4665                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4666                                         amdgpu_inc_vram_lost(tmp_adev);
4667                                 }
4668
4669                                 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));
4670                                 if (r)
4671                                         goto out;
4672
4673                                 r = amdgpu_device_fw_loading(tmp_adev);
4674                                 if (r)
4675                                         return r;
4676
4677                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4678                                 if (r)
4679                                         goto out;
4680
4681                                 if (vram_lost)
4682                                         amdgpu_device_fill_reset_magic(tmp_adev);
4683
4684                                 /*
4685                                  * Add this ASIC as tracked as reset was already
4686                                  * complete successfully.
4687                                  */
4688                                 amdgpu_register_gpu_instance(tmp_adev);
4689
4690                                 if (!reset_context->hive &&
4691                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4692                                         amdgpu_xgmi_add_device(tmp_adev);
4693
4694                                 r = amdgpu_device_ip_late_init(tmp_adev);
4695                                 if (r)
4696                                         goto out;
4697
4698                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4699
4700                                 /*
4701                                  * The GPU enters bad state once faulty pages
4702                                  * by ECC has reached the threshold, and ras
4703                                  * recovery is scheduled next. So add one check
4704                                  * here to break recovery if it indeed exceeds
4705                                  * bad page threshold, and remind user to
4706                                  * retire this GPU or setting one bigger
4707                                  * bad_page_threshold value to fix this once
4708                                  * probing driver again.
4709                                  */
4710                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4711                                         /* must succeed. */
4712                                         amdgpu_ras_resume(tmp_adev);
4713                                 } else {
4714                                         r = -EINVAL;
4715                                         goto out;
4716                                 }
4717
4718                                 /* Update PSP FW topology after reset */
4719                                 if (reset_context->hive &&
4720                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4721                                         r = amdgpu_xgmi_update_topology(
4722                                                 reset_context->hive, tmp_adev);
4723                         }
4724                 }
4725
4726 out:
4727                 if (!r) {
4728                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4729                         r = amdgpu_ib_ring_tests(tmp_adev);
4730                         if (r) {
4731                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4732                                 need_full_reset = true;
4733                                 r = -EAGAIN;
4734                                 goto end;
4735                         }
4736                 }
4737
4738                 if (!r)
4739                         r = amdgpu_device_recover_vram(tmp_adev);
4740                 else
4741                         tmp_adev->asic_reset_res = r;
4742         }
4743
4744 end:
4745         if (need_full_reset)
4746                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4747         else
4748                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4749         return r;
4750 }
4751
4752 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4753                                 struct amdgpu_hive_info *hive)
4754 {
4755         if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4756                 return false;
4757
4758         if (hive) {
4759                 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4760         } else {
4761                 down_write(&adev->reset_sem);
4762         }
4763
4764         switch (amdgpu_asic_reset_method(adev)) {
4765         case AMD_RESET_METHOD_MODE1:
4766                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4767                 break;
4768         case AMD_RESET_METHOD_MODE2:
4769                 adev->mp1_state = PP_MP1_STATE_RESET;
4770                 break;
4771         default:
4772                 adev->mp1_state = PP_MP1_STATE_NONE;
4773                 break;
4774         }
4775
4776         return true;
4777 }
4778
4779 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4780 {
4781         amdgpu_vf_error_trans_all(adev);
4782         adev->mp1_state = PP_MP1_STATE_NONE;
4783         atomic_set(&adev->in_gpu_reset, 0);
4784         up_write(&adev->reset_sem);
4785 }
4786
4787 /*
4788  * to lockup a list of amdgpu devices in a hive safely, if not a hive
4789  * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4790  *
4791  * unlock won't require roll back.
4792  */
4793 static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4794 {
4795         struct amdgpu_device *tmp_adev = NULL;
4796
4797         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
4798                 if (!hive) {
4799                         dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4800                         return -ENODEV;
4801                 }
4802                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4803                         if (!amdgpu_device_lock_adev(tmp_adev, hive))
4804                                 goto roll_back;
4805                 }
4806         } else if (!amdgpu_device_lock_adev(adev, hive))
4807                 return -EAGAIN;
4808
4809         return 0;
4810 roll_back:
4811         if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4812                 /*
4813                  * if the lockup iteration break in the middle of a hive,
4814                  * it may means there may has a race issue,
4815                  * or a hive device locked up independently.
4816                  * we may be in trouble and may not, so will try to roll back
4817                  * the lock and give out a warnning.
4818                  */
4819                 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4820                 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4821                         amdgpu_device_unlock_adev(tmp_adev);
4822                 }
4823         }
4824         return -EAGAIN;
4825 }
4826
4827 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4828 {
4829         struct pci_dev *p = NULL;
4830
4831         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4832                         adev->pdev->bus->number, 1);
4833         if (p) {
4834                 pm_runtime_enable(&(p->dev));
4835                 pm_runtime_resume(&(p->dev));
4836         }
4837 }
4838
4839 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4840 {
4841         enum amd_reset_method reset_method;
4842         struct pci_dev *p = NULL;
4843         u64 expires;
4844
4845         /*
4846          * For now, only BACO and mode1 reset are confirmed
4847          * to suffer the audio issue without proper suspended.
4848          */
4849         reset_method = amdgpu_asic_reset_method(adev);
4850         if ((reset_method != AMD_RESET_METHOD_BACO) &&
4851              (reset_method != AMD_RESET_METHOD_MODE1))
4852                 return -EINVAL;
4853
4854         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4855                         adev->pdev->bus->number, 1);
4856         if (!p)
4857                 return -ENODEV;
4858
4859         expires = pm_runtime_autosuspend_expiration(&(p->dev));
4860         if (!expires)
4861                 /*
4862                  * If we cannot get the audio device autosuspend delay,
4863                  * a fixed 4S interval will be used. Considering 3S is
4864                  * the audio controller default autosuspend delay setting.
4865                  * 4S used here is guaranteed to cover that.
4866                  */
4867                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4868
4869         while (!pm_runtime_status_suspended(&(p->dev))) {
4870                 if (!pm_runtime_suspend(&(p->dev)))
4871                         break;
4872
4873                 if (expires < ktime_get_mono_fast_ns()) {
4874                         dev_warn(adev->dev, "failed to suspend display audio\n");
4875                         /* TODO: abort the succeeding gpu reset? */
4876                         return -ETIMEDOUT;
4877                 }
4878         }
4879
4880         pm_runtime_disable(&(p->dev));
4881
4882         return 0;
4883 }
4884
4885 static void amdgpu_device_recheck_guilty_jobs(
4886         struct amdgpu_device *adev, struct list_head *device_list_handle,
4887         struct amdgpu_reset_context *reset_context)
4888 {
4889         int i, r = 0;
4890
4891         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4892                 struct amdgpu_ring *ring = adev->rings[i];
4893                 int ret = 0;
4894                 struct drm_sched_job *s_job;
4895
4896                 if (!ring || !ring->sched.thread)
4897                         continue;
4898
4899                 s_job = list_first_entry_or_null(&ring->sched.pending_list,
4900                                 struct drm_sched_job, list);
4901                 if (s_job == NULL)
4902                         continue;
4903
4904                 /* clear job's guilty and depend the folowing step to decide the real one */
4905                 drm_sched_reset_karma(s_job);
4906                 /* for the real bad job, it will be resubmitted twice, adding a dma_fence_get
4907                  * to make sure fence is balanced */
4908                 dma_fence_get(s_job->s_fence->parent);
4909                 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
4910
4911                 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
4912                 if (ret == 0) { /* timeout */
4913                         DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
4914                                                 ring->sched.name, s_job->id);
4915
4916                         /* set guilty */
4917                         drm_sched_increase_karma(s_job);
4918 retry:
4919                         /* do hw reset */
4920                         if (amdgpu_sriov_vf(adev)) {
4921                                 amdgpu_virt_fini_data_exchange(adev);
4922                                 r = amdgpu_device_reset_sriov(adev, false);
4923                                 if (r)
4924                                         adev->asic_reset_res = r;
4925                         } else {
4926                                 clear_bit(AMDGPU_SKIP_HW_RESET,
4927                                           &reset_context->flags);
4928                                 r = amdgpu_do_asic_reset(device_list_handle,
4929                                                          reset_context);
4930                                 if (r && r == -EAGAIN)
4931                                         goto retry;
4932                         }
4933
4934                         /*
4935                          * add reset counter so that the following
4936                          * resubmitted job could flush vmid
4937                          */
4938                         atomic_inc(&adev->gpu_reset_counter);
4939                         continue;
4940                 }
4941
4942                 /* got the hw fence, signal finished fence */
4943                 atomic_dec(ring->sched.score);
4944                 dma_fence_put(s_job->s_fence->parent);
4945                 dma_fence_get(&s_job->s_fence->finished);
4946                 dma_fence_signal(&s_job->s_fence->finished);
4947                 dma_fence_put(&s_job->s_fence->finished);
4948
4949                 /* remove node from list and free the job */
4950                 spin_lock(&ring->sched.job_list_lock);
4951                 list_del_init(&s_job->list);
4952                 spin_unlock(&ring->sched.job_list_lock);
4953                 ring->sched.ops->free_job(s_job);
4954         }
4955 }
4956
4957 /**
4958  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4959  *
4960  * @adev: amdgpu_device pointer
4961  * @job: which job trigger hang
4962  *
4963  * Attempt to reset the GPU if it has hung (all asics).
4964  * Attempt to do soft-reset or full-reset and reinitialize Asic
4965  * Returns 0 for success or an error on failure.
4966  */
4967
4968 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4969                               struct amdgpu_job *job)
4970 {
4971         struct list_head device_list, *device_list_handle =  NULL;
4972         bool job_signaled = false;
4973         struct amdgpu_hive_info *hive = NULL;
4974         struct amdgpu_device *tmp_adev = NULL;
4975         int i, r = 0;
4976         bool need_emergency_restart = false;
4977         bool audio_suspended = false;
4978         int tmp_vram_lost_counter;
4979         struct amdgpu_reset_context reset_context;
4980
4981         memset(&reset_context, 0, sizeof(reset_context));
4982
4983         /*
4984          * Special case: RAS triggered and full reset isn't supported
4985          */
4986         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4987
4988         /*
4989          * Flush RAM to disk so that after reboot
4990          * the user can read log and see why the system rebooted.
4991          */
4992         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4993                 DRM_WARN("Emergency reboot.");
4994
4995                 ksys_sync_helper();
4996                 emergency_restart();
4997         }
4998
4999         dev_info(adev->dev, "GPU %s begin!\n",
5000                 need_emergency_restart ? "jobs stop":"reset");
5001
5002         /*
5003          * Here we trylock to avoid chain of resets executing from
5004          * either trigger by jobs on different adevs in XGMI hive or jobs on
5005          * different schedulers for same device while this TO handler is running.
5006          * We always reset all schedulers for device and all devices for XGMI
5007          * hive so that should take care of them too.
5008          */
5009         if (!amdgpu_sriov_vf(adev))
5010                 hive = amdgpu_get_xgmi_hive(adev);
5011         if (hive) {
5012                 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
5013                         DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
5014                                 job ? job->base.id : -1, hive->hive_id);
5015                         amdgpu_put_xgmi_hive(hive);
5016                         if (job && job->vm)
5017                                 drm_sched_increase_karma(&job->base);
5018                         return 0;
5019                 }
5020                 mutex_lock(&hive->hive_lock);
5021         }
5022
5023         reset_context.method = AMD_RESET_METHOD_NONE;
5024         reset_context.reset_req_dev = adev;
5025         reset_context.job = job;
5026         reset_context.hive = hive;
5027         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5028
5029         /*
5030          * lock the device before we try to operate the linked list
5031          * if didn't get the device lock, don't touch the linked list since
5032          * others may iterating it.
5033          */
5034         r = amdgpu_device_lock_hive_adev(adev, hive);
5035         if (r) {
5036                 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
5037                                         job ? job->base.id : -1);
5038
5039                 /* even we skipped this reset, still need to set the job to guilty */
5040                 if (job && job->vm)
5041                         drm_sched_increase_karma(&job->base);
5042                 goto skip_recovery;
5043         }
5044
5045         /*
5046          * Build list of devices to reset.
5047          * In case we are in XGMI hive mode, resort the device list
5048          * to put adev in the 1st position.
5049          */
5050         INIT_LIST_HEAD(&device_list);
5051         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5052                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
5053                         list_add_tail(&tmp_adev->reset_list, &device_list);
5054                 if (!list_is_first(&adev->reset_list, &device_list))
5055                         list_rotate_to_front(&adev->reset_list, &device_list);
5056                 device_list_handle = &device_list;
5057         } else {
5058                 list_add_tail(&adev->reset_list, &device_list);
5059                 device_list_handle = &device_list;
5060         }
5061
5062         /* block all schedulers and reset given job's ring */
5063         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5064                 /*
5065                  * Try to put the audio codec into suspend state
5066                  * before gpu reset started.
5067                  *
5068                  * Due to the power domain of the graphics device
5069                  * is shared with AZ power domain. Without this,
5070                  * we may change the audio hardware from behind
5071                  * the audio driver's back. That will trigger
5072                  * some audio codec errors.
5073                  */
5074                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5075                         audio_suspended = true;
5076
5077                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5078
5079                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5080
5081                 if (!amdgpu_sriov_vf(tmp_adev))
5082                         amdgpu_amdkfd_pre_reset(tmp_adev);
5083
5084                 /*
5085                  * Mark these ASICs to be reseted as untracked first
5086                  * And add them back after reset completed
5087                  */
5088                 amdgpu_unregister_gpu_instance(tmp_adev);
5089
5090                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
5091
5092                 /* disable ras on ALL IPs */
5093                 if (!need_emergency_restart &&
5094                       amdgpu_device_ip_need_full_reset(tmp_adev))
5095                         amdgpu_ras_suspend(tmp_adev);
5096
5097                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5098                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5099
5100                         if (!ring || !ring->sched.thread)
5101                                 continue;
5102
5103                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5104
5105                         if (need_emergency_restart)
5106                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5107                 }
5108                 atomic_inc(&tmp_adev->gpu_reset_counter);
5109         }
5110
5111         if (need_emergency_restart)
5112                 goto skip_sched_resume;
5113
5114         /*
5115          * Must check guilty signal here since after this point all old
5116          * HW fences are force signaled.
5117          *
5118          * job->base holds a reference to parent fence
5119          */
5120         if (job && job->base.s_fence->parent &&
5121             dma_fence_is_signaled(job->base.s_fence->parent)) {
5122                 job_signaled = true;
5123                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5124                 goto skip_hw_reset;
5125         }
5126
5127 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5128         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5129                 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context);
5130                 /*TODO Should we stop ?*/
5131                 if (r) {
5132                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5133                                   r, adev_to_drm(tmp_adev)->unique);
5134                         tmp_adev->asic_reset_res = r;
5135                 }
5136         }
5137
5138         tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
5139         /* Actual ASIC resets if needed.*/
5140         /* Host driver will handle XGMI hive reset for SRIOV */
5141         if (amdgpu_sriov_vf(adev)) {
5142                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5143                 if (r)
5144                         adev->asic_reset_res = r;
5145         } else {
5146                 r = amdgpu_do_asic_reset(device_list_handle, &reset_context);
5147                 if (r && r == -EAGAIN)
5148                         goto retry;
5149         }
5150
5151 skip_hw_reset:
5152
5153         /* Post ASIC reset for all devs .*/
5154         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5155
5156                 /*
5157                  * Sometimes a later bad compute job can block a good gfx job as gfx
5158                  * and compute ring share internal GC HW mutually. We add an additional
5159                  * guilty jobs recheck step to find the real guilty job, it synchronously
5160                  * submits and pends for the first job being signaled. If it gets timeout,
5161                  * we identify it as a real guilty job.
5162                  */
5163                 if (amdgpu_gpu_recovery == 2 &&
5164                         !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5165                         amdgpu_device_recheck_guilty_jobs(
5166                                 tmp_adev, device_list_handle, &reset_context);
5167
5168                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5169                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5170
5171                         if (!ring || !ring->sched.thread)
5172                                 continue;
5173
5174                         /* No point to resubmit jobs if we didn't HW reset*/
5175                         if (!tmp_adev->asic_reset_res && !job_signaled)
5176                                 drm_sched_resubmit_jobs(&ring->sched);
5177
5178                         drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
5179                 }
5180
5181                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5182                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5183                 }
5184
5185                 tmp_adev->asic_reset_res = 0;
5186
5187                 if (r) {
5188                         /* bad news, how to tell it to userspace ? */
5189                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5190                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5191                 } else {
5192                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5193                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5194                                 DRM_WARN("smart shift update failed\n");
5195                 }
5196         }
5197
5198 skip_sched_resume:
5199         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5200                 /* unlock kfd: SRIOV would do it separately */
5201                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5202                         amdgpu_amdkfd_post_reset(tmp_adev);
5203
5204                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5205                  * need to bring up kfd here if it's not be initialized before
5206                  */
5207                 if (!adev->kfd.init_complete)
5208                         amdgpu_amdkfd_device_init(adev);
5209
5210                 if (audio_suspended)
5211                         amdgpu_device_resume_display_audio(tmp_adev);
5212                 amdgpu_device_unlock_adev(tmp_adev);
5213         }
5214
5215 skip_recovery:
5216         if (hive) {
5217                 atomic_set(&hive->in_reset, 0);
5218                 mutex_unlock(&hive->hive_lock);
5219                 amdgpu_put_xgmi_hive(hive);
5220         }
5221
5222         if (r && r != -EAGAIN)
5223                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5224         return r;
5225 }
5226
5227 /**
5228  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5229  *
5230  * @adev: amdgpu_device pointer
5231  *
5232  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5233  * and lanes) of the slot the device is in. Handles APUs and
5234  * virtualized environments where PCIE config space may not be available.
5235  */
5236 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5237 {
5238         struct pci_dev *pdev;
5239         enum pci_bus_speed speed_cap, platform_speed_cap;
5240         enum pcie_link_width platform_link_width;
5241
5242         if (amdgpu_pcie_gen_cap)
5243                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5244
5245         if (amdgpu_pcie_lane_cap)
5246                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5247
5248         /* covers APUs as well */
5249         if (pci_is_root_bus(adev->pdev->bus)) {
5250                 if (adev->pm.pcie_gen_mask == 0)
5251                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5252                 if (adev->pm.pcie_mlw_mask == 0)
5253                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5254                 return;
5255         }
5256
5257         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5258                 return;
5259
5260         pcie_bandwidth_available(adev->pdev, NULL,
5261                                  &platform_speed_cap, &platform_link_width);
5262
5263         if (adev->pm.pcie_gen_mask == 0) {
5264                 /* asic caps */
5265                 pdev = adev->pdev;
5266                 speed_cap = pcie_get_speed_cap(pdev);
5267                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5268                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5269                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5270                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5271                 } else {
5272                         if (speed_cap == PCIE_SPEED_32_0GT)
5273                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5274                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5275                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5276                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5277                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5278                         else if (speed_cap == PCIE_SPEED_16_0GT)
5279                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5280                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5281                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5282                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5283                         else if (speed_cap == PCIE_SPEED_8_0GT)
5284                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5285                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5286                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5287                         else if (speed_cap == PCIE_SPEED_5_0GT)
5288                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5289                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5290                         else
5291                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5292                 }
5293                 /* platform caps */
5294                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5295                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5296                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5297                 } else {
5298                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5299                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5300                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5301                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5302                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5303                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5304                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5305                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5306                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5307                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5308                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5309                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5310                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5311                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5312                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5313                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5314                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5315                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5316                         else
5317                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5318
5319                 }
5320         }
5321         if (adev->pm.pcie_mlw_mask == 0) {
5322                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5323                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5324                 } else {
5325                         switch (platform_link_width) {
5326                         case PCIE_LNK_X32:
5327                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5328                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5329                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5330                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5331                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5332                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5333                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5334                                 break;
5335                         case PCIE_LNK_X16:
5336                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5337                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5338                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5339                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5340                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5341                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5342                                 break;
5343                         case PCIE_LNK_X12:
5344                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5345                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5346                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5347                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5348                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5349                                 break;
5350                         case PCIE_LNK_X8:
5351                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5352                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5353                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5354                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5355                                 break;
5356                         case PCIE_LNK_X4:
5357                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5358                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5359                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5360                                 break;
5361                         case PCIE_LNK_X2:
5362                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5363                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5364                                 break;
5365                         case PCIE_LNK_X1:
5366                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5367                                 break;
5368                         default:
5369                                 break;
5370                         }
5371                 }
5372         }
5373 }
5374
5375 int amdgpu_device_baco_enter(struct drm_device *dev)
5376 {
5377         struct amdgpu_device *adev = drm_to_adev(dev);
5378         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5379
5380         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5381                 return -ENOTSUPP;
5382
5383         if (ras && adev->ras_enabled &&
5384             adev->nbio.funcs->enable_doorbell_interrupt)
5385                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5386
5387         return amdgpu_dpm_baco_enter(adev);
5388 }
5389
5390 int amdgpu_device_baco_exit(struct drm_device *dev)
5391 {
5392         struct amdgpu_device *adev = drm_to_adev(dev);
5393         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5394         int ret = 0;
5395
5396         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5397                 return -ENOTSUPP;
5398
5399         ret = amdgpu_dpm_baco_exit(adev);
5400         if (ret)
5401                 return ret;
5402
5403         if (ras && adev->ras_enabled &&
5404             adev->nbio.funcs->enable_doorbell_interrupt)
5405                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5406
5407         if (amdgpu_passthrough(adev) &&
5408             adev->nbio.funcs->clear_doorbell_interrupt)
5409                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5410
5411         return 0;
5412 }
5413
5414 static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5415 {
5416         int i;
5417
5418         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5419                 struct amdgpu_ring *ring = adev->rings[i];
5420
5421                 if (!ring || !ring->sched.thread)
5422                         continue;
5423
5424                 cancel_delayed_work_sync(&ring->sched.work_tdr);
5425         }
5426 }
5427
5428 /**
5429  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5430  * @pdev: PCI device struct
5431  * @state: PCI channel state
5432  *
5433  * Description: Called when a PCI error is detected.
5434  *
5435  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5436  */
5437 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5438 {
5439         struct drm_device *dev = pci_get_drvdata(pdev);
5440         struct amdgpu_device *adev = drm_to_adev(dev);
5441         int i;
5442
5443         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5444
5445         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5446                 DRM_WARN("No support for XGMI hive yet...");
5447                 return PCI_ERS_RESULT_DISCONNECT;
5448         }
5449
5450         adev->pci_channel_state = state;
5451
5452         switch (state) {
5453         case pci_channel_io_normal:
5454                 return PCI_ERS_RESULT_CAN_RECOVER;
5455         /* Fatal error, prepare for slot reset */
5456         case pci_channel_io_frozen:
5457                 /*
5458                  * Cancel and wait for all TDRs in progress if failing to
5459                  * set  adev->in_gpu_reset in amdgpu_device_lock_adev
5460                  *
5461                  * Locking adev->reset_sem will prevent any external access
5462                  * to GPU during PCI error recovery
5463                  */
5464                 while (!amdgpu_device_lock_adev(adev, NULL))
5465                         amdgpu_cancel_all_tdr(adev);
5466
5467                 /*
5468                  * Block any work scheduling as we do for regular GPU reset
5469                  * for the duration of the recovery
5470                  */
5471                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5472                         struct amdgpu_ring *ring = adev->rings[i];
5473
5474                         if (!ring || !ring->sched.thread)
5475                                 continue;
5476
5477                         drm_sched_stop(&ring->sched, NULL);
5478                 }
5479                 atomic_inc(&adev->gpu_reset_counter);
5480                 return PCI_ERS_RESULT_NEED_RESET;
5481         case pci_channel_io_perm_failure:
5482                 /* Permanent error, prepare for device removal */
5483                 return PCI_ERS_RESULT_DISCONNECT;
5484         }
5485
5486         return PCI_ERS_RESULT_NEED_RESET;
5487 }
5488
5489 /**
5490  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5491  * @pdev: pointer to PCI device
5492  */
5493 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5494 {
5495
5496         DRM_INFO("PCI error: mmio enabled callback!!\n");
5497
5498         /* TODO - dump whatever for debugging purposes */
5499
5500         /* This called only if amdgpu_pci_error_detected returns
5501          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5502          * works, no need to reset slot.
5503          */
5504
5505         return PCI_ERS_RESULT_RECOVERED;
5506 }
5507
5508 /**
5509  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5510  * @pdev: PCI device struct
5511  *
5512  * Description: This routine is called by the pci error recovery
5513  * code after the PCI slot has been reset, just before we
5514  * should resume normal operations.
5515  */
5516 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5517 {
5518         struct drm_device *dev = pci_get_drvdata(pdev);
5519         struct amdgpu_device *adev = drm_to_adev(dev);
5520         int r, i;
5521         struct amdgpu_reset_context reset_context;
5522         u32 memsize;
5523         struct list_head device_list;
5524
5525         DRM_INFO("PCI error: slot reset callback!!\n");
5526
5527         memset(&reset_context, 0, sizeof(reset_context));
5528
5529         INIT_LIST_HEAD(&device_list);
5530         list_add_tail(&adev->reset_list, &device_list);
5531
5532         /* wait for asic to come out of reset */
5533         msleep(500);
5534
5535         /* Restore PCI confspace */
5536         amdgpu_device_load_pci_state(pdev);
5537
5538         /* confirm  ASIC came out of reset */
5539         for (i = 0; i < adev->usec_timeout; i++) {
5540                 memsize = amdgpu_asic_get_config_memsize(adev);
5541
5542                 if (memsize != 0xffffffff)
5543                         break;
5544                 udelay(1);
5545         }
5546         if (memsize == 0xffffffff) {
5547                 r = -ETIME;
5548                 goto out;
5549         }
5550
5551         reset_context.method = AMD_RESET_METHOD_NONE;
5552         reset_context.reset_req_dev = adev;
5553         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5554         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5555
5556         adev->no_hw_access = true;
5557         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5558         adev->no_hw_access = false;
5559         if (r)
5560                 goto out;
5561
5562         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5563
5564 out:
5565         if (!r) {
5566                 if (amdgpu_device_cache_pci_state(adev->pdev))
5567                         pci_restore_state(adev->pdev);
5568
5569                 DRM_INFO("PCIe error recovery succeeded\n");
5570         } else {
5571                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5572                 amdgpu_device_unlock_adev(adev);
5573         }
5574
5575         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5576 }
5577
5578 /**
5579  * amdgpu_pci_resume() - resume normal ops after PCI reset
5580  * @pdev: pointer to PCI device
5581  *
5582  * Called when the error recovery driver tells us that its
5583  * OK to resume normal operation.
5584  */
5585 void amdgpu_pci_resume(struct pci_dev *pdev)
5586 {
5587         struct drm_device *dev = pci_get_drvdata(pdev);
5588         struct amdgpu_device *adev = drm_to_adev(dev);
5589         int i;
5590
5591
5592         DRM_INFO("PCI error: resume callback!!\n");
5593
5594         /* Only continue execution for the case of pci_channel_io_frozen */
5595         if (adev->pci_channel_state != pci_channel_io_frozen)
5596                 return;
5597
5598         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5599                 struct amdgpu_ring *ring = adev->rings[i];
5600
5601                 if (!ring || !ring->sched.thread)
5602                         continue;
5603
5604
5605                 drm_sched_resubmit_jobs(&ring->sched);
5606                 drm_sched_start(&ring->sched, true);
5607         }
5608
5609         amdgpu_device_unlock_adev(adev);
5610 }
5611
5612 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5613 {
5614         struct drm_device *dev = pci_get_drvdata(pdev);
5615         struct amdgpu_device *adev = drm_to_adev(dev);
5616         int r;
5617
5618         r = pci_save_state(pdev);
5619         if (!r) {
5620                 kfree(adev->pci_state);
5621
5622                 adev->pci_state = pci_store_saved_state(pdev);
5623
5624                 if (!adev->pci_state) {
5625                         DRM_ERROR("Failed to store PCI saved state");
5626                         return false;
5627                 }
5628         } else {
5629                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5630                 return false;
5631         }
5632
5633         return true;
5634 }
5635
5636 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5637 {
5638         struct drm_device *dev = pci_get_drvdata(pdev);
5639         struct amdgpu_device *adev = drm_to_adev(dev);
5640         int r;
5641
5642         if (!adev->pci_state)
5643                 return false;
5644
5645         r = pci_load_saved_state(pdev, adev->pci_state);
5646
5647         if (!r) {
5648                 pci_restore_state(pdev);
5649         } else {
5650                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5651                 return false;
5652         }
5653
5654         return true;
5655 }
5656
5657 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5658                 struct amdgpu_ring *ring)
5659 {
5660 #ifdef CONFIG_X86_64
5661         if (adev->flags & AMD_IS_APU)
5662                 return;
5663 #endif
5664         if (adev->gmc.xgmi.connected_to_cpu)
5665                 return;
5666
5667         if (ring && ring->funcs->emit_hdp_flush)
5668                 amdgpu_ring_emit_hdp_flush(ring);
5669         else
5670                 amdgpu_asic_flush_hdp(adev, ring);
5671 }
5672
5673 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5674                 struct amdgpu_ring *ring)
5675 {
5676 #ifdef CONFIG_X86_64
5677         if (adev->flags & AMD_IS_APU)
5678                 return;
5679 #endif
5680         if (adev->gmc.xgmi.connected_to_cpu)
5681                 return;
5682
5683         amdgpu_asic_invalidate_hdp(adev, ring);
5684 }
5685
5686 /**
5687  * amdgpu_device_halt() - bring hardware to some kind of halt state
5688  *
5689  * @adev: amdgpu_device pointer
5690  *
5691  * Bring hardware to some kind of halt state so that no one can touch it
5692  * any more. It will help to maintain error context when error occurred.
5693  * Compare to a simple hang, the system will keep stable at least for SSH
5694  * access. Then it should be trivial to inspect the hardware state and
5695  * see what's going on. Implemented as following:
5696  *
5697  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5698  *    clears all CPU mappings to device, disallows remappings through page faults
5699  * 2. amdgpu_irq_disable_all() disables all interrupts
5700  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5701  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5702  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5703  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5704  *    flush any in flight DMA operations
5705  */
5706 void amdgpu_device_halt(struct amdgpu_device *adev)
5707 {
5708         struct pci_dev *pdev = adev->pdev;
5709         struct drm_device *ddev = &adev->ddev;
5710
5711         drm_dev_unplug(ddev);
5712
5713         amdgpu_irq_disable_all(adev);
5714
5715         amdgpu_fence_driver_hw_fini(adev);
5716
5717         adev->no_hw_access = true;
5718
5719         amdgpu_device_unmap_mmio(adev);
5720
5721         pci_disable_device(pdev);
5722         pci_wait_for_pending_transaction(pdev);
5723 }