drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33
  34 #include <drm/drm_atomic_helper.h>
  35 #include <drm/drm_probe_helper.h>
  36 #include <drm/amdgpu_drm.h>
  37 #include <linux/vgaarb.h>
  38 #include <linux/vga_switcheroo.h>
  39 #include <linux/efi.h>
  40 #include "amdgpu.h"
  41 #include "amdgpu_trace.h"
  42 #include "amdgpu_i2c.h"
  43 #include "atom.h"
  44 #include "amdgpu_atombios.h"
  45 #include "amdgpu_atomfirmware.h"
  46 #include "amd_pcie.h"
  47 #ifdef CONFIG_DRM_AMDGPU_SI
  48 #include "si.h"
  49 #endif
  50 #ifdef CONFIG_DRM_AMDGPU_CIK
  51 #include "cik.h"
  52 #endif
  53 #include "vi.h"
  54 #include "soc15.h"
  55 #include "nv.h"
  56 #include "bif/bif_4_1_d.h"
  57 #include <linux/pci.h>
  58 #include <linux/firmware.h>
  59 #include "amdgpu_vf_error.h"
  60
  61 #include "amdgpu_amdkfd.h"
  62 #include "amdgpu_pm.h"
  63
  64 #include "amdgpu_xgmi.h"
  65 #include "amdgpu_ras.h"
  66 #include "amdgpu_pmu.h"
  67 #include "amdgpu_fru_eeprom.h"
  68 #include "amdgpu_reset.h"
  69
  70 #include <linux/suspend.h>
  71 #include <drm/task_barrier.h>
  72 #include <linux/pm_runtime.h>
  73
  74 #include <drm/drm_drv.h>
  75
  76 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  78 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  79 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  80 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  81 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
  87
  88 #define AMDGPU_RESUME_MS                2000
  89
  90 const char *amdgpu_asic_name[] = {
  91         "TAHITI",
  92         "PITCAIRN",
  93         "VERDE",
  94         "OLAND",
  95         "HAINAN",
  96         "BONAIRE",
  97         "KAVERI",
  98         "KABINI",
  99         "HAWAII",
 100         "MULLINS",
 101         "TOPAZ",
 102         "TONGA",
 103         "FIJI",
 104         "CARRIZO",
 105         "STONEY",
 106         "POLARIS10",
 107         "POLARIS11",
 108         "POLARIS12",
 109         "VEGAM",
 110         "VEGA10",
 111         "VEGA12",
 112         "VEGA20",
 113         "RAVEN",
 114         "ARCTURUS",
 115         "RENOIR",
 116         "ALDEBARAN",
 117         "NAVI10",
 118         "NAVI14",
 119         "NAVI12",
 120         "SIENNA_CICHLID",
 121         "NAVY_FLOUNDER",
 122         "VANGOGH",
 123         "DIMGREY_CAVEFISH",
 124         "LAST",
 125 };
 126
 127 /**
 128  * DOC: pcie_replay_count
 129  *
 130  * The amdgpu driver provides a sysfs API for reporting the total number
 131  * of PCIe replays (NAKs)
 132  * The file pcie_replay_count is used for this and returns the total
 133  * number of replays as a sum of the NAKs generated and NAKs received
 134  */
 135
 136 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 137                 struct device_attribute *attr, char *buf)
 138 {
 139         struct drm_device *ddev = dev_get_drvdata(dev);
 140         struct amdgpu_device *adev = drm_to_adev(ddev);
 141         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 142
 143         return sysfs_emit(buf, "%llu\n", cnt);
 144 }
 145
 146 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 147                 amdgpu_device_get_pcie_replay_count, NULL);
 148
 149 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 150
 151 /**
 152  * DOC: product_name
 153  *
 154  * The amdgpu driver provides a sysfs API for reporting the product name
 155  * for the device
 156  * The file serial_number is used for this and returns the product name
 157  * as returned from the FRU.
 158  * NOTE: This is only available for certain server cards
 159  */
 160
 161 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 162                 struct device_attribute *attr, char *buf)
 163 {
 164         struct drm_device *ddev = dev_get_drvdata(dev);
 165         struct amdgpu_device *adev = drm_to_adev(ddev);
 166
 167         return sysfs_emit(buf, "%s\n", adev->product_name);
 168 }
 169
 170 static DEVICE_ATTR(product_name, S_IRUGO,
 171                 amdgpu_device_get_product_name, NULL);
 172
 173 /**
 174  * DOC: product_number
 175  *
 176  * The amdgpu driver provides a sysfs API for reporting the part number
 177  * for the device
 178  * The file serial_number is used for this and returns the part number
 179  * as returned from the FRU.
 180  * NOTE: This is only available for certain server cards
 181  */
 182
 183 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 184                 struct device_attribute *attr, char *buf)
 185 {
 186         struct drm_device *ddev = dev_get_drvdata(dev);
 187         struct amdgpu_device *adev = drm_to_adev(ddev);
 188
 189         return sysfs_emit(buf, "%s\n", adev->product_number);
 190 }
 191
 192 static DEVICE_ATTR(product_number, S_IRUGO,
 193                 amdgpu_device_get_product_number, NULL);
 194
 195 /**
 196  * DOC: serial_number
 197  *
 198  * The amdgpu driver provides a sysfs API for reporting the serial number
 199  * for the device
 200  * The file serial_number is used for this and returns the serial number
 201  * as returned from the FRU.
 202  * NOTE: This is only available for certain server cards
 203  */
 204
 205 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 206                 struct device_attribute *attr, char *buf)
 207 {
 208         struct drm_device *ddev = dev_get_drvdata(dev);
 209         struct amdgpu_device *adev = drm_to_adev(ddev);
 210
 211         return sysfs_emit(buf, "%s\n", adev->serial);
 212 }
 213
 214 static DEVICE_ATTR(serial_number, S_IRUGO,
 215                 amdgpu_device_get_serial_number, NULL);
 216
 217 /**
 218  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 219  *
 220  * @dev: drm_device pointer
 221  *
 222  * Returns true if the device is a dGPU with ATPX power control,
 223  * otherwise return false.
 224  */
 225 bool amdgpu_device_supports_px(struct drm_device *dev)
 226 {
 227         struct amdgpu_device *adev = drm_to_adev(dev);
 228
 229         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 230                 return true;
 231         return false;
 232 }
 233
 234 /**
 235  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 236  *
 237  * @dev: drm_device pointer
 238  *
 239  * Returns true if the device is a dGPU with ACPI power control,
 240  * otherwise return false.
 241  */
 242 bool amdgpu_device_supports_boco(struct drm_device *dev)
 243 {
 244         struct amdgpu_device *adev = drm_to_adev(dev);
 245
 246         if (adev->has_pr3 ||
 247             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 248                 return true;
 249         return false;
 250 }
 251
 252 /**
 253  * amdgpu_device_supports_baco - Does the device support BACO
 254  *
 255  * @dev: drm_device pointer
 256  *
 257  * Returns true if the device supporte BACO,
 258  * otherwise return false.
 259  */
 260 bool amdgpu_device_supports_baco(struct drm_device *dev)
 261 {
 262         struct amdgpu_device *adev = drm_to_adev(dev);
 263
 264         return amdgpu_asic_supports_baco(adev);
 265 }
 266
 267 /*
 268  * VRAM access helper functions
 269  */
 270
 271 /**
 272  * amdgpu_device_vram_access - read/write a buffer in vram
 273  *
 274  * @adev: amdgpu_device pointer
 275  * @pos: offset of the buffer in vram
 276  * @buf: virtual address of the buffer in system memory
 277  * @size: read/write size, sizeof(@buf) must > @size
 278  * @write: true - write to vram, otherwise - read from vram
 279  */
 280 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 281                                uint32_t *buf, size_t size, bool write)
 282 {
 283         unsigned long flags;
 284         uint32_t hi = ~0;
 285         uint64_t last;
 286         int idx;
 287
 288         if (!drm_dev_enter(&adev->ddev, &idx))
 289                 return;
 290
 291 #ifdef CONFIG_64BIT
 292         last = min(pos + size, adev->gmc.visible_vram_size);
 293         if (last > pos) {
 294                 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
 295                 size_t count = last - pos;
 296
 297                 if (write) {
 298                         memcpy_toio(addr, buf, count);
 299                         mb();
 300                         amdgpu_asic_flush_hdp(adev, NULL);
 301                 } else {
 302                         amdgpu_asic_invalidate_hdp(adev, NULL);
 303                         mb();
 304                         memcpy_fromio(buf, addr, count);
 305                 }
 306
 307                 if (count == size)
 308                         goto exit;
 309
 310                 pos += count;
 311                 buf += count / 4;
 312                 size -= count;
 313         }
 314 #endif
 315
 316         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 317         for (last = pos + size; pos < last; pos += 4) {
 318                 uint32_t tmp = pos >> 31;
 319
 320                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 321                 if (tmp != hi) {
 322                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 323                         hi = tmp;
 324                 }
 325                 if (write)
 326                         WREG32_NO_KIQ(mmMM_DATA, *buf++);
 327                 else
 328                         *buf++ = RREG32_NO_KIQ(mmMM_DATA);
 329         }
 330         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 331
 332 exit:
 333         drm_dev_exit(idx);
 334 }
 335
 336 /*
 337  * register access helper functions.
 338  */
 339
 340 /* Check if hw access should be skipped because of hotplug or device error */
 341 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 342 {
 343         if (adev->in_pci_err_recovery)
 344                 return true;
 345
 346 #ifdef CONFIG_LOCKDEP
 347         /*
 348          * This is a bit complicated to understand, so worth a comment. What we assert
 349          * here is that the GPU reset is not running on another thread in parallel.
 350          *
 351          * For this we trylock the read side of the reset semaphore, if that succeeds
 352          * we know that the reset is not running in paralell.
 353          *
 354          * If the trylock fails we assert that we are either already holding the read
 355          * side of the lock or are the reset thread itself and hold the write side of
 356          * the lock.
 357          */
 358         if (in_task()) {
 359                 if (down_read_trylock(&adev->reset_sem))
 360                         up_read(&adev->reset_sem);
 361                 else
 362                         lockdep_assert_held(&adev->reset_sem);
 363         }
 364 #endif
 365         return false;
 366 }
 367
 368 /**
 369  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 370  *
 371  * @adev: amdgpu_device pointer
 372  * @reg: dword aligned register offset
 373  * @acc_flags: access flags which require special behavior
 374  *
 375  * Returns the 32 bit value from the offset specified.
 376  */
 377 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 378                             uint32_t reg, uint32_t acc_flags)
 379 {
 380         uint32_t ret;
 381
 382         if (amdgpu_device_skip_hw_access(adev))
 383                 return 0;
 384
 385         if ((reg * 4) < adev->rmmio_size) {
 386                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 387                     amdgpu_sriov_runtime(adev) &&
 388                     down_read_trylock(&adev->reset_sem)) {
 389                         ret = amdgpu_kiq_rreg(adev, reg);
 390                         up_read(&adev->reset_sem);
 391                 } else {
 392                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 393                 }
 394         } else {
 395                 ret = adev->pcie_rreg(adev, reg * 4);
 396         }
 397
 398         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 399
 400         return ret;
 401 }
 402
 403 /*
 404  * MMIO register read with bytes helper functions
 405  * @offset:bytes offset from MMIO start
 406  *
 407 */
 408
 409 /**
 410  * amdgpu_mm_rreg8 - read a memory mapped IO register
 411  *
 412  * @adev: amdgpu_device pointer
 413  * @offset: byte aligned register offset
 414  *
 415  * Returns the 8 bit value from the offset specified.
 416  */
 417 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 418 {
 419         if (amdgpu_device_skip_hw_access(adev))
 420                 return 0;
 421
 422         if (offset < adev->rmmio_size)
 423                 return (readb(adev->rmmio + offset));
 424         BUG();
 425 }
 426
 427 /*
 428  * MMIO register write with bytes helper functions
 429  * @offset:bytes offset from MMIO start
 430  * @value: the value want to be written to the register
 431  *
 432 */
 433 /**
 434  * amdgpu_mm_wreg8 - read a memory mapped IO register
 435  *
 436  * @adev: amdgpu_device pointer
 437  * @offset: byte aligned register offset
 438  * @value: 8 bit value to write
 439  *
 440  * Writes the value specified to the offset specified.
 441  */
 442 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 443 {
 444         if (amdgpu_device_skip_hw_access(adev))
 445                 return;
 446
 447         if (offset < adev->rmmio_size)
 448                 writeb(value, adev->rmmio + offset);
 449         else
 450                 BUG();
 451 }
 452
 453 /**
 454  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 455  *
 456  * @adev: amdgpu_device pointer
 457  * @reg: dword aligned register offset
 458  * @v: 32 bit value to write to the register
 459  * @acc_flags: access flags which require special behavior
 460  *
 461  * Writes the value specified to the offset specified.
 462  */
 463 void amdgpu_device_wreg(struct amdgpu_device *adev,
 464                         uint32_t reg, uint32_t v,
 465                         uint32_t acc_flags)
 466 {
 467         if (amdgpu_device_skip_hw_access(adev))
 468                 return;
 469
 470         if ((reg * 4) < adev->rmmio_size) {
 471                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 472                     amdgpu_sriov_runtime(adev) &&
 473                     down_read_trylock(&adev->reset_sem)) {
 474                         amdgpu_kiq_wreg(adev, reg, v);
 475                         up_read(&adev->reset_sem);
 476                 } else {
 477                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 478                 }
 479         } else {
 480                 adev->pcie_wreg(adev, reg * 4, v);
 481         }
 482
 483         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 484 }
 485
 486 /*
 487  * amdgpu_mm_wreg_mmio_rlc -  write register either with mmio or with RLC path if in range
 488  *
 489  * this function is invoked only the debugfs register access
 490  * */
 491 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 492                              uint32_t reg, uint32_t v)
 493 {
 494         if (amdgpu_device_skip_hw_access(adev))
 495                 return;
 496
 497         if (amdgpu_sriov_fullaccess(adev) &&
 498             adev->gfx.rlc.funcs &&
 499             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 500                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 501                         return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0);
 502         } else {
 503                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 504         }
 505 }
 506
 507 /**
 508  * amdgpu_mm_rdoorbell - read a doorbell dword
 509  *
 510  * @adev: amdgpu_device pointer
 511  * @index: doorbell index
 512  *
 513  * Returns the value in the doorbell aperture at the
 514  * requested doorbell index (CIK).
 515  */
 516 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 517 {
 518         if (amdgpu_device_skip_hw_access(adev))
 519                 return 0;
 520
 521         if (index < adev->doorbell.num_doorbells) {
 522                 return readl(adev->doorbell.ptr + index);
 523         } else {
 524                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 525                 return 0;
 526         }
 527 }
 528
 529 /**
 530  * amdgpu_mm_wdoorbell - write a doorbell dword
 531  *
 532  * @adev: amdgpu_device pointer
 533  * @index: doorbell index
 534  * @v: value to write
 535  *
 536  * Writes @v to the doorbell aperture at the
 537  * requested doorbell index (CIK).
 538  */
 539 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 540 {
 541         if (amdgpu_device_skip_hw_access(adev))
 542                 return;
 543
 544         if (index < adev->doorbell.num_doorbells) {
 545                 writel(v, adev->doorbell.ptr + index);
 546         } else {
 547                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 548         }
 549 }
 550
 551 /**
 552  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 553  *
 554  * @adev: amdgpu_device pointer
 555  * @index: doorbell index
 556  *
 557  * Returns the value in the doorbell aperture at the
 558  * requested doorbell index (VEGA10+).
 559  */
 560 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 561 {
 562         if (amdgpu_device_skip_hw_access(adev))
 563                 return 0;
 564
 565         if (index < adev->doorbell.num_doorbells) {
 566                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 567         } else {
 568                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 569                 return 0;
 570         }
 571 }
 572
 573 /**
 574  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 575  *
 576  * @adev: amdgpu_device pointer
 577  * @index: doorbell index
 578  * @v: value to write
 579  *
 580  * Writes @v to the doorbell aperture at the
 581  * requested doorbell index (VEGA10+).
 582  */
 583 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 584 {
 585         if (amdgpu_device_skip_hw_access(adev))
 586                 return;
 587
 588         if (index < adev->doorbell.num_doorbells) {
 589                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 590         } else {
 591                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 592         }
 593 }
 594
 595 /**
 596  * amdgpu_device_indirect_rreg - read an indirect register
 597  *
 598  * @adev: amdgpu_device pointer
 599  * @pcie_index: mmio register offset
 600  * @pcie_data: mmio register offset
 601  * @reg_addr: indirect register address to read from
 602  *
 603  * Returns the value of indirect register @reg_addr
 604  */
 605 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 606                                 u32 pcie_index, u32 pcie_data,
 607                                 u32 reg_addr)
 608 {
 609         unsigned long flags;
 610         u32 r;
 611         void __iomem *pcie_index_offset;
 612         void __iomem *pcie_data_offset;
 613
 614         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 615         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 616         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 617
 618         writel(reg_addr, pcie_index_offset);
 619         readl(pcie_index_offset);
 620         r = readl(pcie_data_offset);
 621         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 622
 623         return r;
 624 }
 625
 626 /**
 627  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 628  *
 629  * @adev: amdgpu_device pointer
 630  * @pcie_index: mmio register offset
 631  * @pcie_data: mmio register offset
 632  * @reg_addr: indirect register address to read from
 633  *
 634  * Returns the value of indirect register @reg_addr
 635  */
 636 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 637                                   u32 pcie_index, u32 pcie_data,
 638                                   u32 reg_addr)
 639 {
 640         unsigned long flags;
 641         u64 r;
 642         void __iomem *pcie_index_offset;
 643         void __iomem *pcie_data_offset;
 644
 645         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 646         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 647         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 648
 649         /* read low 32 bits */
 650         writel(reg_addr, pcie_index_offset);
 651         readl(pcie_index_offset);
 652         r = readl(pcie_data_offset);
 653         /* read high 32 bits */
 654         writel(reg_addr + 4, pcie_index_offset);
 655         readl(pcie_index_offset);
 656         r |= ((u64)readl(pcie_data_offset) << 32);
 657         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 658
 659         return r;
 660 }
 661
 662 /**
 663  * amdgpu_device_indirect_wreg - write an indirect register address
 664  *
 665  * @adev: amdgpu_device pointer
 666  * @pcie_index: mmio register offset
 667  * @pcie_data: mmio register offset
 668  * @reg_addr: indirect register offset
 669  * @reg_data: indirect register data
 670  *
 671  */
 672 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 673                                  u32 pcie_index, u32 pcie_data,
 674                                  u32 reg_addr, u32 reg_data)
 675 {
 676         unsigned long flags;
 677         void __iomem *pcie_index_offset;
 678         void __iomem *pcie_data_offset;
 679
 680         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 681         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 682         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 683
 684         writel(reg_addr, pcie_index_offset);
 685         readl(pcie_index_offset);
 686         writel(reg_data, pcie_data_offset);
 687         readl(pcie_data_offset);
 688         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 689 }
 690
 691 /**
 692  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 693  *
 694  * @adev: amdgpu_device pointer
 695  * @pcie_index: mmio register offset
 696  * @pcie_data: mmio register offset
 697  * @reg_addr: indirect register offset
 698  * @reg_data: indirect register data
 699  *
 700  */
 701 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 702                                    u32 pcie_index, u32 pcie_data,
 703                                    u32 reg_addr, u64 reg_data)
 704 {
 705         unsigned long flags;
 706         void __iomem *pcie_index_offset;
 707         void __iomem *pcie_data_offset;
 708
 709         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 710         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 711         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 712
 713         /* write low 32 bits */
 714         writel(reg_addr, pcie_index_offset);
 715         readl(pcie_index_offset);
 716         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 717         readl(pcie_data_offset);
 718         /* write high 32 bits */
 719         writel(reg_addr + 4, pcie_index_offset);
 720         readl(pcie_index_offset);
 721         writel((u32)(reg_data >> 32), pcie_data_offset);
 722         readl(pcie_data_offset);
 723         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 724 }
 725
 726 /**
 727  * amdgpu_invalid_rreg - dummy reg read function
 728  *
 729  * @adev: amdgpu_device pointer
 730  * @reg: offset of register
 731  *
 732  * Dummy register read function.  Used for register blocks
 733  * that certain asics don't have (all asics).
 734  * Returns the value in the register.
 735  */
 736 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 737 {
 738         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 739         BUG();
 740         return 0;
 741 }
 742
 743 /**
 744  * amdgpu_invalid_wreg - dummy reg write function
 745  *
 746  * @adev: amdgpu_device pointer
 747  * @reg: offset of register
 748  * @v: value to write to the register
 749  *
 750  * Dummy register read function.  Used for register blocks
 751  * that certain asics don't have (all asics).
 752  */
 753 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 754 {
 755         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 756                   reg, v);
 757         BUG();
 758 }
 759
 760 /**
 761  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 762  *
 763  * @adev: amdgpu_device pointer
 764  * @reg: offset of register
 765  *
 766  * Dummy register read function.  Used for register blocks
 767  * that certain asics don't have (all asics).
 768  * Returns the value in the register.
 769  */
 770 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 771 {
 772         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 773         BUG();
 774         return 0;
 775 }
 776
 777 /**
 778  * amdgpu_invalid_wreg64 - dummy reg write function
 779  *
 780  * @adev: amdgpu_device pointer
 781  * @reg: offset of register
 782  * @v: value to write to the register
 783  *
 784  * Dummy register read function.  Used for register blocks
 785  * that certain asics don't have (all asics).
 786  */
 787 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 788 {
 789         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 790                   reg, v);
 791         BUG();
 792 }
 793
 794 /**
 795  * amdgpu_block_invalid_rreg - dummy reg read function
 796  *
 797  * @adev: amdgpu_device pointer
 798  * @block: offset of instance
 799  * @reg: offset of register
 800  *
 801  * Dummy register read function.  Used for register blocks
 802  * that certain asics don't have (all asics).
 803  * Returns the value in the register.
 804  */
 805 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 806                                           uint32_t block, uint32_t reg)
 807 {
 808         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 809                   reg, block);
 810         BUG();
 811         return 0;
 812 }
 813
 814 /**
 815  * amdgpu_block_invalid_wreg - dummy reg write function
 816  *
 817  * @adev: amdgpu_device pointer
 818  * @block: offset of instance
 819  * @reg: offset of register
 820  * @v: value to write to the register
 821  *
 822  * Dummy register read function.  Used for register blocks
 823  * that certain asics don't have (all asics).
 824  */
 825 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 826                                       uint32_t block,
 827                                       uint32_t reg, uint32_t v)
 828 {
 829         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 830                   reg, block, v);
 831         BUG();
 832 }
 833
 834 /**
 835  * amdgpu_device_asic_init - Wrapper for atom asic_init
 836  *
 837  * @adev: amdgpu_device pointer
 838  *
 839  * Does any asic specific work and then calls atom asic init.
 840  */
 841 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 842 {
 843         amdgpu_asic_pre_asic_init(adev);
 844
 845         return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 846 }
 847
 848 /**
 849  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 850  *
 851  * @adev: amdgpu_device pointer
 852  *
 853  * Allocates a scratch page of VRAM for use by various things in the
 854  * driver.
 855  */
 856 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 857 {
 858         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 859                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 860                                        &adev->vram_scratch.robj,
 861                                        &adev->vram_scratch.gpu_addr,
 862                                        (void **)&adev->vram_scratch.ptr);
 863 }
 864
 865 /**
 866  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 867  *
 868  * @adev: amdgpu_device pointer
 869  *
 870  * Frees the VRAM scratch page.
 871  */
 872 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 873 {
 874         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 875 }
 876
 877 /**
 878  * amdgpu_device_program_register_sequence - program an array of registers.
 879  *
 880  * @adev: amdgpu_device pointer
 881  * @registers: pointer to the register array
 882  * @array_size: size of the register array
 883  *
 884  * Programs an array or registers with and and or masks.
 885  * This is a helper for setting golden registers.
 886  */
 887 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 888                                              const u32 *registers,
 889                                              const u32 array_size)
 890 {
 891         u32 tmp, reg, and_mask, or_mask;
 892         int i;
 893
 894         if (array_size % 3)
 895                 return;
 896
 897         for (i = 0; i < array_size; i +=3) {
 898                 reg = registers[i + 0];
 899                 and_mask = registers[i + 1];
 900                 or_mask = registers[i + 2];
 901
 902                 if (and_mask == 0xffffffff) {
 903                         tmp = or_mask;
 904                 } else {
 905                         tmp = RREG32(reg);
 906                         tmp &= ~and_mask;
 907                         if (adev->family >= AMDGPU_FAMILY_AI)
 908                                 tmp |= (or_mask & and_mask);
 909                         else
 910                                 tmp |= or_mask;
 911                 }
 912                 WREG32(reg, tmp);
 913         }
 914 }
 915
 916 /**
 917  * amdgpu_device_pci_config_reset - reset the GPU
 918  *
 919  * @adev: amdgpu_device pointer
 920  *
 921  * Resets the GPU using the pci config reset sequence.
 922  * Only applicable to asics prior to vega10.
 923  */
 924 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 925 {
 926         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 927 }
 928
 929 /**
 930  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
 931  *
 932  * @adev: amdgpu_device pointer
 933  *
 934  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
 935  */
 936 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
 937 {
 938         return pci_reset_function(adev->pdev);
 939 }
 940
 941 /*
 942  * GPU doorbell aperture helpers function.
 943  */
 944 /**
 945  * amdgpu_device_doorbell_init - Init doorbell driver information.
 946  *
 947  * @adev: amdgpu_device pointer
 948  *
 949  * Init doorbell driver information (CIK)
 950  * Returns 0 on success, error on failure.
 951  */
 952 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 953 {
 954
 955         /* No doorbell on SI hardware generation */
 956         if (adev->asic_type < CHIP_BONAIRE) {
 957                 adev->doorbell.base = 0;
 958                 adev->doorbell.size = 0;
 959                 adev->doorbell.num_doorbells = 0;
 960                 adev->doorbell.ptr = NULL;
 961                 return 0;
 962         }
 963
 964         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 965                 return -EINVAL;
 966
 967         amdgpu_asic_init_doorbell_index(adev);
 968
 969         /* doorbell bar mapping */
 970         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 971         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 972
 973         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 974                                              adev->doorbell_index.max_assignment+1);
 975         if (adev->doorbell.num_doorbells == 0)
 976                 return -EINVAL;
 977
 978         /* For Vega, reserve and map two pages on doorbell BAR since SDMA
 979          * paging queue doorbell use the second page. The
 980          * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
 981          * doorbells are in the first page. So with paging queue enabled,
 982          * the max num_doorbells should + 1 page (0x400 in dword)
 983          */
 984         if (adev->asic_type >= CHIP_VEGA10)
 985                 adev->doorbell.num_doorbells += 0x400;
 986
 987         adev->doorbell.ptr = ioremap(adev->doorbell.base,
 988                                      adev->doorbell.num_doorbells *
 989                                      sizeof(u32));
 990         if (adev->doorbell.ptr == NULL)
 991                 return -ENOMEM;
 992
 993         return 0;
 994 }
 995
 996 /**
 997  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
 998  *
 999  * @adev: amdgpu_device pointer
1000  *
1001  * Tear down doorbell driver information (CIK)
1002  */
1003 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1004 {
1005         iounmap(adev->doorbell.ptr);
1006         adev->doorbell.ptr = NULL;
1007 }
1008
1009
1010
1011 /*
1012  * amdgpu_device_wb_*()
1013  * Writeback is the method by which the GPU updates special pages in memory
1014  * with the status of certain GPU events (fences, ring pointers,etc.).
1015  */
1016
1017 /**
1018  * amdgpu_device_wb_fini - Disable Writeback and free memory
1019  *
1020  * @adev: amdgpu_device pointer
1021  *
1022  * Disables Writeback and frees the Writeback memory (all asics).
1023  * Used at driver shutdown.
1024  */
1025 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1026 {
1027         if (adev->wb.wb_obj) {
1028                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1029                                       &adev->wb.gpu_addr,
1030                                       (void **)&adev->wb.wb);
1031                 adev->wb.wb_obj = NULL;
1032         }
1033 }
1034
1035 /**
1036  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1037  *
1038  * @adev: amdgpu_device pointer
1039  *
1040  * Initializes writeback and allocates writeback memory (all asics).
1041  * Used at driver startup.
1042  * Returns 0 on success or an -error on failure.
1043  */
1044 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1045 {
1046         int r;
1047
1048         if (adev->wb.wb_obj == NULL) {
1049                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1050                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1051                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1052                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1053                                             (void **)&adev->wb.wb);
1054                 if (r) {
1055                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1056                         return r;
1057                 }
1058
1059                 adev->wb.num_wb = AMDGPU_MAX_WB;
1060                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1061
1062                 /* clear wb memory */
1063                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1064         }
1065
1066         return 0;
1067 }
1068
1069 /**
1070  * amdgpu_device_wb_get - Allocate a wb entry
1071  *
1072  * @adev: amdgpu_device pointer
1073  * @wb: wb index
1074  *
1075  * Allocate a wb slot for use by the driver (all asics).
1076  * Returns 0 on success or -EINVAL on failure.
1077  */
1078 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1079 {
1080         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1081
1082         if (offset < adev->wb.num_wb) {
1083                 __set_bit(offset, adev->wb.used);
1084                 *wb = offset << 3; /* convert to dw offset */
1085                 return 0;
1086         } else {
1087                 return -EINVAL;
1088         }
1089 }
1090
1091 /**
1092  * amdgpu_device_wb_free - Free a wb entry
1093  *
1094  * @adev: amdgpu_device pointer
1095  * @wb: wb index
1096  *
1097  * Free a wb slot allocated for use by the driver (all asics)
1098  */
1099 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1100 {
1101         wb >>= 3;
1102         if (wb < adev->wb.num_wb)
1103                 __clear_bit(wb, adev->wb.used);
1104 }
1105
1106 /**
1107  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1108  *
1109  * @adev: amdgpu_device pointer
1110  *
1111  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1112  * to fail, but if any of the BARs is not accessible after the size we abort
1113  * driver loading by returning -ENODEV.
1114  */
1115 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1116 {
1117         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1118         struct pci_bus *root;
1119         struct resource *res;
1120         unsigned i;
1121         u16 cmd;
1122         int r;
1123
1124         /* Bypass for VF */
1125         if (amdgpu_sriov_vf(adev))
1126                 return 0;
1127
1128         /* skip if the bios has already enabled large BAR */
1129         if (adev->gmc.real_vram_size &&
1130             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1131                 return 0;
1132
1133         /* Check if the root BUS has 64bit memory resources */
1134         root = adev->pdev->bus;
1135         while (root->parent)
1136                 root = root->parent;
1137
1138         pci_bus_for_each_resource(root, res, i) {
1139                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1140                     res->start > 0x100000000ull)
1141                         break;
1142         }
1143
1144         /* Trying to resize is pointless without a root hub window above 4GB */
1145         if (!res)
1146                 return 0;
1147
1148         /* Limit the BAR size to what is available */
1149         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1150                         rbar_size);
1151
1152         /* Disable memory decoding while we change the BAR addresses and size */
1153         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1154         pci_write_config_word(adev->pdev, PCI_COMMAND,
1155                               cmd & ~PCI_COMMAND_MEMORY);
1156
1157         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1158         amdgpu_device_doorbell_fini(adev);
1159         if (adev->asic_type >= CHIP_BONAIRE)
1160                 pci_release_resource(adev->pdev, 2);
1161
1162         pci_release_resource(adev->pdev, 0);
1163
1164         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1165         if (r == -ENOSPC)
1166                 DRM_INFO("Not enough PCI address space for a large BAR.");
1167         else if (r && r != -ENOTSUPP)
1168                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1169
1170         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1171
1172         /* When the doorbell or fb BAR isn't available we have no chance of
1173          * using the device.
1174          */
1175         r = amdgpu_device_doorbell_init(adev);
1176         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1177                 return -ENODEV;
1178
1179         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1180
1181         return 0;
1182 }
1183
1184 /*
1185  * GPU helpers function.
1186  */
1187 /**
1188  * amdgpu_device_need_post - check if the hw need post or not
1189  *
1190  * @adev: amdgpu_device pointer
1191  *
1192  * Check if the asic has been initialized (all asics) at driver startup
1193  * or post is needed if  hw reset is performed.
1194  * Returns true if need or false if not.
1195  */
1196 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1197 {
1198         uint32_t reg;
1199
1200         if (amdgpu_sriov_vf(adev))
1201                 return false;
1202
1203         if (amdgpu_passthrough(adev)) {
1204                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1205                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1206                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1207                  * vpost executed for smc version below 22.15
1208                  */
1209                 if (adev->asic_type == CHIP_FIJI) {
1210                         int err;
1211                         uint32_t fw_ver;
1212                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1213                         /* force vPost if error occured */
1214                         if (err)
1215                                 return true;
1216
1217                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1218                         if (fw_ver < 0x00160e00)
1219                                 return true;
1220                 }
1221         }
1222
1223         /* Don't post if we need to reset whole hive on init */
1224         if (adev->gmc.xgmi.pending_reset)
1225                 return false;
1226
1227         if (adev->has_hw_reset) {
1228                 adev->has_hw_reset = false;
1229                 return true;
1230         }
1231
1232         /* bios scratch used on CIK+ */
1233         if (adev->asic_type >= CHIP_BONAIRE)
1234                 return amdgpu_atombios_scratch_need_asic_init(adev);
1235
1236         /* check MEM_SIZE for older asics */
1237         reg = amdgpu_asic_get_config_memsize(adev);
1238
1239         if ((reg != 0) && (reg != 0xffffffff))
1240                 return false;
1241
1242         return true;
1243 }
1244
1245 /* if we get transitioned to only one device, take VGA back */
1246 /**
1247  * amdgpu_device_vga_set_decode - enable/disable vga decode
1248  *
1249  * @cookie: amdgpu_device pointer
1250  * @state: enable/disable vga decode
1251  *
1252  * Enable/disable vga decode (all asics).
1253  * Returns VGA resource flags.
1254  */
1255 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
1256 {
1257         struct amdgpu_device *adev = cookie;
1258         amdgpu_asic_set_vga_state(adev, state);
1259         if (state)
1260                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1261                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1262         else
1263                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1264 }
1265
1266 /**
1267  * amdgpu_device_check_block_size - validate the vm block size
1268  *
1269  * @adev: amdgpu_device pointer
1270  *
1271  * Validates the vm block size specified via module parameter.
1272  * The vm block size defines number of bits in page table versus page directory,
1273  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1274  * page table and the remaining bits are in the page directory.
1275  */
1276 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1277 {
1278         /* defines number of bits in page table versus page directory,
1279          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1280          * page table and the remaining bits are in the page directory */
1281         if (amdgpu_vm_block_size == -1)
1282                 return;
1283
1284         if (amdgpu_vm_block_size < 9) {
1285                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1286                          amdgpu_vm_block_size);
1287                 amdgpu_vm_block_size = -1;
1288         }
1289 }
1290
1291 /**
1292  * amdgpu_device_check_vm_size - validate the vm size
1293  *
1294  * @adev: amdgpu_device pointer
1295  *
1296  * Validates the vm size in GB specified via module parameter.
1297  * The VM size is the size of the GPU virtual memory space in GB.
1298  */
1299 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1300 {
1301         /* no need to check the default value */
1302         if (amdgpu_vm_size == -1)
1303                 return;
1304
1305         if (amdgpu_vm_size < 1) {
1306                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1307                          amdgpu_vm_size);
1308                 amdgpu_vm_size = -1;
1309         }
1310 }
1311
1312 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1313 {
1314         struct sysinfo si;
1315         bool is_os_64 = (sizeof(void *) == 8);
1316         uint64_t total_memory;
1317         uint64_t dram_size_seven_GB = 0x1B8000000;
1318         uint64_t dram_size_three_GB = 0xB8000000;
1319
1320         if (amdgpu_smu_memory_pool_size == 0)
1321                 return;
1322
1323         if (!is_os_64) {
1324                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1325                 goto def_value;
1326         }
1327         si_meminfo(&si);
1328         total_memory = (uint64_t)si.totalram * si.mem_unit;
1329
1330         if ((amdgpu_smu_memory_pool_size == 1) ||
1331                 (amdgpu_smu_memory_pool_size == 2)) {
1332                 if (total_memory < dram_size_three_GB)
1333                         goto def_value1;
1334         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1335                 (amdgpu_smu_memory_pool_size == 8)) {
1336                 if (total_memory < dram_size_seven_GB)
1337                         goto def_value1;
1338         } else {
1339                 DRM_WARN("Smu memory pool size not supported\n");
1340                 goto def_value;
1341         }
1342         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1343
1344         return;
1345
1346 def_value1:
1347         DRM_WARN("No enough system memory\n");
1348 def_value:
1349         adev->pm.smu_prv_buffer_size = 0;
1350 }
1351
1352 /**
1353  * amdgpu_device_check_arguments - validate module params
1354  *
1355  * @adev: amdgpu_device pointer
1356  *
1357  * Validates certain module parameters and updates
1358  * the associated values used by the driver (all asics).
1359  */
1360 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1361 {
1362         if (amdgpu_sched_jobs < 4) {
1363                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1364                          amdgpu_sched_jobs);
1365                 amdgpu_sched_jobs = 4;
1366         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1367                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1368                          amdgpu_sched_jobs);
1369                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1370         }
1371
1372         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1373                 /* gart size must be greater or equal to 32M */
1374                 dev_warn(adev->dev, "gart size (%d) too small\n",
1375                          amdgpu_gart_size);
1376                 amdgpu_gart_size = -1;
1377         }
1378
1379         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1380                 /* gtt size must be greater or equal to 32M */
1381                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1382                                  amdgpu_gtt_size);
1383                 amdgpu_gtt_size = -1;
1384         }
1385
1386         /* valid range is between 4 and 9 inclusive */
1387         if (amdgpu_vm_fragment_size != -1 &&
1388             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1389                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1390                 amdgpu_vm_fragment_size = -1;
1391         }
1392
1393         if (amdgpu_sched_hw_submission < 2) {
1394                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1395                          amdgpu_sched_hw_submission);
1396                 amdgpu_sched_hw_submission = 2;
1397         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1398                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1399                          amdgpu_sched_hw_submission);
1400                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1401         }
1402
1403         amdgpu_device_check_smu_prv_buffer_size(adev);
1404
1405         amdgpu_device_check_vm_size(adev);
1406
1407         amdgpu_device_check_block_size(adev);
1408
1409         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1410
1411         amdgpu_gmc_tmz_set(adev);
1412
1413         amdgpu_gmc_noretry_set(adev);
1414
1415         return 0;
1416 }
1417
1418 /**
1419  * amdgpu_switcheroo_set_state - set switcheroo state
1420  *
1421  * @pdev: pci dev pointer
1422  * @state: vga_switcheroo state
1423  *
1424  * Callback for the switcheroo driver.  Suspends or resumes the
1425  * the asics before or after it is powered up using ACPI methods.
1426  */
1427 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1428                                         enum vga_switcheroo_state state)
1429 {
1430         struct drm_device *dev = pci_get_drvdata(pdev);
1431         int r;
1432
1433         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1434                 return;
1435
1436         if (state == VGA_SWITCHEROO_ON) {
1437                 pr_info("switched on\n");
1438                 /* don't suspend or resume card normally */
1439                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1440
1441                 pci_set_power_state(pdev, PCI_D0);
1442                 amdgpu_device_load_pci_state(pdev);
1443                 r = pci_enable_device(pdev);
1444                 if (r)
1445                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1446                 amdgpu_device_resume(dev, true);
1447
1448                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1449         } else {
1450                 pr_info("switched off\n");
1451                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1452                 amdgpu_device_suspend(dev, true);
1453                 amdgpu_device_cache_pci_state(pdev);
1454                 /* Shut down the device */
1455                 pci_disable_device(pdev);
1456                 pci_set_power_state(pdev, PCI_D3cold);
1457                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1458         }
1459 }
1460
1461 /**
1462  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1463  *
1464  * @pdev: pci dev pointer
1465  *
1466  * Callback for the switcheroo driver.  Check of the switcheroo
1467  * state can be changed.
1468  * Returns true if the state can be changed, false if not.
1469  */
1470 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1471 {
1472         struct drm_device *dev = pci_get_drvdata(pdev);
1473
1474         /*
1475         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1476         * locking inversion with the driver load path. And the access here is
1477         * completely racy anyway. So don't bother with locking for now.
1478         */
1479         return atomic_read(&dev->open_count) == 0;
1480 }
1481
1482 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1483         .set_gpu_state = amdgpu_switcheroo_set_state,
1484         .reprobe = NULL,
1485         .can_switch = amdgpu_switcheroo_can_switch,
1486 };
1487
1488 /**
1489  * amdgpu_device_ip_set_clockgating_state - set the CG state
1490  *
1491  * @dev: amdgpu_device pointer
1492  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1493  * @state: clockgating state (gate or ungate)
1494  *
1495  * Sets the requested clockgating state for all instances of
1496  * the hardware IP specified.
1497  * Returns the error code from the last instance.
1498  */
1499 int amdgpu_device_ip_set_clockgating_state(void *dev,
1500                                            enum amd_ip_block_type block_type,
1501                                            enum amd_clockgating_state state)
1502 {
1503         struct amdgpu_device *adev = dev;
1504         int i, r = 0;
1505
1506         for (i = 0; i < adev->num_ip_blocks; i++) {
1507                 if (!adev->ip_blocks[i].status.valid)
1508                         continue;
1509                 if (adev->ip_blocks[i].version->type != block_type)
1510                         continue;
1511                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1512                         continue;
1513                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1514                         (void *)adev, state);
1515                 if (r)
1516                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1517                                   adev->ip_blocks[i].version->funcs->name, r);
1518         }
1519         return r;
1520 }
1521
1522 /**
1523  * amdgpu_device_ip_set_powergating_state - set the PG state
1524  *
1525  * @dev: amdgpu_device pointer
1526  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1527  * @state: powergating state (gate or ungate)
1528  *
1529  * Sets the requested powergating state for all instances of
1530  * the hardware IP specified.
1531  * Returns the error code from the last instance.
1532  */
1533 int amdgpu_device_ip_set_powergating_state(void *dev,
1534                                            enum amd_ip_block_type block_type,
1535                                            enum amd_powergating_state state)
1536 {
1537         struct amdgpu_device *adev = dev;
1538         int i, r = 0;
1539
1540         for (i = 0; i < adev->num_ip_blocks; i++) {
1541                 if (!adev->ip_blocks[i].status.valid)
1542                         continue;
1543                 if (adev->ip_blocks[i].version->type != block_type)
1544                         continue;
1545                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1546                         continue;
1547                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1548                         (void *)adev, state);
1549                 if (r)
1550                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1551                                   adev->ip_blocks[i].version->funcs->name, r);
1552         }
1553         return r;
1554 }
1555
1556 /**
1557  * amdgpu_device_ip_get_clockgating_state - get the CG state
1558  *
1559  * @adev: amdgpu_device pointer
1560  * @flags: clockgating feature flags
1561  *
1562  * Walks the list of IPs on the device and updates the clockgating
1563  * flags for each IP.
1564  * Updates @flags with the feature flags for each hardware IP where
1565  * clockgating is enabled.
1566  */
1567 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1568                                             u32 *flags)
1569 {
1570         int i;
1571
1572         for (i = 0; i < adev->num_ip_blocks; i++) {
1573                 if (!adev->ip_blocks[i].status.valid)
1574                         continue;
1575                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1576                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1577         }
1578 }
1579
1580 /**
1581  * amdgpu_device_ip_wait_for_idle - wait for idle
1582  *
1583  * @adev: amdgpu_device pointer
1584  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1585  *
1586  * Waits for the request hardware IP to be idle.
1587  * Returns 0 for success or a negative error code on failure.
1588  */
1589 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1590                                    enum amd_ip_block_type block_type)
1591 {
1592         int i, r;
1593
1594         for (i = 0; i < adev->num_ip_blocks; i++) {
1595                 if (!adev->ip_blocks[i].status.valid)
1596                         continue;
1597                 if (adev->ip_blocks[i].version->type == block_type) {
1598                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1599                         if (r)
1600                                 return r;
1601                         break;
1602                 }
1603         }
1604         return 0;
1605
1606 }
1607
1608 /**
1609  * amdgpu_device_ip_is_idle - is the hardware IP idle
1610  *
1611  * @adev: amdgpu_device pointer
1612  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1613  *
1614  * Check if the hardware IP is idle or not.
1615  * Returns true if it the IP is idle, false if not.
1616  */
1617 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1618                               enum amd_ip_block_type block_type)
1619 {
1620         int i;
1621
1622         for (i = 0; i < adev->num_ip_blocks; i++) {
1623                 if (!adev->ip_blocks[i].status.valid)
1624                         continue;
1625                 if (adev->ip_blocks[i].version->type == block_type)
1626                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1627         }
1628         return true;
1629
1630 }
1631
1632 /**
1633  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1634  *
1635  * @adev: amdgpu_device pointer
1636  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1637  *
1638  * Returns a pointer to the hardware IP block structure
1639  * if it exists for the asic, otherwise NULL.
1640  */
1641 struct amdgpu_ip_block *
1642 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1643                               enum amd_ip_block_type type)
1644 {
1645         int i;
1646
1647         for (i = 0; i < adev->num_ip_blocks; i++)
1648                 if (adev->ip_blocks[i].version->type == type)
1649                         return &adev->ip_blocks[i];
1650
1651         return NULL;
1652 }
1653
1654 /**
1655  * amdgpu_device_ip_block_version_cmp
1656  *
1657  * @adev: amdgpu_device pointer
1658  * @type: enum amd_ip_block_type
1659  * @major: major version
1660  * @minor: minor version
1661  *
1662  * return 0 if equal or greater
1663  * return 1 if smaller or the ip_block doesn't exist
1664  */
1665 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1666                                        enum amd_ip_block_type type,
1667                                        u32 major, u32 minor)
1668 {
1669         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1670
1671         if (ip_block && ((ip_block->version->major > major) ||
1672                         ((ip_block->version->major == major) &&
1673                         (ip_block->version->minor >= minor))))
1674                 return 0;
1675
1676         return 1;
1677 }
1678
1679 /**
1680  * amdgpu_device_ip_block_add
1681  *
1682  * @adev: amdgpu_device pointer
1683  * @ip_block_version: pointer to the IP to add
1684  *
1685  * Adds the IP block driver information to the collection of IPs
1686  * on the asic.
1687  */
1688 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1689                                const struct amdgpu_ip_block_version *ip_block_version)
1690 {
1691         if (!ip_block_version)
1692                 return -EINVAL;
1693
1694         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1695                   ip_block_version->funcs->name);
1696
1697         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1698
1699         return 0;
1700 }
1701
1702 /**
1703  * amdgpu_device_enable_virtual_display - enable virtual display feature
1704  *
1705  * @adev: amdgpu_device pointer
1706  *
1707  * Enabled the virtual display feature if the user has enabled it via
1708  * the module parameter virtual_display.  This feature provides a virtual
1709  * display hardware on headless boards or in virtualized environments.
1710  * This function parses and validates the configuration string specified by
1711  * the user and configues the virtual display configuration (number of
1712  * virtual connectors, crtcs, etc.) specified.
1713  */
1714 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1715 {
1716         adev->enable_virtual_display = false;
1717
1718         if (amdgpu_virtual_display) {
1719                 const char *pci_address_name = pci_name(adev->pdev);
1720                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1721
1722                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1723                 pciaddstr_tmp = pciaddstr;
1724                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1725                         pciaddname = strsep(&pciaddname_tmp, ",");
1726                         if (!strcmp("all", pciaddname)
1727                             || !strcmp(pci_address_name, pciaddname)) {
1728                                 long num_crtc;
1729                                 int res = -1;
1730
1731                                 adev->enable_virtual_display = true;
1732
1733                                 if (pciaddname_tmp)
1734                                         res = kstrtol(pciaddname_tmp, 10,
1735                                                       &num_crtc);
1736
1737                                 if (!res) {
1738                                         if (num_crtc < 1)
1739                                                 num_crtc = 1;
1740                                         if (num_crtc > 6)
1741                                                 num_crtc = 6;
1742                                         adev->mode_info.num_crtc = num_crtc;
1743                                 } else {
1744                                         adev->mode_info.num_crtc = 1;
1745                                 }
1746                                 break;
1747                         }
1748                 }
1749
1750                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1751                          amdgpu_virtual_display, pci_address_name,
1752                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1753
1754                 kfree(pciaddstr);
1755         }
1756 }
1757
1758 /**
1759  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1760  *
1761  * @adev: amdgpu_device pointer
1762  *
1763  * Parses the asic configuration parameters specified in the gpu info
1764  * firmware and makes them availale to the driver for use in configuring
1765  * the asic.
1766  * Returns 0 on success, -EINVAL on failure.
1767  */
1768 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1769 {
1770         const char *chip_name;
1771         char fw_name[40];
1772         int err;
1773         const struct gpu_info_firmware_header_v1_0 *hdr;
1774
1775         adev->firmware.gpu_info_fw = NULL;
1776
1777         if (adev->mman.discovery_bin) {
1778                 amdgpu_discovery_get_gfx_info(adev);
1779
1780                 /*
1781                  * FIXME: The bounding box is still needed by Navi12, so
1782                  * temporarily read it from gpu_info firmware. Should be droped
1783                  * when DAL no longer needs it.
1784                  */
1785                 if (adev->asic_type != CHIP_NAVI12)
1786                         return 0;
1787         }
1788
1789         switch (adev->asic_type) {
1790 #ifdef CONFIG_DRM_AMDGPU_SI
1791         case CHIP_VERDE:
1792         case CHIP_TAHITI:
1793         case CHIP_PITCAIRN:
1794         case CHIP_OLAND:
1795         case CHIP_HAINAN:
1796 #endif
1797 #ifdef CONFIG_DRM_AMDGPU_CIK
1798         case CHIP_BONAIRE:
1799         case CHIP_HAWAII:
1800         case CHIP_KAVERI:
1801         case CHIP_KABINI:
1802         case CHIP_MULLINS:
1803 #endif
1804         case CHIP_TOPAZ:
1805         case CHIP_TONGA:
1806         case CHIP_FIJI:
1807         case CHIP_POLARIS10:
1808         case CHIP_POLARIS11:
1809         case CHIP_POLARIS12:
1810         case CHIP_VEGAM:
1811         case CHIP_CARRIZO:
1812         case CHIP_STONEY:
1813         case CHIP_VEGA20:
1814         case CHIP_ALDEBARAN:
1815         case CHIP_SIENNA_CICHLID:
1816         case CHIP_NAVY_FLOUNDER:
1817         case CHIP_DIMGREY_CAVEFISH:
1818         default:
1819                 return 0;
1820         case CHIP_VEGA10:
1821                 chip_name = "vega10";
1822                 break;
1823         case CHIP_VEGA12:
1824                 chip_name = "vega12";
1825                 break;
1826         case CHIP_RAVEN:
1827                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1828                         chip_name = "raven2";
1829                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1830                         chip_name = "picasso";
1831                 else
1832                         chip_name = "raven";
1833                 break;
1834         case CHIP_ARCTURUS:
1835                 chip_name = "arcturus";
1836                 break;
1837         case CHIP_RENOIR:
1838                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1839                         chip_name = "renoir";
1840                 else
1841                         chip_name = "green_sardine";
1842                 break;
1843         case CHIP_NAVI10:
1844                 chip_name = "navi10";
1845                 break;
1846         case CHIP_NAVI14:
1847                 chip_name = "navi14";
1848                 break;
1849         case CHIP_NAVI12:
1850                 chip_name = "navi12";
1851                 break;
1852         case CHIP_VANGOGH:
1853                 chip_name = "vangogh";
1854                 break;
1855         }
1856
1857         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1858         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1859         if (err) {
1860                 dev_err(adev->dev,
1861                         "Failed to load gpu_info firmware \"%s\"\n",
1862                         fw_name);
1863                 goto out;
1864         }
1865         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1866         if (err) {
1867                 dev_err(adev->dev,
1868                         "Failed to validate gpu_info firmware \"%s\"\n",
1869                         fw_name);
1870                 goto out;
1871         }
1872
1873         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1874         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1875
1876         switch (hdr->version_major) {
1877         case 1:
1878         {
1879                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1880                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1881                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1882
1883                 /*
1884                  * Should be droped when DAL no longer needs it.
1885                  */
1886                 if (adev->asic_type == CHIP_NAVI12)
1887                         goto parse_soc_bounding_box;
1888
1889                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1890                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1891                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1892                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1893                 adev->gfx.config.max_texture_channel_caches =
1894                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1895                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1896                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1897                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1898                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1899                 adev->gfx.config.double_offchip_lds_buf =
1900                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1901                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1902                 adev->gfx.cu_info.max_waves_per_simd =
1903                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1904                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1905                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1906                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1907                 if (hdr->version_minor >= 1) {
1908                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1909                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1910                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1911                         adev->gfx.config.num_sc_per_sh =
1912                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1913                         adev->gfx.config.num_packer_per_sc =
1914                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1915                 }
1916
1917 parse_soc_bounding_box:
1918                 /*
1919                  * soc bounding box info is not integrated in disocovery table,
1920                  * we always need to parse it from gpu info firmware if needed.
1921                  */
1922                 if (hdr->version_minor == 2) {
1923                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1924                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1925                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1926                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1927                 }
1928                 break;
1929         }
1930         default:
1931                 dev_err(adev->dev,
1932                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
1933                 err = -EINVAL;
1934                 goto out;
1935         }
1936 out:
1937         return err;
1938 }
1939
1940 /**
1941  * amdgpu_device_ip_early_init - run early init for hardware IPs
1942  *
1943  * @adev: amdgpu_device pointer
1944  *
1945  * Early initialization pass for hardware IPs.  The hardware IPs that make
1946  * up each asic are discovered each IP's early_init callback is run.  This
1947  * is the first stage in initializing the asic.
1948  * Returns 0 on success, negative error code on failure.
1949  */
1950 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
1951 {
1952         int i, r;
1953
1954         amdgpu_device_enable_virtual_display(adev);
1955
1956         if (amdgpu_sriov_vf(adev)) {
1957                 r = amdgpu_virt_request_full_gpu(adev, true);
1958                 if (r)
1959                         return r;
1960         }
1961
1962         switch (adev->asic_type) {
1963 #ifdef CONFIG_DRM_AMDGPU_SI
1964         case CHIP_VERDE:
1965         case CHIP_TAHITI:
1966         case CHIP_PITCAIRN:
1967         case CHIP_OLAND:
1968         case CHIP_HAINAN:
1969                 adev->family = AMDGPU_FAMILY_SI;
1970                 r = si_set_ip_blocks(adev);
1971                 if (r)
1972                         return r;
1973                 break;
1974 #endif
1975 #ifdef CONFIG_DRM_AMDGPU_CIK
1976         case CHIP_BONAIRE:
1977         case CHIP_HAWAII:
1978         case CHIP_KAVERI:
1979         case CHIP_KABINI:
1980         case CHIP_MULLINS:
1981                 if (adev->flags & AMD_IS_APU)
1982                         adev->family = AMDGPU_FAMILY_KV;
1983                 else
1984                         adev->family = AMDGPU_FAMILY_CI;
1985
1986                 r = cik_set_ip_blocks(adev);
1987                 if (r)
1988                         return r;
1989                 break;
1990 #endif
1991         case CHIP_TOPAZ:
1992         case CHIP_TONGA:
1993         case CHIP_FIJI:
1994         case CHIP_POLARIS10:
1995         case CHIP_POLARIS11:
1996         case CHIP_POLARIS12:
1997         case CHIP_VEGAM:
1998         case CHIP_CARRIZO:
1999         case CHIP_STONEY:
2000                 if (adev->flags & AMD_IS_APU)
2001                         adev->family = AMDGPU_FAMILY_CZ;
2002                 else
2003                         adev->family = AMDGPU_FAMILY_VI;
2004
2005                 r = vi_set_ip_blocks(adev);
2006                 if (r)
2007                         return r;
2008                 break;
2009         case CHIP_VEGA10:
2010         case CHIP_VEGA12:
2011         case CHIP_VEGA20:
2012         case CHIP_RAVEN:
2013         case CHIP_ARCTURUS:
2014         case CHIP_RENOIR:
2015         case CHIP_ALDEBARAN:
2016                 if (adev->flags & AMD_IS_APU)
2017                         adev->family = AMDGPU_FAMILY_RV;
2018                 else
2019                         adev->family = AMDGPU_FAMILY_AI;
2020
2021                 r = soc15_set_ip_blocks(adev);
2022                 if (r)
2023                         return r;
2024                 break;
2025         case  CHIP_NAVI10:
2026         case  CHIP_NAVI14:
2027         case  CHIP_NAVI12:
2028         case  CHIP_SIENNA_CICHLID:
2029         case  CHIP_NAVY_FLOUNDER:
2030         case  CHIP_DIMGREY_CAVEFISH:
2031         case CHIP_VANGOGH:
2032                 if (adev->asic_type == CHIP_VANGOGH)
2033                         adev->family = AMDGPU_FAMILY_VGH;
2034                 else
2035                         adev->family = AMDGPU_FAMILY_NV;
2036
2037                 r = nv_set_ip_blocks(adev);
2038                 if (r)
2039                         return r;
2040                 break;
2041         default:
2042                 /* FIXME: not supported yet */
2043                 return -EINVAL;
2044         }
2045
2046         amdgpu_amdkfd_device_probe(adev);
2047
2048         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2049         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2050                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2051         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2052                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2053
2054         for (i = 0; i < adev->num_ip_blocks; i++) {
2055                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2056                         DRM_ERROR("disabled ip block: %d <%s>\n",
2057                                   i, adev->ip_blocks[i].version->funcs->name);
2058                         adev->ip_blocks[i].status.valid = false;
2059                 } else {
2060                         if (adev->ip_blocks[i].version->funcs->early_init) {
2061                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2062                                 if (r == -ENOENT) {
2063                                         adev->ip_blocks[i].status.valid = false;
2064                                 } else if (r) {
2065                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2066                                                   adev->ip_blocks[i].version->funcs->name, r);
2067                                         return r;
2068                                 } else {
2069                                         adev->ip_blocks[i].status.valid = true;
2070                                 }
2071                         } else {
2072                                 adev->ip_blocks[i].status.valid = true;
2073                         }
2074                 }
2075                 /* get the vbios after the asic_funcs are set up */
2076                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2077                         r = amdgpu_device_parse_gpu_info_fw(adev);
2078                         if (r)
2079                                 return r;
2080
2081                         /* Read BIOS */
2082                         if (!amdgpu_get_bios(adev))
2083                                 return -EINVAL;
2084
2085                         r = amdgpu_atombios_init(adev);
2086                         if (r) {
2087                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2088                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2089                                 return r;
2090                         }
2091
2092                         /*get pf2vf msg info at it's earliest time*/
2093                         if (amdgpu_sriov_vf(adev))
2094                                 amdgpu_virt_init_data_exchange(adev);
2095
2096                 }
2097         }
2098
2099         adev->cg_flags &= amdgpu_cg_mask;
2100         adev->pg_flags &= amdgpu_pg_mask;
2101
2102         return 0;
2103 }
2104
2105 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2106 {
2107         int i, r;
2108
2109         for (i = 0; i < adev->num_ip_blocks; i++) {
2110                 if (!adev->ip_blocks[i].status.sw)
2111                         continue;
2112                 if (adev->ip_blocks[i].status.hw)
2113                         continue;
2114                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2115                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2116                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2117                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2118                         if (r) {
2119                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2120                                           adev->ip_blocks[i].version->funcs->name, r);
2121                                 return r;
2122                         }
2123                         adev->ip_blocks[i].status.hw = true;
2124                 }
2125         }
2126
2127         return 0;
2128 }
2129
2130 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2131 {
2132         int i, r;
2133
2134         for (i = 0; i < adev->num_ip_blocks; i++) {
2135                 if (!adev->ip_blocks[i].status.sw)
2136                         continue;
2137                 if (adev->ip_blocks[i].status.hw)
2138                         continue;
2139                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2140                 if (r) {
2141                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2142                                   adev->ip_blocks[i].version->funcs->name, r);
2143                         return r;
2144                 }
2145                 adev->ip_blocks[i].status.hw = true;
2146         }
2147
2148         return 0;
2149 }
2150
2151 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2152 {
2153         int r = 0;
2154         int i;
2155         uint32_t smu_version;
2156
2157         if (adev->asic_type >= CHIP_VEGA10) {
2158                 for (i = 0; i < adev->num_ip_blocks; i++) {
2159                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2160                                 continue;
2161
2162                         if (!adev->ip_blocks[i].status.sw)
2163                                 continue;
2164
2165                         /* no need to do the fw loading again if already done*/
2166                         if (adev->ip_blocks[i].status.hw == true)
2167                                 break;
2168
2169                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2170                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2171                                 if (r) {
2172                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2173                                                           adev->ip_blocks[i].version->funcs->name, r);
2174                                         return r;
2175                                 }
2176                         } else {
2177                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2178                                 if (r) {
2179                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2180                                                           adev->ip_blocks[i].version->funcs->name, r);
2181                                         return r;
2182                                 }
2183                         }
2184
2185                         adev->ip_blocks[i].status.hw = true;
2186                         break;
2187                 }
2188         }
2189
2190         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2191                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2192
2193         return r;
2194 }
2195
2196 /**
2197  * amdgpu_device_ip_init - run init for hardware IPs
2198  *
2199  * @adev: amdgpu_device pointer
2200  *
2201  * Main initialization pass for hardware IPs.  The list of all the hardware
2202  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2203  * are run.  sw_init initializes the software state associated with each IP
2204  * and hw_init initializes the hardware associated with each IP.
2205  * Returns 0 on success, negative error code on failure.
2206  */
2207 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2208 {
2209         int i, r;
2210
2211         r = amdgpu_ras_init(adev);
2212         if (r)
2213                 return r;
2214
2215         for (i = 0; i < adev->num_ip_blocks; i++) {
2216                 if (!adev->ip_blocks[i].status.valid)
2217                         continue;
2218                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2219                 if (r) {
2220                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2221                                   adev->ip_blocks[i].version->funcs->name, r);
2222                         goto init_failed;
2223                 }
2224                 adev->ip_blocks[i].status.sw = true;
2225
2226                 /* need to do gmc hw init early so we can allocate gpu mem */
2227                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2228                         r = amdgpu_device_vram_scratch_init(adev);
2229                         if (r) {
2230                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2231                                 goto init_failed;
2232                         }
2233                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2234                         if (r) {
2235                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2236                                 goto init_failed;
2237                         }
2238                         r = amdgpu_device_wb_init(adev);
2239                         if (r) {
2240                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2241                                 goto init_failed;
2242                         }
2243                         adev->ip_blocks[i].status.hw = true;
2244
2245                         /* right after GMC hw init, we create CSA */
2246                         if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2247                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2248                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2249                                                                 AMDGPU_CSA_SIZE);
2250                                 if (r) {
2251                                         DRM_ERROR("allocate CSA failed %d\n", r);
2252                                         goto init_failed;
2253                                 }
2254                         }
2255                 }
2256         }
2257
2258         if (amdgpu_sriov_vf(adev))
2259                 amdgpu_virt_init_data_exchange(adev);
2260
2261         r = amdgpu_ib_pool_init(adev);
2262         if (r) {
2263                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2264                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2265                 goto init_failed;
2266         }
2267
2268         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2269         if (r)
2270                 goto init_failed;
2271
2272         r = amdgpu_device_ip_hw_init_phase1(adev);
2273         if (r)
2274                 goto init_failed;
2275
2276         r = amdgpu_device_fw_loading(adev);
2277         if (r)
2278                 goto init_failed;
2279
2280         r = amdgpu_device_ip_hw_init_phase2(adev);
2281         if (r)
2282                 goto init_failed;
2283
2284         /*
2285          * retired pages will be loaded from eeprom and reserved here,
2286          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2287          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2288          * for I2C communication which only true at this point.
2289          *
2290          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2291          * failure from bad gpu situation and stop amdgpu init process
2292          * accordingly. For other failed cases, it will still release all
2293          * the resource and print error message, rather than returning one
2294          * negative value to upper level.
2295          *
2296          * Note: theoretically, this should be called before all vram allocations
2297          * to protect retired page from abusing
2298          */
2299         r = amdgpu_ras_recovery_init(adev);
2300         if (r)
2301                 goto init_failed;
2302
2303         if (adev->gmc.xgmi.num_physical_nodes > 1)
2304                 amdgpu_xgmi_add_device(adev);
2305
2306         /* Don't init kfd if whole hive need to be reset during init */
2307         if (!adev->gmc.xgmi.pending_reset)
2308                 amdgpu_amdkfd_device_init(adev);
2309
2310         amdgpu_fru_get_product_info(adev);
2311
2312 init_failed:
2313         if (amdgpu_sriov_vf(adev))
2314                 amdgpu_virt_release_full_gpu(adev, true);
2315
2316         return r;
2317 }
2318
2319 /**
2320  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2321  *
2322  * @adev: amdgpu_device pointer
2323  *
2324  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2325  * this function before a GPU reset.  If the value is retained after a
2326  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2327  */
2328 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2329 {
2330         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2331 }
2332
2333 /**
2334  * amdgpu_device_check_vram_lost - check if vram is valid
2335  *
2336  * @adev: amdgpu_device pointer
2337  *
2338  * Checks the reset magic value written to the gart pointer in VRAM.
2339  * The driver calls this after a GPU reset to see if the contents of
2340  * VRAM is lost or now.
2341  * returns true if vram is lost, false if not.
2342  */
2343 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2344 {
2345         if (memcmp(adev->gart.ptr, adev->reset_magic,
2346                         AMDGPU_RESET_MAGIC_NUM))
2347                 return true;
2348
2349         if (!amdgpu_in_reset(adev))
2350                 return false;
2351
2352         /*
2353          * For all ASICs with baco/mode1 reset, the VRAM is
2354          * always assumed to be lost.
2355          */
2356         switch (amdgpu_asic_reset_method(adev)) {
2357         case AMD_RESET_METHOD_BACO:
2358         case AMD_RESET_METHOD_MODE1:
2359                 return true;
2360         default:
2361                 return false;
2362         }
2363 }
2364
2365 /**
2366  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2367  *
2368  * @adev: amdgpu_device pointer
2369  * @state: clockgating state (gate or ungate)
2370  *
2371  * The list of all the hardware IPs that make up the asic is walked and the
2372  * set_clockgating_state callbacks are run.
2373  * Late initialization pass enabling clockgating for hardware IPs.
2374  * Fini or suspend, pass disabling clockgating for hardware IPs.
2375  * Returns 0 on success, negative error code on failure.
2376  */
2377
2378 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2379                                enum amd_clockgating_state state)
2380 {
2381         int i, j, r;
2382
2383         if (amdgpu_emu_mode == 1)
2384                 return 0;
2385
2386         for (j = 0; j < adev->num_ip_blocks; j++) {
2387                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2388                 if (!adev->ip_blocks[i].status.late_initialized)
2389                         continue;
2390                 /* skip CG for GFX on S0ix */
2391                 if (adev->in_s0ix &&
2392                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2393                         continue;
2394                 /* skip CG for VCE/UVD, it's handled specially */
2395                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2396                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2397                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2398                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2399                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2400                         /* enable clockgating to save power */
2401                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2402                                                                                      state);
2403                         if (r) {
2404                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2405                                           adev->ip_blocks[i].version->funcs->name, r);
2406                                 return r;
2407                         }
2408                 }
2409         }
2410
2411         return 0;
2412 }
2413
2414 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2415                                enum amd_powergating_state state)
2416 {
2417         int i, j, r;
2418
2419         if (amdgpu_emu_mode == 1)
2420                 return 0;
2421
2422         for (j = 0; j < adev->num_ip_blocks; j++) {
2423                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2424                 if (!adev->ip_blocks[i].status.late_initialized)
2425                         continue;
2426                 /* skip PG for GFX on S0ix */
2427                 if (adev->in_s0ix &&
2428                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2429                         continue;
2430                 /* skip CG for VCE/UVD, it's handled specially */
2431                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2432                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2433                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2434                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2435                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2436                         /* enable powergating to save power */
2437                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2438                                                                                         state);
2439                         if (r) {
2440                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2441                                           adev->ip_blocks[i].version->funcs->name, r);
2442                                 return r;
2443                         }
2444                 }
2445         }
2446         return 0;
2447 }
2448
2449 static int amdgpu_device_enable_mgpu_fan_boost(void)
2450 {
2451         struct amdgpu_gpu_instance *gpu_ins;
2452         struct amdgpu_device *adev;
2453         int i, ret = 0;
2454
2455         mutex_lock(&mgpu_info.mutex);
2456
2457         /*
2458          * MGPU fan boost feature should be enabled
2459          * only when there are two or more dGPUs in
2460          * the system
2461          */
2462         if (mgpu_info.num_dgpu < 2)
2463                 goto out;
2464
2465         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2466                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2467                 adev = gpu_ins->adev;
2468                 if (!(adev->flags & AMD_IS_APU) &&
2469                     !gpu_ins->mgpu_fan_enabled) {
2470                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2471                         if (ret)
2472                                 break;
2473
2474                         gpu_ins->mgpu_fan_enabled = 1;
2475                 }
2476         }
2477
2478 out:
2479         mutex_unlock(&mgpu_info.mutex);
2480
2481         return ret;
2482 }
2483
2484 /**
2485  * amdgpu_device_ip_late_init - run late init for hardware IPs
2486  *
2487  * @adev: amdgpu_device pointer
2488  *
2489  * Late initialization pass for hardware IPs.  The list of all the hardware
2490  * IPs that make up the asic is walked and the late_init callbacks are run.
2491  * late_init covers any special initialization that an IP requires
2492  * after all of the have been initialized or something that needs to happen
2493  * late in the init process.
2494  * Returns 0 on success, negative error code on failure.
2495  */
2496 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2497 {
2498         struct amdgpu_gpu_instance *gpu_instance;
2499         int i = 0, r;
2500
2501         for (i = 0; i < adev->num_ip_blocks; i++) {
2502                 if (!adev->ip_blocks[i].status.hw)
2503                         continue;
2504                 if (adev->ip_blocks[i].version->funcs->late_init) {
2505                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2506                         if (r) {
2507                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2508                                           adev->ip_blocks[i].version->funcs->name, r);
2509                                 return r;
2510                         }
2511                 }
2512                 adev->ip_blocks[i].status.late_initialized = true;
2513         }
2514
2515         amdgpu_ras_set_error_query_ready(adev, true);
2516
2517         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2518         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2519
2520         amdgpu_device_fill_reset_magic(adev);
2521
2522         r = amdgpu_device_enable_mgpu_fan_boost();
2523         if (r)
2524                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2525
2526         /* For XGMI + passthrough configuration on arcturus, enable light SBR */
2527         if (adev->asic_type == CHIP_ARCTURUS &&
2528             amdgpu_passthrough(adev) &&
2529             adev->gmc.xgmi.num_physical_nodes > 1)
2530                 smu_set_light_sbr(&adev->smu, true);
2531
2532         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2533                 mutex_lock(&mgpu_info.mutex);
2534
2535                 /*
2536                  * Reset device p-state to low as this was booted with high.
2537                  *
2538                  * This should be performed only after all devices from the same
2539                  * hive get initialized.
2540                  *
2541                  * However, it's unknown how many device in the hive in advance.
2542                  * As this is counted one by one during devices initializations.
2543                  *
2544                  * So, we wait for all XGMI interlinked devices initialized.
2545                  * This may bring some delays as those devices may come from
2546                  * different hives. But that should be OK.
2547                  */
2548                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2549                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2550                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2551                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2552                                         continue;
2553
2554                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2555                                                 AMDGPU_XGMI_PSTATE_MIN);
2556                                 if (r) {
2557                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2558                                         break;
2559                                 }
2560                         }
2561                 }
2562
2563                 mutex_unlock(&mgpu_info.mutex);
2564         }
2565
2566         return 0;
2567 }
2568
2569 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2570 {
2571         int i, r;
2572
2573         for (i = 0; i < adev->num_ip_blocks; i++) {
2574                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2575                         continue;
2576
2577                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2578                 if (r) {
2579                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2580                                   adev->ip_blocks[i].version->funcs->name, r);
2581                 }
2582         }
2583
2584         amdgpu_amdkfd_suspend(adev, false);
2585
2586         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2587         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2588
2589         /* need to disable SMC first */
2590         for (i = 0; i < adev->num_ip_blocks; i++) {
2591                 if (!adev->ip_blocks[i].status.hw)
2592                         continue;
2593                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2594                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2595                         /* XXX handle errors */
2596                         if (r) {
2597                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2598                                           adev->ip_blocks[i].version->funcs->name, r);
2599                         }
2600                         adev->ip_blocks[i].status.hw = false;
2601                         break;
2602                 }
2603         }
2604
2605         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2606                 if (!adev->ip_blocks[i].status.hw)
2607                         continue;
2608
2609                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2610                 /* XXX handle errors */
2611                 if (r) {
2612                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2613                                   adev->ip_blocks[i].version->funcs->name, r);
2614                 }
2615
2616                 adev->ip_blocks[i].status.hw = false;
2617         }
2618
2619         return 0;
2620 }
2621
2622 /**
2623  * amdgpu_device_ip_fini - run fini for hardware IPs
2624  *
2625  * @adev: amdgpu_device pointer
2626  *
2627  * Main teardown pass for hardware IPs.  The list of all the hardware
2628  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2629  * are run.  hw_fini tears down the hardware associated with each IP
2630  * and sw_fini tears down any software state associated with each IP.
2631  * Returns 0 on success, negative error code on failure.
2632  */
2633 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2634 {
2635         int i, r;
2636
2637         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2638                 amdgpu_virt_release_ras_err_handler_data(adev);
2639
2640         amdgpu_ras_pre_fini(adev);
2641
2642         if (adev->gmc.xgmi.num_physical_nodes > 1)
2643                 amdgpu_xgmi_remove_device(adev);
2644
2645         amdgpu_amdkfd_device_fini_sw(adev);
2646
2647         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2648                 if (!adev->ip_blocks[i].status.sw)
2649                         continue;
2650
2651                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2652                         amdgpu_ucode_free_bo(adev);
2653                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2654                         amdgpu_device_wb_fini(adev);
2655                         amdgpu_device_vram_scratch_fini(adev);
2656                         amdgpu_ib_pool_fini(adev);
2657                 }
2658
2659                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2660                 /* XXX handle errors */
2661                 if (r) {
2662                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2663                                   adev->ip_blocks[i].version->funcs->name, r);
2664                 }
2665                 adev->ip_blocks[i].status.sw = false;
2666                 adev->ip_blocks[i].status.valid = false;
2667         }
2668
2669         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2670                 if (!adev->ip_blocks[i].status.late_initialized)
2671                         continue;
2672                 if (adev->ip_blocks[i].version->funcs->late_fini)
2673                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2674                 adev->ip_blocks[i].status.late_initialized = false;
2675         }
2676
2677         amdgpu_ras_fini(adev);
2678
2679         if (amdgpu_sriov_vf(adev))
2680                 if (amdgpu_virt_release_full_gpu(adev, false))
2681                         DRM_ERROR("failed to release exclusive mode on fini\n");
2682
2683         return 0;
2684 }
2685
2686 /**
2687  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2688  *
2689  * @work: work_struct.
2690  */
2691 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2692 {
2693         struct amdgpu_device *adev =
2694                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2695         int r;
2696
2697         r = amdgpu_ib_ring_tests(adev);
2698         if (r)
2699                 DRM_ERROR("ib ring test failed (%d).\n", r);
2700 }
2701
2702 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2703 {
2704         struct amdgpu_device *adev =
2705                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2706
2707         mutex_lock(&adev->gfx.gfx_off_mutex);
2708         if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2709                 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2710                         adev->gfx.gfx_off_state = true;
2711         }
2712         mutex_unlock(&adev->gfx.gfx_off_mutex);
2713 }
2714
2715 /**
2716  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2717  *
2718  * @adev: amdgpu_device pointer
2719  *
2720  * Main suspend function for hardware IPs.  The list of all the hardware
2721  * IPs that make up the asic is walked, clockgating is disabled and the
2722  * suspend callbacks are run.  suspend puts the hardware and software state
2723  * in each IP into a state suitable for suspend.
2724  * Returns 0 on success, negative error code on failure.
2725  */
2726 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2727 {
2728         int i, r;
2729
2730         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2731         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2732
2733         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2734                 if (!adev->ip_blocks[i].status.valid)
2735                         continue;
2736
2737                 /* displays are handled separately */
2738                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2739                         continue;
2740
2741                 /* XXX handle errors */
2742                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2743                 /* XXX handle errors */
2744                 if (r) {
2745                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2746                                   adev->ip_blocks[i].version->funcs->name, r);
2747                         return r;
2748                 }
2749
2750                 adev->ip_blocks[i].status.hw = false;
2751         }
2752
2753         return 0;
2754 }
2755
2756 /**
2757  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2758  *
2759  * @adev: amdgpu_device pointer
2760  *
2761  * Main suspend function for hardware IPs.  The list of all the hardware
2762  * IPs that make up the asic is walked, clockgating is disabled and the
2763  * suspend callbacks are run.  suspend puts the hardware and software state
2764  * in each IP into a state suitable for suspend.
2765  * Returns 0 on success, negative error code on failure.
2766  */
2767 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2768 {
2769         int i, r;
2770
2771         if (adev->in_s0ix)
2772                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
2773
2774         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2775                 if (!adev->ip_blocks[i].status.valid)
2776                         continue;
2777                 /* displays are handled in phase1 */
2778                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2779                         continue;
2780                 /* PSP lost connection when err_event_athub occurs */
2781                 if (amdgpu_ras_intr_triggered() &&
2782                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2783                         adev->ip_blocks[i].status.hw = false;
2784                         continue;
2785                 }
2786
2787                 /* skip unnecessary suspend if we do not initialize them yet */
2788                 if (adev->gmc.xgmi.pending_reset &&
2789                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2790                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2791                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2792                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2793                         adev->ip_blocks[i].status.hw = false;
2794                         continue;
2795                 }
2796
2797                 /* skip suspend of gfx and psp for S0ix
2798                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
2799                  * like at runtime. PSP is also part of the always on hardware
2800                  * so no need to suspend it.
2801                  */
2802                 if (adev->in_s0ix &&
2803                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
2804                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
2805                         continue;
2806
2807                 /* XXX handle errors */
2808                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2809                 /* XXX handle errors */
2810                 if (r) {
2811                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2812                                   adev->ip_blocks[i].version->funcs->name, r);
2813                 }
2814                 adev->ip_blocks[i].status.hw = false;
2815                 /* handle putting the SMC in the appropriate state */
2816                 if(!amdgpu_sriov_vf(adev)){
2817                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2818                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2819                                 if (r) {
2820                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2821                                                         adev->mp1_state, r);
2822                                         return r;
2823                                 }
2824                         }
2825                 }
2826         }
2827
2828         return 0;
2829 }
2830
2831 /**
2832  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2833  *
2834  * @adev: amdgpu_device pointer
2835  *
2836  * Main suspend function for hardware IPs.  The list of all the hardware
2837  * IPs that make up the asic is walked, clockgating is disabled and the
2838  * suspend callbacks are run.  suspend puts the hardware and software state
2839  * in each IP into a state suitable for suspend.
2840  * Returns 0 on success, negative error code on failure.
2841  */
2842 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2843 {
2844         int r;
2845
2846         if (amdgpu_sriov_vf(adev)) {
2847                 amdgpu_virt_fini_data_exchange(adev);
2848                 amdgpu_virt_request_full_gpu(adev, false);
2849         }
2850
2851         r = amdgpu_device_ip_suspend_phase1(adev);
2852         if (r)
2853                 return r;
2854         r = amdgpu_device_ip_suspend_phase2(adev);
2855
2856         if (amdgpu_sriov_vf(adev))
2857                 amdgpu_virt_release_full_gpu(adev, false);
2858
2859         return r;
2860 }
2861
2862 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2863 {
2864         int i, r;
2865
2866         static enum amd_ip_block_type ip_order[] = {
2867                 AMD_IP_BLOCK_TYPE_GMC,
2868                 AMD_IP_BLOCK_TYPE_COMMON,
2869                 AMD_IP_BLOCK_TYPE_PSP,
2870                 AMD_IP_BLOCK_TYPE_IH,
2871         };
2872
2873         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2874                 int j;
2875                 struct amdgpu_ip_block *block;
2876
2877                 block = &adev->ip_blocks[i];
2878                 block->status.hw = false;
2879
2880                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2881
2882                         if (block->version->type != ip_order[j] ||
2883                                 !block->status.valid)
2884                                 continue;
2885
2886                         r = block->version->funcs->hw_init(adev);
2887                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2888                         if (r)
2889                                 return r;
2890                         block->status.hw = true;
2891                 }
2892         }
2893
2894         return 0;
2895 }
2896
2897 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2898 {
2899         int i, r;
2900
2901         static enum amd_ip_block_type ip_order[] = {
2902                 AMD_IP_BLOCK_TYPE_SMC,
2903                 AMD_IP_BLOCK_TYPE_DCE,
2904                 AMD_IP_BLOCK_TYPE_GFX,
2905                 AMD_IP_BLOCK_TYPE_SDMA,
2906                 AMD_IP_BLOCK_TYPE_UVD,
2907                 AMD_IP_BLOCK_TYPE_VCE,
2908                 AMD_IP_BLOCK_TYPE_VCN
2909         };
2910
2911         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2912                 int j;
2913                 struct amdgpu_ip_block *block;
2914
2915                 for (j = 0; j < adev->num_ip_blocks; j++) {
2916                         block = &adev->ip_blocks[j];
2917
2918                         if (block->version->type != ip_order[i] ||
2919                                 !block->status.valid ||
2920                                 block->status.hw)
2921                                 continue;
2922
2923                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
2924                                 r = block->version->funcs->resume(adev);
2925                         else
2926                                 r = block->version->funcs->hw_init(adev);
2927
2928                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2929                         if (r)
2930                                 return r;
2931                         block->status.hw = true;
2932                 }
2933         }
2934
2935         return 0;
2936 }
2937
2938 /**
2939  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
2940  *
2941  * @adev: amdgpu_device pointer
2942  *
2943  * First resume function for hardware IPs.  The list of all the hardware
2944  * IPs that make up the asic is walked and the resume callbacks are run for
2945  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
2946  * after a suspend and updates the software state as necessary.  This
2947  * function is also used for restoring the GPU after a GPU reset.
2948  * Returns 0 on success, negative error code on failure.
2949  */
2950 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
2951 {
2952         int i, r;
2953
2954         for (i = 0; i < adev->num_ip_blocks; i++) {
2955                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2956                         continue;
2957                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2958                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2959                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2960
2961                         r = adev->ip_blocks[i].version->funcs->resume(adev);
2962                         if (r) {
2963                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
2964                                           adev->ip_blocks[i].version->funcs->name, r);
2965                                 return r;
2966                         }
2967                         adev->ip_blocks[i].status.hw = true;
2968                 }
2969         }
2970
2971         return 0;
2972 }
2973
2974 /**
2975  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
2976  *
2977  * @adev: amdgpu_device pointer
2978  *
2979  * First resume function for hardware IPs.  The list of all the hardware
2980  * IPs that make up the asic is walked and the resume callbacks are run for
2981  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
2982  * functional state after a suspend and updates the software state as
2983  * necessary.  This function is also used for restoring the GPU after a GPU
2984  * reset.
2985  * Returns 0 on success, negative error code on failure.
2986  */
2987 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
2988 {
2989         int i, r;
2990
2991         for (i = 0; i < adev->num_ip_blocks; i++) {
2992                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
2993                         continue;
2994                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2995                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2996                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
2997                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
2998                         continue;
2999                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3000                 if (r) {
3001                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3002                                   adev->ip_blocks[i].version->funcs->name, r);
3003                         return r;
3004                 }
3005                 adev->ip_blocks[i].status.hw = true;
3006         }
3007
3008         return 0;
3009 }
3010
3011 /**
3012  * amdgpu_device_ip_resume - run resume for hardware IPs
3013  *
3014  * @adev: amdgpu_device pointer
3015  *
3016  * Main resume function for hardware IPs.  The hardware IPs
3017  * are split into two resume functions because they are
3018  * are also used in in recovering from a GPU reset and some additional
3019  * steps need to be take between them.  In this case (S3/S4) they are
3020  * run sequentially.
3021  * Returns 0 on success, negative error code on failure.
3022  */
3023 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3024 {
3025         int r;
3026
3027         r = amdgpu_device_ip_resume_phase1(adev);
3028         if (r)
3029                 return r;
3030
3031         r = amdgpu_device_fw_loading(adev);
3032         if (r)
3033                 return r;
3034
3035         r = amdgpu_device_ip_resume_phase2(adev);
3036
3037         return r;
3038 }
3039
3040 /**
3041  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3042  *
3043  * @adev: amdgpu_device pointer
3044  *
3045  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3046  */
3047 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3048 {
3049         if (amdgpu_sriov_vf(adev)) {
3050                 if (adev->is_atom_fw) {
3051                         if (amdgpu_atomfirmware_gpu_supports_virtualization(adev))
3052                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3053                 } else {
3054                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3055                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3056                 }
3057
3058                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3059                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3060         }
3061 }
3062
3063 /**
3064  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3065  *
3066  * @asic_type: AMD asic type
3067  *
3068  * Check if there is DC (new modesetting infrastructre) support for an asic.
3069  * returns true if DC has support, false if not.
3070  */
3071 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3072 {
3073         switch (asic_type) {
3074 #if defined(CONFIG_DRM_AMD_DC)
3075 #if defined(CONFIG_DRM_AMD_DC_SI)
3076         case CHIP_TAHITI:
3077         case CHIP_PITCAIRN:
3078         case CHIP_VERDE:
3079         case CHIP_OLAND:
3080 #endif
3081         case CHIP_BONAIRE:
3082         case CHIP_KAVERI:
3083         case CHIP_KABINI:
3084         case CHIP_MULLINS:
3085                 /*
3086                  * We have systems in the wild with these ASICs that require
3087                  * LVDS and VGA support which is not supported with DC.
3088                  *
3089                  * Fallback to the non-DC driver here by default so as not to
3090                  * cause regressions.
3091                  */
3092                 return amdgpu_dc > 0;
3093         case CHIP_HAWAII:
3094         case CHIP_CARRIZO:
3095         case CHIP_STONEY:
3096         case CHIP_POLARIS10:
3097         case CHIP_POLARIS11:
3098         case CHIP_POLARIS12:
3099         case CHIP_VEGAM:
3100         case CHIP_TONGA:
3101         case CHIP_FIJI:
3102         case CHIP_VEGA10:
3103         case CHIP_VEGA12:
3104         case CHIP_VEGA20:
3105 #if defined(CONFIG_DRM_AMD_DC_DCN)
3106         case CHIP_RAVEN:
3107         case CHIP_NAVI10:
3108         case CHIP_NAVI14:
3109         case CHIP_NAVI12:
3110         case CHIP_RENOIR:
3111         case CHIP_SIENNA_CICHLID:
3112         case CHIP_NAVY_FLOUNDER:
3113         case CHIP_DIMGREY_CAVEFISH:
3114         case CHIP_VANGOGH:
3115 #endif
3116                 return amdgpu_dc != 0;
3117 #endif
3118         default:
3119                 if (amdgpu_dc > 0)
3120                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3121                                          "but isn't supported by ASIC, ignoring\n");
3122                 return false;
3123         }
3124 }
3125
3126 /**
3127  * amdgpu_device_has_dc_support - check if dc is supported
3128  *
3129  * @adev: amdgpu_device pointer
3130  *
3131  * Returns true for supported, false for not supported
3132  */
3133 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3134 {
3135         if (amdgpu_sriov_vf(adev) || adev->enable_virtual_display)
3136                 return false;
3137
3138         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3139 }
3140
3141
3142 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3143 {
3144         struct amdgpu_device *adev =
3145                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3146         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3147
3148         /* It's a bug to not have a hive within this function */
3149         if (WARN_ON(!hive))
3150                 return;
3151
3152         /*
3153          * Use task barrier to synchronize all xgmi reset works across the
3154          * hive. task_barrier_enter and task_barrier_exit will block
3155          * until all the threads running the xgmi reset works reach
3156          * those points. task_barrier_full will do both blocks.
3157          */
3158         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3159
3160                 task_barrier_enter(&hive->tb);
3161                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3162
3163                 if (adev->asic_reset_res)
3164                         goto fail;
3165
3166                 task_barrier_exit(&hive->tb);
3167                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3168
3169                 if (adev->asic_reset_res)
3170                         goto fail;
3171
3172                 if (adev->mmhub.ras_funcs &&
3173                     adev->mmhub.ras_funcs->reset_ras_error_count)
3174                         adev->mmhub.ras_funcs->reset_ras_error_count(adev);
3175         } else {
3176
3177                 task_barrier_full(&hive->tb);
3178                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3179         }
3180
3181 fail:
3182         if (adev->asic_reset_res)
3183                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3184                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3185         amdgpu_put_xgmi_hive(hive);
3186 }
3187
3188 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3189 {
3190         char *input = amdgpu_lockup_timeout;
3191         char *timeout_setting = NULL;
3192         int index = 0;
3193         long timeout;
3194         int ret = 0;
3195
3196         /*
3197          * By default timeout for non compute jobs is 10000.
3198          * And there is no timeout enforced on compute jobs.
3199          * In SR-IOV or passthrough mode, timeout for compute
3200          * jobs are 60000 by default.
3201          */
3202         adev->gfx_timeout = msecs_to_jiffies(10000);
3203         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3204         if (amdgpu_sriov_vf(adev))
3205                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3206                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3207         else if (amdgpu_passthrough(adev))
3208                 adev->compute_timeout =  msecs_to_jiffies(60000);
3209         else
3210                 adev->compute_timeout = MAX_SCHEDULE_TIMEOUT;
3211
3212         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3213                 while ((timeout_setting = strsep(&input, ",")) &&
3214                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3215                         ret = kstrtol(timeout_setting, 0, &timeout);
3216                         if (ret)
3217                                 return ret;
3218
3219                         if (timeout == 0) {
3220                                 index++;
3221                                 continue;
3222                         } else if (timeout < 0) {
3223                                 timeout = MAX_SCHEDULE_TIMEOUT;
3224                         } else {
3225                                 timeout = msecs_to_jiffies(timeout);
3226                         }
3227
3228                         switch (index++) {
3229                         case 0:
3230                                 adev->gfx_timeout = timeout;
3231                                 break;
3232                         case 1:
3233                                 adev->compute_timeout = timeout;
3234                                 break;
3235                         case 2:
3236                                 adev->sdma_timeout = timeout;
3237                                 break;
3238                         case 3:
3239                                 adev->video_timeout = timeout;
3240                                 break;
3241                         default:
3242                                 break;
3243                         }
3244                 }
3245                 /*
3246                  * There is only one value specified and
3247                  * it should apply to all non-compute jobs.
3248                  */
3249                 if (index == 1) {
3250                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3251                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3252                                 adev->compute_timeout = adev->gfx_timeout;
3253                 }
3254         }
3255
3256         return ret;
3257 }
3258
3259 static const struct attribute *amdgpu_dev_attributes[] = {
3260         &dev_attr_product_name.attr,
3261         &dev_attr_product_number.attr,
3262         &dev_attr_serial_number.attr,
3263         &dev_attr_pcie_replay_count.attr,
3264         NULL
3265 };
3266
3267 /**
3268  * amdgpu_device_init - initialize the driver
3269  *
3270  * @adev: amdgpu_device pointer
3271  * @flags: driver flags
3272  *
3273  * Initializes the driver info and hw (all asics).
3274  * Returns 0 for success or an error on failure.
3275  * Called at driver startup.
3276  */
3277 int amdgpu_device_init(struct amdgpu_device *adev,
3278                        uint32_t flags)
3279 {
3280         struct drm_device *ddev = adev_to_drm(adev);
3281         struct pci_dev *pdev = adev->pdev;
3282         int r, i;
3283         bool px = false;
3284         u32 max_MBps;
3285
3286         adev->shutdown = false;
3287         adev->flags = flags;
3288
3289         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3290                 adev->asic_type = amdgpu_force_asic_type;
3291         else
3292                 adev->asic_type = flags & AMD_ASIC_MASK;
3293
3294         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3295         if (amdgpu_emu_mode == 1)
3296                 adev->usec_timeout *= 10;
3297         adev->gmc.gart_size = 512 * 1024 * 1024;
3298         adev->accel_working = false;
3299         adev->num_rings = 0;
3300         adev->mman.buffer_funcs = NULL;
3301         adev->mman.buffer_funcs_ring = NULL;
3302         adev->vm_manager.vm_pte_funcs = NULL;
3303         adev->vm_manager.vm_pte_num_scheds = 0;
3304         adev->gmc.gmc_funcs = NULL;
3305         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3306         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3307
3308         adev->smc_rreg = &amdgpu_invalid_rreg;
3309         adev->smc_wreg = &amdgpu_invalid_wreg;
3310         adev->pcie_rreg = &amdgpu_invalid_rreg;
3311         adev->pcie_wreg = &amdgpu_invalid_wreg;
3312         adev->pciep_rreg = &amdgpu_invalid_rreg;
3313         adev->pciep_wreg = &amdgpu_invalid_wreg;
3314         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3315         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3316         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3317         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3318         adev->didt_rreg = &amdgpu_invalid_rreg;
3319         adev->didt_wreg = &amdgpu_invalid_wreg;
3320         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3321         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3322         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3323         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3324
3325         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3326                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3327                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3328
3329         /* mutex initialization are all done here so we
3330          * can recall function without having locking issues */
3331         mutex_init(&adev->firmware.mutex);
3332         mutex_init(&adev->pm.mutex);
3333         mutex_init(&adev->gfx.gpu_clock_mutex);
3334         mutex_init(&adev->srbm_mutex);
3335         mutex_init(&adev->gfx.pipe_reserve_mutex);
3336         mutex_init(&adev->gfx.gfx_off_mutex);
3337         mutex_init(&adev->grbm_idx_mutex);
3338         mutex_init(&adev->mn_lock);
3339         mutex_init(&adev->virt.vf_errors.lock);
3340         hash_init(adev->mn_hash);
3341         atomic_set(&adev->in_gpu_reset, 0);
3342         init_rwsem(&adev->reset_sem);
3343         mutex_init(&adev->psp.mutex);
3344         mutex_init(&adev->notifier_lock);
3345
3346         r = amdgpu_device_check_arguments(adev);
3347         if (r)
3348                 return r;
3349
3350         spin_lock_init(&adev->mmio_idx_lock);
3351         spin_lock_init(&adev->smc_idx_lock);
3352         spin_lock_init(&adev->pcie_idx_lock);
3353         spin_lock_init(&adev->uvd_ctx_idx_lock);
3354         spin_lock_init(&adev->didt_idx_lock);
3355         spin_lock_init(&adev->gc_cac_idx_lock);
3356         spin_lock_init(&adev->se_cac_idx_lock);
3357         spin_lock_init(&adev->audio_endpt_idx_lock);
3358         spin_lock_init(&adev->mm_stats.lock);
3359
3360         INIT_LIST_HEAD(&adev->shadow_list);
3361         mutex_init(&adev->shadow_list_lock);
3362
3363         INIT_LIST_HEAD(&adev->reset_list);
3364
3365         INIT_DELAYED_WORK(&adev->delayed_init_work,
3366                           amdgpu_device_delayed_init_work_handler);
3367         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3368                           amdgpu_device_delay_enable_gfx_off);
3369
3370         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3371
3372         adev->gfx.gfx_off_req_count = 1;
3373         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3374
3375         atomic_set(&adev->throttling_logging_enabled, 1);
3376         /*
3377          * If throttling continues, logging will be performed every minute
3378          * to avoid log flooding. "-1" is subtracted since the thermal
3379          * throttling interrupt comes every second. Thus, the total logging
3380          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3381          * for throttling interrupt) = 60 seconds.
3382          */
3383         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3384         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3385
3386         /* Registers mapping */
3387         /* TODO: block userspace mapping of io register */
3388         if (adev->asic_type >= CHIP_BONAIRE) {
3389                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3390                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3391         } else {
3392                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3393                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3394         }
3395
3396         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3397         if (adev->rmmio == NULL) {
3398                 return -ENOMEM;
3399         }
3400         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3401         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3402
3403         /* enable PCIE atomic ops */
3404         r = pci_enable_atomic_ops_to_root(adev->pdev,
3405                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3406                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3407         if (r) {
3408                 adev->have_atomics_support = false;
3409                 DRM_INFO("PCIE atomic ops is not supported\n");
3410         } else {
3411                 adev->have_atomics_support = true;
3412         }
3413
3414         amdgpu_device_get_pcie_info(adev);
3415
3416         if (amdgpu_mcbp)
3417                 DRM_INFO("MCBP is enabled\n");
3418
3419         if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3420                 adev->enable_mes = true;
3421
3422         /* detect hw virtualization here */
3423         amdgpu_detect_virtualization(adev);
3424
3425         r = amdgpu_device_get_job_timeout_settings(adev);
3426         if (r) {
3427                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3428                 goto failed_unmap;
3429         }
3430
3431         /* early init functions */
3432         r = amdgpu_device_ip_early_init(adev);
3433         if (r)
3434                 goto failed_unmap;
3435
3436         /* doorbell bar mapping and doorbell index init*/
3437         amdgpu_device_doorbell_init(adev);
3438
3439         if (amdgpu_emu_mode == 1) {
3440                 /* post the asic on emulation mode */
3441                 emu_soc_asic_init(adev);
3442                 goto fence_driver_init;
3443         }
3444
3445         amdgpu_reset_init(adev);
3446
3447         /* detect if we are with an SRIOV vbios */
3448         amdgpu_device_detect_sriov_bios(adev);
3449
3450         /* check if we need to reset the asic
3451          *  E.g., driver was not cleanly unloaded previously, etc.
3452          */
3453         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3454                 if (adev->gmc.xgmi.num_physical_nodes) {
3455                         dev_info(adev->dev, "Pending hive reset.\n");
3456                         adev->gmc.xgmi.pending_reset = true;
3457                         /* Only need to init necessary block for SMU to handle the reset */
3458                         for (i = 0; i < adev->num_ip_blocks; i++) {
3459                                 if (!adev->ip_blocks[i].status.valid)
3460                                         continue;
3461                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3462                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3463                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3464                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3465                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3466                                                 adev->ip_blocks[i].version->funcs->name);
3467                                         adev->ip_blocks[i].status.hw = true;
3468                                 }
3469                         }
3470                 } else {
3471                         r = amdgpu_asic_reset(adev);
3472                         if (r) {
3473                                 dev_err(adev->dev, "asic reset on init failed\n");
3474                                 goto failed;
3475                         }
3476                 }
3477         }
3478
3479         pci_enable_pcie_error_reporting(adev->pdev);
3480
3481         /* Post card if necessary */
3482         if (amdgpu_device_need_post(adev)) {
3483                 if (!adev->bios) {
3484                         dev_err(adev->dev, "no vBIOS found\n");
3485                         r = -EINVAL;
3486                         goto failed;
3487                 }
3488                 DRM_INFO("GPU posting now...\n");
3489                 r = amdgpu_device_asic_init(adev);
3490                 if (r) {
3491                         dev_err(adev->dev, "gpu post error!\n");
3492                         goto failed;
3493                 }
3494         }
3495
3496         if (adev->is_atom_fw) {
3497                 /* Initialize clocks */
3498                 r = amdgpu_atomfirmware_get_clock_info(adev);
3499                 if (r) {
3500                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3501                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3502                         goto failed;
3503                 }
3504         } else {
3505                 /* Initialize clocks */
3506                 r = amdgpu_atombios_get_clock_info(adev);
3507                 if (r) {
3508                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3509                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3510                         goto failed;
3511                 }
3512                 /* init i2c buses */
3513                 if (!amdgpu_device_has_dc_support(adev))
3514                         amdgpu_atombios_i2c_init(adev);
3515         }
3516
3517 fence_driver_init:
3518         /* Fence driver */
3519         r = amdgpu_fence_driver_init(adev);
3520         if (r) {
3521                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
3522                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3523                 goto failed;
3524         }
3525
3526         /* init the mode config */
3527         drm_mode_config_init(adev_to_drm(adev));
3528
3529         r = amdgpu_device_ip_init(adev);
3530         if (r) {
3531                 /* failed in exclusive mode due to timeout */
3532                 if (amdgpu_sriov_vf(adev) &&
3533                     !amdgpu_sriov_runtime(adev) &&
3534                     amdgpu_virt_mmio_blocked(adev) &&
3535                     !amdgpu_virt_wait_reset(adev)) {
3536                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3537                         /* Don't send request since VF is inactive. */
3538                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3539                         adev->virt.ops = NULL;
3540                         r = -EAGAIN;
3541                         goto release_ras_con;
3542                 }
3543                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3544                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3545                 goto release_ras_con;
3546         }
3547
3548         dev_info(adev->dev,
3549                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3550                         adev->gfx.config.max_shader_engines,
3551                         adev->gfx.config.max_sh_per_se,
3552                         adev->gfx.config.max_cu_per_sh,
3553                         adev->gfx.cu_info.number);
3554
3555         adev->accel_working = true;
3556
3557         amdgpu_vm_check_compute_bug(adev);
3558
3559         /* Initialize the buffer migration limit. */
3560         if (amdgpu_moverate >= 0)
3561                 max_MBps = amdgpu_moverate;
3562         else
3563                 max_MBps = 8; /* Allow 8 MB/s. */
3564         /* Get a log2 for easy divisions. */
3565         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3566
3567         amdgpu_fbdev_init(adev);
3568
3569         r = amdgpu_pm_sysfs_init(adev);
3570         if (r) {
3571                 adev->pm_sysfs_en = false;
3572                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3573         } else
3574                 adev->pm_sysfs_en = true;
3575
3576         r = amdgpu_ucode_sysfs_init(adev);
3577         if (r) {
3578                 adev->ucode_sysfs_en = false;
3579                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3580         } else
3581                 adev->ucode_sysfs_en = true;
3582
3583         if ((amdgpu_testing & 1)) {
3584                 if (adev->accel_working)
3585                         amdgpu_test_moves(adev);
3586                 else
3587                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3588         }
3589         if (amdgpu_benchmarking) {
3590                 if (adev->accel_working)
3591                         amdgpu_benchmark(adev, amdgpu_benchmarking);
3592                 else
3593                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3594         }
3595
3596         /*
3597          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3598          * Otherwise the mgpu fan boost feature will be skipped due to the
3599          * gpu instance is counted less.
3600          */
3601         amdgpu_register_gpu_instance(adev);
3602
3603         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3604          * explicit gating rather than handling it automatically.
3605          */
3606         if (!adev->gmc.xgmi.pending_reset) {
3607                 r = amdgpu_device_ip_late_init(adev);
3608                 if (r) {
3609                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3610                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3611                         goto release_ras_con;
3612                 }
3613                 /* must succeed. */
3614                 amdgpu_ras_resume(adev);
3615                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3616                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3617         }
3618
3619         if (amdgpu_sriov_vf(adev))
3620                 flush_delayed_work(&adev->delayed_init_work);
3621
3622         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3623         if (r)
3624                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3625
3626         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3627                 r = amdgpu_pmu_init(adev);
3628         if (r)
3629                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3630
3631         /* Have stored pci confspace at hand for restore in sudden PCI error */
3632         if (amdgpu_device_cache_pci_state(adev->pdev))
3633                 pci_restore_state(pdev);
3634
3635         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3636         /* this will fail for cards that aren't VGA class devices, just
3637          * ignore it */
3638         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3639                 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
3640
3641         if (amdgpu_device_supports_px(ddev)) {
3642                 px = true;
3643                 vga_switcheroo_register_client(adev->pdev,
3644                                                &amdgpu_switcheroo_ops, px);
3645                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3646         }
3647
3648         if (adev->gmc.xgmi.pending_reset)
3649                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3650                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3651
3652         return 0;
3653
3654 release_ras_con:
3655         amdgpu_release_ras_context(adev);
3656
3657 failed:
3658         amdgpu_vf_error_trans_all(adev);
3659
3660 failed_unmap:
3661         iounmap(adev->rmmio);
3662         adev->rmmio = NULL;
3663
3664         return r;
3665 }
3666
3667 /**
3668  * amdgpu_device_fini - tear down the driver
3669  *
3670  * @adev: amdgpu_device pointer
3671  *
3672  * Tear down the driver info (all asics).
3673  * Called at driver shutdown.
3674  */
3675 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3676 {
3677         dev_info(adev->dev, "amdgpu: finishing device.\n");
3678         flush_delayed_work(&adev->delayed_init_work);
3679         ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3680         adev->shutdown = true;
3681
3682         /* make sure IB test finished before entering exclusive mode
3683          * to avoid preemption on IB test
3684          * */
3685         if (amdgpu_sriov_vf(adev)) {
3686                 amdgpu_virt_request_full_gpu(adev, false);
3687                 amdgpu_virt_fini_data_exchange(adev);
3688         }
3689
3690         /* disable all interrupts */
3691         amdgpu_irq_disable_all(adev);
3692         if (adev->mode_info.mode_config_initialized){
3693                 if (!amdgpu_device_has_dc_support(adev))
3694                         drm_helper_force_disable_all(adev_to_drm(adev));
3695                 else
3696                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3697         }
3698         amdgpu_fence_driver_fini_hw(adev);
3699
3700         if (adev->pm_sysfs_en)
3701                 amdgpu_pm_sysfs_fini(adev);
3702         if (adev->ucode_sysfs_en)
3703                 amdgpu_ucode_sysfs_fini(adev);
3704         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3705
3706         amdgpu_fbdev_fini(adev);
3707
3708         amdgpu_irq_fini_hw(adev);
3709
3710         amdgpu_device_ip_fini_early(adev);
3711
3712         amdgpu_gart_dummy_page_fini(adev);
3713 }
3714
3715 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
3716 {
3717         amdgpu_device_ip_fini(adev);
3718         amdgpu_fence_driver_fini_sw(adev);
3719         release_firmware(adev->firmware.gpu_info_fw);
3720         adev->firmware.gpu_info_fw = NULL;
3721         adev->accel_working = false;
3722
3723         amdgpu_reset_fini(adev);
3724
3725         /* free i2c buses */
3726         if (!amdgpu_device_has_dc_support(adev))
3727                 amdgpu_i2c_fini(adev);
3728
3729         if (amdgpu_emu_mode != 1)
3730                 amdgpu_atombios_fini(adev);
3731
3732         kfree(adev->bios);
3733         adev->bios = NULL;
3734         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3735                 vga_switcheroo_unregister_client(adev->pdev);
3736                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3737         }
3738         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3739                 vga_client_register(adev->pdev, NULL, NULL, NULL);
3740         iounmap(adev->rmmio);
3741         adev->rmmio = NULL;
3742         amdgpu_device_doorbell_fini(adev);
3743
3744         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3745                 amdgpu_pmu_fini(adev);
3746         if (adev->mman.discovery_bin)
3747                 amdgpu_discovery_fini(adev);
3748
3749         kfree(adev->pci_state);
3750
3751 }
3752
3753
3754 /*
3755  * Suspend & resume.
3756  */
3757 /**
3758  * amdgpu_device_suspend - initiate device suspend
3759  *
3760  * @dev: drm dev pointer
3761  * @fbcon : notify the fbdev of suspend
3762  *
3763  * Puts the hw in the suspend state (all asics).
3764  * Returns 0 for success or an error on failure.
3765  * Called at driver suspend.
3766  */
3767 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3768 {
3769         struct amdgpu_device *adev = drm_to_adev(dev);
3770         int r;
3771
3772         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3773                 return 0;
3774
3775         adev->in_suspend = true;
3776         drm_kms_helper_poll_disable(dev);
3777
3778         if (fbcon)
3779                 amdgpu_fbdev_set_suspend(adev, 1);
3780
3781         cancel_delayed_work_sync(&adev->delayed_init_work);
3782
3783         amdgpu_ras_suspend(adev);
3784
3785         r = amdgpu_device_ip_suspend_phase1(adev);
3786
3787         if (!adev->in_s0ix)
3788                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
3789
3790         /* evict vram memory */
3791         amdgpu_bo_evict_vram(adev);
3792
3793         amdgpu_fence_driver_suspend(adev);
3794
3795         r = amdgpu_device_ip_suspend_phase2(adev);
3796         /* evict remaining vram memory
3797          * This second call to evict vram is to evict the gart page table
3798          * using the CPU.
3799          */
3800         amdgpu_bo_evict_vram(adev);
3801
3802         return 0;
3803 }
3804
3805 /**
3806  * amdgpu_device_resume - initiate device resume
3807  *
3808  * @dev: drm dev pointer
3809  * @fbcon : notify the fbdev of resume
3810  *
3811  * Bring the hw back to operating state (all asics).
3812  * Returns 0 for success or an error on failure.
3813  * Called at driver resume.
3814  */
3815 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
3816 {
3817         struct amdgpu_device *adev = drm_to_adev(dev);
3818         int r = 0;
3819
3820         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3821                 return 0;
3822
3823         if (adev->in_s0ix)
3824                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
3825
3826         /* post card */
3827         if (amdgpu_device_need_post(adev)) {
3828                 r = amdgpu_device_asic_init(adev);
3829                 if (r)
3830                         dev_err(adev->dev, "amdgpu asic init failed\n");
3831         }
3832
3833         r = amdgpu_device_ip_resume(adev);
3834         if (r) {
3835                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
3836                 return r;
3837         }
3838         amdgpu_fence_driver_resume(adev);
3839
3840
3841         r = amdgpu_device_ip_late_init(adev);
3842         if (r)
3843                 return r;
3844
3845         queue_delayed_work(system_wq, &adev->delayed_init_work,
3846                            msecs_to_jiffies(AMDGPU_RESUME_MS));
3847
3848         if (!adev->in_s0ix) {
3849                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
3850                 if (r)
3851                         return r;
3852         }
3853
3854         /* Make sure IB tests flushed */
3855         flush_delayed_work(&adev->delayed_init_work);
3856
3857         if (fbcon)
3858                 amdgpu_fbdev_set_suspend(adev, 0);
3859
3860         drm_kms_helper_poll_enable(dev);
3861
3862         amdgpu_ras_resume(adev);
3863
3864         /*
3865          * Most of the connector probing functions try to acquire runtime pm
3866          * refs to ensure that the GPU is powered on when connector polling is
3867          * performed. Since we're calling this from a runtime PM callback,
3868          * trying to acquire rpm refs will cause us to deadlock.
3869          *
3870          * Since we're guaranteed to be holding the rpm lock, it's safe to
3871          * temporarily disable the rpm helpers so this doesn't deadlock us.
3872          */
3873 #ifdef CONFIG_PM
3874         dev->dev->power.disable_depth++;
3875 #endif
3876         if (!amdgpu_device_has_dc_support(adev))
3877                 drm_helper_hpd_irq_event(dev);
3878         else
3879                 drm_kms_helper_hotplug_event(dev);
3880 #ifdef CONFIG_PM
3881         dev->dev->power.disable_depth--;
3882 #endif
3883         adev->in_suspend = false;
3884
3885         return 0;
3886 }
3887
3888 /**
3889  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3890  *
3891  * @adev: amdgpu_device pointer
3892  *
3893  * The list of all the hardware IPs that make up the asic is walked and
3894  * the check_soft_reset callbacks are run.  check_soft_reset determines
3895  * if the asic is still hung or not.
3896  * Returns true if any of the IPs are still in a hung state, false if not.
3897  */
3898 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
3899 {
3900         int i;
3901         bool asic_hang = false;
3902
3903         if (amdgpu_sriov_vf(adev))
3904                 return true;
3905
3906         if (amdgpu_asic_need_full_reset(adev))
3907                 return true;
3908
3909         for (i = 0; i < adev->num_ip_blocks; i++) {
3910                 if (!adev->ip_blocks[i].status.valid)
3911                         continue;
3912                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
3913                         adev->ip_blocks[i].status.hang =
3914                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
3915                 if (adev->ip_blocks[i].status.hang) {
3916                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
3917                         asic_hang = true;
3918                 }
3919         }
3920         return asic_hang;
3921 }
3922
3923 /**
3924  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
3925  *
3926  * @adev: amdgpu_device pointer
3927  *
3928  * The list of all the hardware IPs that make up the asic is walked and the
3929  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
3930  * handles any IP specific hardware or software state changes that are
3931  * necessary for a soft reset to succeed.
3932  * Returns 0 on success, negative error code on failure.
3933  */
3934 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
3935 {
3936         int i, r = 0;
3937
3938         for (i = 0; i < adev->num_ip_blocks; i++) {
3939                 if (!adev->ip_blocks[i].status.valid)
3940                         continue;
3941                 if (adev->ip_blocks[i].status.hang &&
3942                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
3943                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
3944                         if (r)
3945                                 return r;
3946                 }
3947         }
3948
3949         return 0;
3950 }
3951
3952 /**
3953  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
3954  *
3955  * @adev: amdgpu_device pointer
3956  *
3957  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
3958  * reset is necessary to recover.
3959  * Returns true if a full asic reset is required, false if not.
3960  */
3961 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
3962 {
3963         int i;
3964
3965         if (amdgpu_asic_need_full_reset(adev))
3966                 return true;
3967
3968         for (i = 0; i < adev->num_ip_blocks; i++) {
3969                 if (!adev->ip_blocks[i].status.valid)
3970                         continue;
3971                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
3972                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
3973                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
3974                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
3975                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3976                         if (adev->ip_blocks[i].status.hang) {
3977                                 dev_info(adev->dev, "Some block need full reset!\n");
3978                                 return true;
3979                         }
3980                 }
3981         }
3982         return false;
3983 }
3984
3985 /**
3986  * amdgpu_device_ip_soft_reset - do a soft reset
3987  *
3988  * @adev: amdgpu_device pointer
3989  *
3990  * The list of all the hardware IPs that make up the asic is walked and the
3991  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
3992  * IP specific hardware or software state changes that are necessary to soft
3993  * reset the IP.
3994  * Returns 0 on success, negative error code on failure.
3995  */
3996 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
3997 {
3998         int i, r = 0;
3999
4000         for (i = 0; i < adev->num_ip_blocks; i++) {
4001                 if (!adev->ip_blocks[i].status.valid)
4002                         continue;
4003                 if (adev->ip_blocks[i].status.hang &&
4004                     adev->ip_blocks[i].version->funcs->soft_reset) {
4005                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4006                         if (r)
4007                                 return r;
4008                 }
4009         }
4010
4011         return 0;
4012 }
4013
4014 /**
4015  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4016  *
4017  * @adev: amdgpu_device pointer
4018  *
4019  * The list of all the hardware IPs that make up the asic is walked and the
4020  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4021  * handles any IP specific hardware or software state changes that are
4022  * necessary after the IP has been soft reset.
4023  * Returns 0 on success, negative error code on failure.
4024  */
4025 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4026 {
4027         int i, r = 0;
4028
4029         for (i = 0; i < adev->num_ip_blocks; i++) {
4030                 if (!adev->ip_blocks[i].status.valid)
4031                         continue;
4032                 if (adev->ip_blocks[i].status.hang &&
4033                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4034                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4035                 if (r)
4036                         return r;
4037         }
4038
4039         return 0;
4040 }
4041
4042 /**
4043  * amdgpu_device_recover_vram - Recover some VRAM contents
4044  *
4045  * @adev: amdgpu_device pointer
4046  *
4047  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4048  * restore things like GPUVM page tables after a GPU reset where
4049  * the contents of VRAM might be lost.
4050  *
4051  * Returns:
4052  * 0 on success, negative error code on failure.
4053  */
4054 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4055 {
4056         struct dma_fence *fence = NULL, *next = NULL;
4057         struct amdgpu_bo *shadow;
4058         long r = 1, tmo;
4059
4060         if (amdgpu_sriov_runtime(adev))
4061                 tmo = msecs_to_jiffies(8000);
4062         else
4063                 tmo = msecs_to_jiffies(100);
4064
4065         dev_info(adev->dev, "recover vram bo from shadow start\n");
4066         mutex_lock(&adev->shadow_list_lock);
4067         list_for_each_entry(shadow, &adev->shadow_list, shadow_list) {
4068
4069                 /* No need to recover an evicted BO */
4070                 if (shadow->tbo.mem.mem_type != TTM_PL_TT ||
4071                     shadow->tbo.mem.start == AMDGPU_BO_INVALID_OFFSET ||
4072                     shadow->parent->tbo.mem.mem_type != TTM_PL_VRAM)
4073                         continue;
4074
4075                 r = amdgpu_bo_restore_shadow(shadow, &next);
4076                 if (r)
4077                         break;
4078
4079                 if (fence) {
4080                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4081                         dma_fence_put(fence);
4082                         fence = next;
4083                         if (tmo == 0) {
4084                                 r = -ETIMEDOUT;
4085                                 break;
4086                         } else if (tmo < 0) {
4087                                 r = tmo;
4088                                 break;
4089                         }
4090                 } else {
4091                         fence = next;
4092                 }
4093         }
4094         mutex_unlock(&adev->shadow_list_lock);
4095
4096         if (fence)
4097                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4098         dma_fence_put(fence);
4099
4100         if (r < 0 || tmo <= 0) {
4101                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4102                 return -EIO;
4103         }
4104
4105         dev_info(adev->dev, "recover vram bo from shadow done\n");
4106         return 0;
4107 }
4108
4109
4110 /**
4111  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4112  *
4113  * @adev: amdgpu_device pointer
4114  * @from_hypervisor: request from hypervisor
4115  *
4116  * do VF FLR and reinitialize Asic
4117  * return 0 means succeeded otherwise failed
4118  */
4119 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4120                                      bool from_hypervisor)
4121 {
4122         int r;
4123
4124         if (from_hypervisor)
4125                 r = amdgpu_virt_request_full_gpu(adev, true);
4126         else
4127                 r = amdgpu_virt_reset_gpu(adev);
4128         if (r)
4129                 return r;
4130
4131         amdgpu_amdkfd_pre_reset(adev);
4132
4133         /* Resume IP prior to SMC */
4134         r = amdgpu_device_ip_reinit_early_sriov(adev);
4135         if (r)
4136                 goto error;
4137
4138         amdgpu_virt_init_data_exchange(adev);
4139         /* we need recover gart prior to run SMC/CP/SDMA resume */
4140         amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));
4141
4142         r = amdgpu_device_fw_loading(adev);
4143         if (r)
4144                 return r;
4145
4146         /* now we are okay to resume SMC/CP/SDMA */
4147         r = amdgpu_device_ip_reinit_late_sriov(adev);
4148         if (r)
4149                 goto error;
4150
4151         amdgpu_irq_gpu_reset_resume_helper(adev);
4152         r = amdgpu_ib_ring_tests(adev);
4153         amdgpu_amdkfd_post_reset(adev);
4154
4155 error:
4156         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4157                 amdgpu_inc_vram_lost(adev);
4158                 r = amdgpu_device_recover_vram(adev);
4159         }
4160         amdgpu_virt_release_full_gpu(adev, true);
4161
4162         return r;
4163 }
4164
4165 /**
4166  * amdgpu_device_has_job_running - check if there is any job in mirror list
4167  *
4168  * @adev: amdgpu_device pointer
4169  *
4170  * check if there is any job in mirror list
4171  */
4172 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4173 {
4174         int i;
4175         struct drm_sched_job *job;
4176
4177         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4178                 struct amdgpu_ring *ring = adev->rings[i];
4179
4180                 if (!ring || !ring->sched.thread)
4181                         continue;
4182
4183                 spin_lock(&ring->sched.job_list_lock);
4184                 job = list_first_entry_or_null(&ring->sched.pending_list,
4185                                                struct drm_sched_job, list);
4186                 spin_unlock(&ring->sched.job_list_lock);
4187                 if (job)
4188                         return true;
4189         }
4190         return false;
4191 }
4192
4193 /**
4194  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4195  *
4196  * @adev: amdgpu_device pointer
4197  *
4198  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4199  * a hung GPU.
4200  */
4201 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4202 {
4203         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4204                 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4205                 return false;
4206         }
4207
4208         if (amdgpu_gpu_recovery == 0)
4209                 goto disabled;
4210
4211         if (amdgpu_sriov_vf(adev))
4212                 return true;
4213
4214         if (amdgpu_gpu_recovery == -1) {
4215                 switch (adev->asic_type) {
4216                 case CHIP_BONAIRE:
4217                 case CHIP_HAWAII:
4218                 case CHIP_TOPAZ:
4219                 case CHIP_TONGA:
4220                 case CHIP_FIJI:
4221                 case CHIP_POLARIS10:
4222                 case CHIP_POLARIS11:
4223                 case CHIP_POLARIS12:
4224                 case CHIP_VEGAM:
4225                 case CHIP_VEGA20:
4226                 case CHIP_VEGA10:
4227                 case CHIP_VEGA12:
4228                 case CHIP_RAVEN:
4229                 case CHIP_ARCTURUS:
4230                 case CHIP_RENOIR:
4231                 case CHIP_NAVI10:
4232                 case CHIP_NAVI14:
4233                 case CHIP_NAVI12:
4234                 case CHIP_SIENNA_CICHLID:
4235                 case CHIP_NAVY_FLOUNDER:
4236                 case CHIP_DIMGREY_CAVEFISH:
4237                 case CHIP_VANGOGH:
4238                 case CHIP_ALDEBARAN:
4239                         break;
4240                 default:
4241                         goto disabled;
4242                 }
4243         }
4244
4245         return true;
4246
4247 disabled:
4248                 dev_info(adev->dev, "GPU recovery disabled.\n");
4249                 return false;
4250 }
4251
4252 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4253 {
4254         u32 i;
4255         int ret = 0;
4256
4257         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4258
4259         dev_info(adev->dev, "GPU mode1 reset\n");
4260
4261         /* disable BM */
4262         pci_clear_master(adev->pdev);
4263
4264         amdgpu_device_cache_pci_state(adev->pdev);
4265
4266         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4267                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4268                 ret = amdgpu_dpm_mode1_reset(adev);
4269         } else {
4270                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4271                 ret = psp_gpu_reset(adev);
4272         }
4273
4274         if (ret)
4275                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4276
4277         amdgpu_device_load_pci_state(adev->pdev);
4278
4279         /* wait for asic to come out of reset */
4280         for (i = 0; i < adev->usec_timeout; i++) {
4281                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4282
4283                 if (memsize != 0xffffffff)
4284                         break;
4285                 udelay(1);
4286         }
4287
4288         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4289         return ret;
4290 }
4291
4292 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4293                                  struct amdgpu_reset_context *reset_context)
4294 {
4295         int i, r = 0;
4296         struct amdgpu_job *job = NULL;
4297         bool need_full_reset =
4298                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4299
4300         if (reset_context->reset_req_dev == adev)
4301                 job = reset_context->job;
4302
4303         /* no need to dump if device is not in good state during probe period */
4304         if (!adev->gmc.xgmi.pending_reset)
4305                 amdgpu_debugfs_wait_dump(adev);
4306
4307         if (amdgpu_sriov_vf(adev)) {
4308                 /* stop the data exchange thread */
4309                 amdgpu_virt_fini_data_exchange(adev);
4310         }
4311
4312         /* block all schedulers and reset given job's ring */
4313         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4314                 struct amdgpu_ring *ring = adev->rings[i];
4315
4316                 if (!ring || !ring->sched.thread)
4317                         continue;
4318
4319                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4320                 amdgpu_fence_driver_force_completion(ring);
4321         }
4322
4323         if(job)
4324                 drm_sched_increase_karma(&job->base);
4325
4326         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4327         /* If reset handler not implemented, continue; otherwise return */
4328         if (r == -ENOSYS)
4329                 r = 0;
4330         else
4331                 return r;
4332
4333         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4334         if (!amdgpu_sriov_vf(adev)) {
4335
4336                 if (!need_full_reset)
4337                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4338
4339                 if (!need_full_reset) {
4340                         amdgpu_device_ip_pre_soft_reset(adev);
4341                         r = amdgpu_device_ip_soft_reset(adev);
4342                         amdgpu_device_ip_post_soft_reset(adev);
4343                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4344                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4345                                 need_full_reset = true;
4346                         }
4347                 }
4348
4349                 if (need_full_reset)
4350                         r = amdgpu_device_ip_suspend(adev);
4351                 if (need_full_reset)
4352                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4353                 else
4354                         clear_bit(AMDGPU_NEED_FULL_RESET,
4355                                   &reset_context->flags);
4356         }
4357
4358         return r;
4359 }
4360
4361 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4362                          struct amdgpu_reset_context *reset_context)
4363 {
4364         struct amdgpu_device *tmp_adev = NULL;
4365         bool need_full_reset, skip_hw_reset, vram_lost = false;
4366         int r = 0;
4367
4368         /* Try reset handler method first */
4369         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4370                                     reset_list);
4371         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4372         /* If reset handler not implemented, continue; otherwise return */
4373         if (r == -ENOSYS)
4374                 r = 0;
4375         else
4376                 return r;
4377
4378         /* Reset handler not implemented, use the default method */
4379         need_full_reset =
4380                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4381         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4382
4383         /*
4384          * ASIC reset has to be done on all XGMI hive nodes ASAP
4385          * to allow proper links negotiation in FW (within 1 sec)
4386          */
4387         if (!skip_hw_reset && need_full_reset) {
4388                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4389                         /* For XGMI run all resets in parallel to speed up the process */
4390                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4391                                 tmp_adev->gmc.xgmi.pending_reset = false;
4392                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4393                                         r = -EALREADY;
4394                         } else
4395                                 r = amdgpu_asic_reset(tmp_adev);
4396
4397                         if (r) {
4398                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4399                                          r, adev_to_drm(tmp_adev)->unique);
4400                                 break;
4401                         }
4402                 }
4403
4404                 /* For XGMI wait for all resets to complete before proceed */
4405                 if (!r) {
4406                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4407                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4408                                         flush_work(&tmp_adev->xgmi_reset_work);
4409                                         r = tmp_adev->asic_reset_res;
4410                                         if (r)
4411                                                 break;
4412                                 }
4413                         }
4414                 }
4415         }
4416
4417         if (!r && amdgpu_ras_intr_triggered()) {
4418                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4419                         if (tmp_adev->mmhub.ras_funcs &&
4420                             tmp_adev->mmhub.ras_funcs->reset_ras_error_count)
4421                                 tmp_adev->mmhub.ras_funcs->reset_ras_error_count(tmp_adev);
4422                 }
4423
4424                 amdgpu_ras_intr_cleared();
4425         }
4426
4427         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4428                 if (need_full_reset) {
4429                         /* post card */
4430                         r = amdgpu_device_asic_init(tmp_adev);
4431                         if (r) {
4432                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4433                         } else {
4434                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4435                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4436                                 if (r)
4437                                         goto out;
4438
4439                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4440                                 if (vram_lost) {
4441                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4442                                         amdgpu_inc_vram_lost(tmp_adev);
4443                                 }
4444
4445                                 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));
4446                                 if (r)
4447                                         goto out;
4448
4449                                 r = amdgpu_device_fw_loading(tmp_adev);
4450                                 if (r)
4451                                         return r;
4452
4453                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4454                                 if (r)
4455                                         goto out;
4456
4457                                 if (vram_lost)
4458                                         amdgpu_device_fill_reset_magic(tmp_adev);
4459
4460                                 /*
4461                                  * Add this ASIC as tracked as reset was already
4462                                  * complete successfully.
4463                                  */
4464                                 amdgpu_register_gpu_instance(tmp_adev);
4465
4466                                 if (!reset_context->hive &&
4467                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4468                                         amdgpu_xgmi_add_device(tmp_adev);
4469
4470                                 r = amdgpu_device_ip_late_init(tmp_adev);
4471                                 if (r)
4472                                         goto out;
4473
4474                                 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4475
4476                                 /*
4477                                  * The GPU enters bad state once faulty pages
4478                                  * by ECC has reached the threshold, and ras
4479                                  * recovery is scheduled next. So add one check
4480                                  * here to break recovery if it indeed exceeds
4481                                  * bad page threshold, and remind user to
4482                                  * retire this GPU or setting one bigger
4483                                  * bad_page_threshold value to fix this once
4484                                  * probing driver again.
4485                                  */
4486                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4487                                         /* must succeed. */
4488                                         amdgpu_ras_resume(tmp_adev);
4489                                 } else {
4490                                         r = -EINVAL;
4491                                         goto out;
4492                                 }
4493
4494                                 /* Update PSP FW topology after reset */
4495                                 if (reset_context->hive &&
4496                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4497                                         r = amdgpu_xgmi_update_topology(
4498                                                 reset_context->hive, tmp_adev);
4499                         }
4500                 }
4501
4502 out:
4503                 if (!r) {
4504                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4505                         r = amdgpu_ib_ring_tests(tmp_adev);
4506                         if (r) {
4507                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4508                                 r = amdgpu_device_ip_suspend(tmp_adev);
4509                                 need_full_reset = true;
4510                                 r = -EAGAIN;
4511                                 goto end;
4512                         }
4513                 }
4514
4515                 if (!r)
4516                         r = amdgpu_device_recover_vram(tmp_adev);
4517                 else
4518                         tmp_adev->asic_reset_res = r;
4519         }
4520
4521 end:
4522         if (need_full_reset)
4523                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4524         else
4525                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4526         return r;
4527 }
4528
4529 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4530                                 struct amdgpu_hive_info *hive)
4531 {
4532         if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4533                 return false;
4534
4535         if (hive) {
4536                 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4537         } else {
4538                 down_write(&adev->reset_sem);
4539         }
4540
4541         switch (amdgpu_asic_reset_method(adev)) {
4542         case AMD_RESET_METHOD_MODE1:
4543                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4544                 break;
4545         case AMD_RESET_METHOD_MODE2:
4546                 adev->mp1_state = PP_MP1_STATE_RESET;
4547                 break;
4548         default:
4549                 adev->mp1_state = PP_MP1_STATE_NONE;
4550                 break;
4551         }
4552
4553         return true;
4554 }
4555
4556 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4557 {
4558         amdgpu_vf_error_trans_all(adev);
4559         adev->mp1_state = PP_MP1_STATE_NONE;
4560         atomic_set(&adev->in_gpu_reset, 0);
4561         up_write(&adev->reset_sem);
4562 }
4563
4564 /*
4565  * to lockup a list of amdgpu devices in a hive safely, if not a hive
4566  * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4567  *
4568  * unlock won't require roll back.
4569  */
4570 static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4571 {
4572         struct amdgpu_device *tmp_adev = NULL;
4573
4574         if (adev->gmc.xgmi.num_physical_nodes > 1) {
4575                 if (!hive) {
4576                         dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4577                         return -ENODEV;
4578                 }
4579                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4580                         if (!amdgpu_device_lock_adev(tmp_adev, hive))
4581                                 goto roll_back;
4582                 }
4583         } else if (!amdgpu_device_lock_adev(adev, hive))
4584                 return -EAGAIN;
4585
4586         return 0;
4587 roll_back:
4588         if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4589                 /*
4590                  * if the lockup iteration break in the middle of a hive,
4591                  * it may means there may has a race issue,
4592                  * or a hive device locked up independently.
4593                  * we may be in trouble and may not, so will try to roll back
4594                  * the lock and give out a warnning.
4595                  */
4596                 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4597                 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4598                         amdgpu_device_unlock_adev(tmp_adev);
4599                 }
4600         }
4601         return -EAGAIN;
4602 }
4603
4604 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4605 {
4606         struct pci_dev *p = NULL;
4607
4608         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4609                         adev->pdev->bus->number, 1);
4610         if (p) {
4611                 pm_runtime_enable(&(p->dev));
4612                 pm_runtime_resume(&(p->dev));
4613         }
4614 }
4615
4616 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4617 {
4618         enum amd_reset_method reset_method;
4619         struct pci_dev *p = NULL;
4620         u64 expires;
4621
4622         /*
4623          * For now, only BACO and mode1 reset are confirmed
4624          * to suffer the audio issue without proper suspended.
4625          */
4626         reset_method = amdgpu_asic_reset_method(adev);
4627         if ((reset_method != AMD_RESET_METHOD_BACO) &&
4628              (reset_method != AMD_RESET_METHOD_MODE1))
4629                 return -EINVAL;
4630
4631         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4632                         adev->pdev->bus->number, 1);
4633         if (!p)
4634                 return -ENODEV;
4635
4636         expires = pm_runtime_autosuspend_expiration(&(p->dev));
4637         if (!expires)
4638                 /*
4639                  * If we cannot get the audio device autosuspend delay,
4640                  * a fixed 4S interval will be used. Considering 3S is
4641                  * the audio controller default autosuspend delay setting.
4642                  * 4S used here is guaranteed to cover that.
4643                  */
4644                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4645
4646         while (!pm_runtime_status_suspended(&(p->dev))) {
4647                 if (!pm_runtime_suspend(&(p->dev)))
4648                         break;
4649
4650                 if (expires < ktime_get_mono_fast_ns()) {
4651                         dev_warn(adev->dev, "failed to suspend display audio\n");
4652                         /* TODO: abort the succeeding gpu reset? */
4653                         return -ETIMEDOUT;
4654                 }
4655         }
4656
4657         pm_runtime_disable(&(p->dev));
4658
4659         return 0;
4660 }
4661
4662 void amdgpu_device_recheck_guilty_jobs(
4663         struct amdgpu_device *adev, struct list_head *device_list_handle,
4664         struct amdgpu_reset_context *reset_context)
4665 {
4666         int i, r = 0;
4667
4668         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4669                 struct amdgpu_ring *ring = adev->rings[i];
4670                 int ret = 0;
4671                 struct drm_sched_job *s_job;
4672
4673                 if (!ring || !ring->sched.thread)
4674                         continue;
4675
4676                 s_job = list_first_entry_or_null(&ring->sched.pending_list,
4677                                 struct drm_sched_job, list);
4678                 if (s_job == NULL)
4679                         continue;
4680
4681                 /* clear job's guilty and depend the folowing step to decide the real one */
4682                 drm_sched_reset_karma(s_job);
4683                 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
4684
4685                 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
4686                 if (ret == 0) { /* timeout */
4687                         DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
4688                                                 ring->sched.name, s_job->id);
4689
4690                         /* set guilty */
4691                         drm_sched_increase_karma(s_job);
4692 retry:
4693                         /* do hw reset */
4694                         if (amdgpu_sriov_vf(adev)) {
4695                                 amdgpu_virt_fini_data_exchange(adev);
4696                                 r = amdgpu_device_reset_sriov(adev, false);
4697                                 if (r)
4698                                         adev->asic_reset_res = r;
4699                         } else {
4700                                 clear_bit(AMDGPU_SKIP_HW_RESET,
4701                                           &reset_context->flags);
4702                                 r = amdgpu_do_asic_reset(device_list_handle,
4703                                                          reset_context);
4704                                 if (r && r == -EAGAIN)
4705                                         goto retry;
4706                         }
4707
4708                         /*
4709                          * add reset counter so that the following
4710                          * resubmitted job could flush vmid
4711                          */
4712                         atomic_inc(&adev->gpu_reset_counter);
4713                         continue;
4714                 }
4715
4716                 /* got the hw fence, signal finished fence */
4717                 atomic_dec(ring->sched.score);
4718                 dma_fence_get(&s_job->s_fence->finished);
4719                 dma_fence_signal(&s_job->s_fence->finished);
4720                 dma_fence_put(&s_job->s_fence->finished);
4721
4722                 /* remove node from list and free the job */
4723                 spin_lock(&ring->sched.job_list_lock);
4724                 list_del_init(&s_job->list);
4725                 spin_unlock(&ring->sched.job_list_lock);
4726                 ring->sched.ops->free_job(s_job);
4727         }
4728 }
4729
4730 /**
4731  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4732  *
4733  * @adev: amdgpu_device pointer
4734  * @job: which job trigger hang
4735  *
4736  * Attempt to reset the GPU if it has hung (all asics).
4737  * Attempt to do soft-reset or full-reset and reinitialize Asic
4738  * Returns 0 for success or an error on failure.
4739  */
4740
4741 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4742                               struct amdgpu_job *job)
4743 {
4744         struct list_head device_list, *device_list_handle =  NULL;
4745         bool job_signaled = false;
4746         struct amdgpu_hive_info *hive = NULL;
4747         struct amdgpu_device *tmp_adev = NULL;
4748         int i, r = 0;
4749         bool need_emergency_restart = false;
4750         bool audio_suspended = false;
4751         int tmp_vram_lost_counter;
4752         struct amdgpu_reset_context reset_context;
4753
4754         memset(&reset_context, 0, sizeof(reset_context));
4755
4756         /*
4757          * Special case: RAS triggered and full reset isn't supported
4758          */
4759         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4760
4761         /*
4762          * Flush RAM to disk so that after reboot
4763          * the user can read log and see why the system rebooted.
4764          */
4765         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4766                 DRM_WARN("Emergency reboot.");
4767
4768                 ksys_sync_helper();
4769                 emergency_restart();
4770         }
4771
4772         dev_info(adev->dev, "GPU %s begin!\n",
4773                 need_emergency_restart ? "jobs stop":"reset");
4774
4775         /*
4776          * Here we trylock to avoid chain of resets executing from
4777          * either trigger by jobs on different adevs in XGMI hive or jobs on
4778          * different schedulers for same device while this TO handler is running.
4779          * We always reset all schedulers for device and all devices for XGMI
4780          * hive so that should take care of them too.
4781          */
4782         hive = amdgpu_get_xgmi_hive(adev);
4783         if (hive) {
4784                 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
4785                         DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
4786                                 job ? job->base.id : -1, hive->hive_id);
4787                         amdgpu_put_xgmi_hive(hive);
4788                         if (job)
4789                                 drm_sched_increase_karma(&job->base);
4790                         return 0;
4791                 }
4792                 mutex_lock(&hive->hive_lock);
4793         }
4794
4795         reset_context.method = AMD_RESET_METHOD_NONE;
4796         reset_context.reset_req_dev = adev;
4797         reset_context.job = job;
4798         reset_context.hive = hive;
4799         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
4800
4801         /*
4802          * lock the device before we try to operate the linked list
4803          * if didn't get the device lock, don't touch the linked list since
4804          * others may iterating it.
4805          */
4806         r = amdgpu_device_lock_hive_adev(adev, hive);
4807         if (r) {
4808                 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
4809                                         job ? job->base.id : -1);
4810
4811                 /* even we skipped this reset, still need to set the job to guilty */
4812                 if (job)
4813                         drm_sched_increase_karma(&job->base);
4814                 goto skip_recovery;
4815         }
4816
4817         /*
4818          * Build list of devices to reset.
4819          * In case we are in XGMI hive mode, resort the device list
4820          * to put adev in the 1st position.
4821          */
4822         INIT_LIST_HEAD(&device_list);
4823         if (adev->gmc.xgmi.num_physical_nodes > 1) {
4824                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
4825                         list_add_tail(&tmp_adev->reset_list, &device_list);
4826                 if (!list_is_first(&adev->reset_list, &device_list))
4827                         list_rotate_to_front(&adev->reset_list, &device_list);
4828                 device_list_handle = &device_list;
4829         } else {
4830                 list_add_tail(&adev->reset_list, &device_list);
4831                 device_list_handle = &device_list;
4832         }
4833
4834         /* block all schedulers and reset given job's ring */
4835         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4836                 /*
4837                  * Try to put the audio codec into suspend state
4838                  * before gpu reset started.
4839                  *
4840                  * Due to the power domain of the graphics device
4841                  * is shared with AZ power domain. Without this,
4842                  * we may change the audio hardware from behind
4843                  * the audio driver's back. That will trigger
4844                  * some audio codec errors.
4845                  */
4846                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4847                         audio_suspended = true;
4848
4849                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
4850
4851                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4852
4853                 if (!amdgpu_sriov_vf(tmp_adev))
4854                         amdgpu_amdkfd_pre_reset(tmp_adev);
4855
4856                 /*
4857                  * Mark these ASICs to be reseted as untracked first
4858                  * And add them back after reset completed
4859                  */
4860                 amdgpu_unregister_gpu_instance(tmp_adev);
4861
4862                 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4863
4864                 /* disable ras on ALL IPs */
4865                 if (!need_emergency_restart &&
4866                       amdgpu_device_ip_need_full_reset(tmp_adev))
4867                         amdgpu_ras_suspend(tmp_adev);
4868
4869                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4870                         struct amdgpu_ring *ring = tmp_adev->rings[i];
4871
4872                         if (!ring || !ring->sched.thread)
4873                                 continue;
4874
4875                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
4876
4877                         if (need_emergency_restart)
4878                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4879                 }
4880                 atomic_inc(&tmp_adev->gpu_reset_counter);
4881         }
4882
4883         if (need_emergency_restart)
4884                 goto skip_sched_resume;
4885
4886         /*
4887          * Must check guilty signal here since after this point all old
4888          * HW fences are force signaled.
4889          *
4890          * job->base holds a reference to parent fence
4891          */
4892         if (job && job->base.s_fence->parent &&
4893             dma_fence_is_signaled(job->base.s_fence->parent)) {
4894                 job_signaled = true;
4895                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
4896                 goto skip_hw_reset;
4897         }
4898
4899 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
4900         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4901                 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context);
4902                 /*TODO Should we stop ?*/
4903                 if (r) {
4904                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
4905                                   r, adev_to_drm(tmp_adev)->unique);
4906                         tmp_adev->asic_reset_res = r;
4907                 }
4908         }
4909
4910         tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
4911         /* Actual ASIC resets if needed.*/
4912         /* TODO Implement XGMI hive reset logic for SRIOV */
4913         if (amdgpu_sriov_vf(adev)) {
4914                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
4915                 if (r)
4916                         adev->asic_reset_res = r;
4917         } else {
4918                 r = amdgpu_do_asic_reset(device_list_handle, &reset_context);
4919                 if (r && r == -EAGAIN)
4920                         goto retry;
4921         }
4922
4923 skip_hw_reset:
4924
4925         /* Post ASIC reset for all devs .*/
4926         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4927
4928                 /*
4929                  * Sometimes a later bad compute job can block a good gfx job as gfx
4930                  * and compute ring share internal GC HW mutually. We add an additional
4931                  * guilty jobs recheck step to find the real guilty job, it synchronously
4932                  * submits and pends for the first job being signaled. If it gets timeout,
4933                  * we identify it as a real guilty job.
4934                  */
4935                 if (amdgpu_gpu_recovery == 2 &&
4936                         !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
4937                         amdgpu_device_recheck_guilty_jobs(
4938                                 tmp_adev, device_list_handle, &reset_context);
4939
4940                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4941                         struct amdgpu_ring *ring = tmp_adev->rings[i];
4942
4943                         if (!ring || !ring->sched.thread)
4944                                 continue;
4945
4946                         /* No point to resubmit jobs if we didn't HW reset*/
4947                         if (!tmp_adev->asic_reset_res && !job_signaled)
4948                                 drm_sched_resubmit_jobs(&ring->sched);
4949
4950                         drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
4951                 }
4952
4953                 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
4954                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
4955                 }
4956
4957                 tmp_adev->asic_reset_res = 0;
4958
4959                 if (r) {
4960                         /* bad news, how to tell it to userspace ? */
4961                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
4962                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
4963                 } else {
4964                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
4965                 }
4966         }
4967
4968 skip_sched_resume:
4969         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4970                 /* unlock kfd: SRIOV would do it separately */
4971                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
4972                         amdgpu_amdkfd_post_reset(tmp_adev);
4973
4974                 /* kfd_post_reset will do nothing if kfd device is not initialized,
4975                  * need to bring up kfd here if it's not be initialized before
4976                  */
4977                 if (!adev->kfd.init_complete)
4978                         amdgpu_amdkfd_device_init(adev);
4979
4980                 if (audio_suspended)
4981                         amdgpu_device_resume_display_audio(tmp_adev);
4982                 amdgpu_device_unlock_adev(tmp_adev);
4983         }
4984
4985 skip_recovery:
4986         if (hive) {
4987                 atomic_set(&hive->in_reset, 0);
4988                 mutex_unlock(&hive->hive_lock);
4989                 amdgpu_put_xgmi_hive(hive);
4990         }
4991
4992         if (r && r != -EAGAIN)
4993                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
4994         return r;
4995 }
4996
4997 /**
4998  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
4999  *
5000  * @adev: amdgpu_device pointer
5001  *
5002  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5003  * and lanes) of the slot the device is in. Handles APUs and
5004  * virtualized environments where PCIE config space may not be available.
5005  */
5006 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5007 {
5008         struct pci_dev *pdev;
5009         enum pci_bus_speed speed_cap, platform_speed_cap;
5010         enum pcie_link_width platform_link_width;
5011
5012         if (amdgpu_pcie_gen_cap)
5013                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5014
5015         if (amdgpu_pcie_lane_cap)
5016                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5017
5018         /* covers APUs as well */
5019         if (pci_is_root_bus(adev->pdev->bus)) {
5020                 if (adev->pm.pcie_gen_mask == 0)
5021                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5022                 if (adev->pm.pcie_mlw_mask == 0)
5023                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5024                 return;
5025         }
5026
5027         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5028                 return;
5029
5030         pcie_bandwidth_available(adev->pdev, NULL,
5031                                  &platform_speed_cap, &platform_link_width);
5032
5033         if (adev->pm.pcie_gen_mask == 0) {
5034                 /* asic caps */
5035                 pdev = adev->pdev;
5036                 speed_cap = pcie_get_speed_cap(pdev);
5037                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5038                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5039                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5040                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5041                 } else {
5042                         if (speed_cap == PCIE_SPEED_32_0GT)
5043                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5044                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5045                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5046                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5047                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5048                         else if (speed_cap == PCIE_SPEED_16_0GT)
5049                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5050                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5051                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5052                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5053                         else if (speed_cap == PCIE_SPEED_8_0GT)
5054                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5055                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5056                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5057                         else if (speed_cap == PCIE_SPEED_5_0GT)
5058                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5059                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5060                         else
5061                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5062                 }
5063                 /* platform caps */
5064                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5065                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5066                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5067                 } else {
5068                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5069                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5070                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5071                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5072                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5073                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5074                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5075                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5076                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5077                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5078                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5079                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5080                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5081                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5082                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5083                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5084                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5085                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5086                         else
5087                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5088
5089                 }
5090         }
5091         if (adev->pm.pcie_mlw_mask == 0) {
5092                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5093                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5094                 } else {
5095                         switch (platform_link_width) {
5096                         case PCIE_LNK_X32:
5097                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5098                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5099                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5100                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5101                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5102                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5103                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5104                                 break;
5105                         case PCIE_LNK_X16:
5106                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5107                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5108                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5109                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5110                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5111                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5112                                 break;
5113                         case PCIE_LNK_X12:
5114                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5115                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5116                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5117                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5118                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5119                                 break;
5120                         case PCIE_LNK_X8:
5121                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5122                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5123                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5124                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5125                                 break;
5126                         case PCIE_LNK_X4:
5127                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5128                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5129                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5130                                 break;
5131                         case PCIE_LNK_X2:
5132                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5133                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5134                                 break;
5135                         case PCIE_LNK_X1:
5136                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5137                                 break;
5138                         default:
5139                                 break;
5140                         }
5141                 }
5142         }
5143 }
5144
5145 int amdgpu_device_baco_enter(struct drm_device *dev)
5146 {
5147         struct amdgpu_device *adev = drm_to_adev(dev);
5148         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5149
5150         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5151                 return -ENOTSUPP;
5152
5153         if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
5154                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5155
5156         return amdgpu_dpm_baco_enter(adev);
5157 }
5158
5159 int amdgpu_device_baco_exit(struct drm_device *dev)
5160 {
5161         struct amdgpu_device *adev = drm_to_adev(dev);
5162         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5163         int ret = 0;
5164
5165         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5166                 return -ENOTSUPP;
5167
5168         ret = amdgpu_dpm_baco_exit(adev);
5169         if (ret)
5170                 return ret;
5171
5172         if (ras && ras->supported && adev->nbio.funcs->enable_doorbell_interrupt)
5173                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5174
5175         return 0;
5176 }
5177
5178 static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5179 {
5180         int i;
5181
5182         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5183                 struct amdgpu_ring *ring = adev->rings[i];
5184
5185                 if (!ring || !ring->sched.thread)
5186                         continue;
5187
5188                 cancel_delayed_work_sync(&ring->sched.work_tdr);
5189         }
5190 }
5191
5192 /**
5193  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5194  * @pdev: PCI device struct
5195  * @state: PCI channel state
5196  *
5197  * Description: Called when a PCI error is detected.
5198  *
5199  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5200  */
5201 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5202 {
5203         struct drm_device *dev = pci_get_drvdata(pdev);
5204         struct amdgpu_device *adev = drm_to_adev(dev);
5205         int i;
5206
5207         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5208
5209         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5210                 DRM_WARN("No support for XGMI hive yet...");
5211                 return PCI_ERS_RESULT_DISCONNECT;
5212         }
5213
5214         switch (state) {
5215         case pci_channel_io_normal:
5216                 return PCI_ERS_RESULT_CAN_RECOVER;
5217         /* Fatal error, prepare for slot reset */
5218         case pci_channel_io_frozen:
5219                 /*
5220                  * Cancel and wait for all TDRs in progress if failing to
5221                  * set  adev->in_gpu_reset in amdgpu_device_lock_adev
5222                  *
5223                  * Locking adev->reset_sem will prevent any external access
5224                  * to GPU during PCI error recovery
5225                  */
5226                 while (!amdgpu_device_lock_adev(adev, NULL))
5227                         amdgpu_cancel_all_tdr(adev);
5228
5229                 /*
5230                  * Block any work scheduling as we do for regular GPU reset
5231                  * for the duration of the recovery
5232                  */
5233                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5234                         struct amdgpu_ring *ring = adev->rings[i];
5235
5236                         if (!ring || !ring->sched.thread)
5237                                 continue;
5238
5239                         drm_sched_stop(&ring->sched, NULL);
5240                 }
5241                 atomic_inc(&adev->gpu_reset_counter);
5242                 return PCI_ERS_RESULT_NEED_RESET;
5243         case pci_channel_io_perm_failure:
5244                 /* Permanent error, prepare for device removal */
5245                 return PCI_ERS_RESULT_DISCONNECT;
5246         }
5247
5248         return PCI_ERS_RESULT_NEED_RESET;
5249 }
5250
5251 /**
5252  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5253  * @pdev: pointer to PCI device
5254  */
5255 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5256 {
5257
5258         DRM_INFO("PCI error: mmio enabled callback!!\n");
5259
5260         /* TODO - dump whatever for debugging purposes */
5261
5262         /* This called only if amdgpu_pci_error_detected returns
5263          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5264          * works, no need to reset slot.
5265          */
5266
5267         return PCI_ERS_RESULT_RECOVERED;
5268 }
5269
5270 /**
5271  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5272  * @pdev: PCI device struct
5273  *
5274  * Description: This routine is called by the pci error recovery
5275  * code after the PCI slot has been reset, just before we
5276  * should resume normal operations.
5277  */
5278 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5279 {
5280         struct drm_device *dev = pci_get_drvdata(pdev);
5281         struct amdgpu_device *adev = drm_to_adev(dev);
5282         int r, i;
5283         struct amdgpu_reset_context reset_context;
5284         u32 memsize;
5285         struct list_head device_list;
5286
5287         DRM_INFO("PCI error: slot reset callback!!\n");
5288
5289         memset(&reset_context, 0, sizeof(reset_context));
5290
5291         INIT_LIST_HEAD(&device_list);
5292         list_add_tail(&adev->reset_list, &device_list);
5293
5294         /* wait for asic to come out of reset */
5295         msleep(500);
5296
5297         /* Restore PCI confspace */
5298         amdgpu_device_load_pci_state(pdev);
5299
5300         /* confirm  ASIC came out of reset */
5301         for (i = 0; i < adev->usec_timeout; i++) {
5302                 memsize = amdgpu_asic_get_config_memsize(adev);
5303
5304                 if (memsize != 0xffffffff)
5305                         break;
5306                 udelay(1);
5307         }
5308         if (memsize == 0xffffffff) {
5309                 r = -ETIME;
5310                 goto out;
5311         }
5312
5313         reset_context.method = AMD_RESET_METHOD_NONE;
5314         reset_context.reset_req_dev = adev;
5315         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5316         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5317
5318         adev->in_pci_err_recovery = true;
5319         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5320         adev->in_pci_err_recovery = false;
5321         if (r)
5322                 goto out;
5323
5324         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5325
5326 out:
5327         if (!r) {
5328                 if (amdgpu_device_cache_pci_state(adev->pdev))
5329                         pci_restore_state(adev->pdev);
5330
5331                 DRM_INFO("PCIe error recovery succeeded\n");
5332         } else {
5333                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5334                 amdgpu_device_unlock_adev(adev);
5335         }
5336
5337         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5338 }
5339
5340 /**
5341  * amdgpu_pci_resume() - resume normal ops after PCI reset
5342  * @pdev: pointer to PCI device
5343  *
5344  * Called when the error recovery driver tells us that its
5345  * OK to resume normal operation.
5346  */
5347 void amdgpu_pci_resume(struct pci_dev *pdev)
5348 {
5349         struct drm_device *dev = pci_get_drvdata(pdev);
5350         struct amdgpu_device *adev = drm_to_adev(dev);
5351         int i;
5352
5353
5354         DRM_INFO("PCI error: resume callback!!\n");
5355
5356         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5357                 struct amdgpu_ring *ring = adev->rings[i];
5358
5359                 if (!ring || !ring->sched.thread)
5360                         continue;
5361
5362
5363                 drm_sched_resubmit_jobs(&ring->sched);
5364                 drm_sched_start(&ring->sched, true);
5365         }
5366
5367         amdgpu_device_unlock_adev(adev);
5368 }
5369
5370 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5371 {
5372         struct drm_device *dev = pci_get_drvdata(pdev);
5373         struct amdgpu_device *adev = drm_to_adev(dev);
5374         int r;
5375
5376         r = pci_save_state(pdev);
5377         if (!r) {
5378                 kfree(adev->pci_state);
5379
5380                 adev->pci_state = pci_store_saved_state(pdev);
5381
5382                 if (!adev->pci_state) {
5383                         DRM_ERROR("Failed to store PCI saved state");
5384                         return false;
5385                 }
5386         } else {
5387                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5388                 return false;
5389         }
5390
5391         return true;
5392 }
5393
5394 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5395 {
5396         struct drm_device *dev = pci_get_drvdata(pdev);
5397         struct amdgpu_device *adev = drm_to_adev(dev);
5398         int r;
5399
5400         if (!adev->pci_state)
5401                 return false;
5402
5403         r = pci_load_saved_state(pdev, adev->pci_state);
5404
5405         if (!r) {
5406                 pci_restore_state(pdev);
5407         } else {
5408                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5409                 return false;
5410         }
5411
5412         return true;
5413 }
5414
5415