drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33
  34 #include <drm/drm_atomic_helper.h>
  35 #include <drm/drm_probe_helper.h>
  36 #include <drm/amdgpu_drm.h>
  37 #include <linux/vgaarb.h>
  38 #include <linux/vga_switcheroo.h>
  39 #include <linux/efi.h>
  40 #include "amdgpu.h"
  41 #include "amdgpu_trace.h"
  42 #include "amdgpu_i2c.h"
  43 #include "atom.h"
  44 #include "amdgpu_atombios.h"
  45 #include "amdgpu_atomfirmware.h"
  46 #include "amd_pcie.h"
  47 #ifdef CONFIG_DRM_AMDGPU_SI
  48 #include "si.h"
  49 #endif
  50 #ifdef CONFIG_DRM_AMDGPU_CIK
  51 #include "cik.h"
  52 #endif
  53 #include "vi.h"
  54 #include "soc15.h"
  55 #include "nv.h"
  56 #include "bif/bif_4_1_d.h"
  57 #include <linux/pci.h>
  58 #include <linux/firmware.h>
  59 #include "amdgpu_vf_error.h"
  60
  61 #include "amdgpu_amdkfd.h"
  62 #include "amdgpu_pm.h"
  63
  64 #include "amdgpu_xgmi.h"
  65 #include "amdgpu_ras.h"
  66 #include "amdgpu_pmu.h"
  67 #include "amdgpu_fru_eeprom.h"
  68 #include "amdgpu_reset.h"
  69
  70 #include <linux/suspend.h>
  71 #include <drm/task_barrier.h>
  72 #include <linux/pm_runtime.h>
  73
  74 #include <drm/drm_drv.h>
  75
  76 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  77 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  78 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  79 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  80 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  81 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/renoir_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/navi10_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/navi14_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/vangogh_gpu_info.bin");
  87 MODULE_FIRMWARE("amdgpu/yellow_carp_gpu_info.bin");
  88
  89 #define AMDGPU_RESUME_MS                2000
  90
  91 const char *amdgpu_asic_name[] = {
  92         "TAHITI",
  93         "PITCAIRN",
  94         "VERDE",
  95         "OLAND",
  96         "HAINAN",
  97         "BONAIRE",
  98         "KAVERI",
  99         "KABINI",
 100         "HAWAII",
 101         "MULLINS",
 102         "TOPAZ",
 103         "TONGA",
 104         "FIJI",
 105         "CARRIZO",
 106         "STONEY",
 107         "POLARIS10",
 108         "POLARIS11",
 109         "POLARIS12",
 110         "VEGAM",
 111         "VEGA10",
 112         "VEGA12",
 113         "VEGA20",
 114         "RAVEN",
 115         "ARCTURUS",
 116         "RENOIR",
 117         "ALDEBARAN",
 118         "NAVI10",
 119         "NAVI14",
 120         "NAVI12",
 121         "SIENNA_CICHLID",
 122         "NAVY_FLOUNDER",
 123         "VANGOGH",
 124         "DIMGREY_CAVEFISH",
 125         "BEIGE_GOBY",
 126         "YELLOW_CARP",
 127         "LAST",
 128 };
 129
 130 /**
 131  * DOC: pcie_replay_count
 132  *
 133  * The amdgpu driver provides a sysfs API for reporting the total number
 134  * of PCIe replays (NAKs)
 135  * The file pcie_replay_count is used for this and returns the total
 136  * number of replays as a sum of the NAKs generated and NAKs received
 137  */
 138
 139 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 140                 struct device_attribute *attr, char *buf)
 141 {
 142         struct drm_device *ddev = dev_get_drvdata(dev);
 143         struct amdgpu_device *adev = drm_to_adev(ddev);
 144         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 145
 146         return sysfs_emit(buf, "%llu\n", cnt);
 147 }
 148
 149 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 150                 amdgpu_device_get_pcie_replay_count, NULL);
 151
 152 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 153
 154 /**
 155  * DOC: product_name
 156  *
 157  * The amdgpu driver provides a sysfs API for reporting the product name
 158  * for the device
 159  * The file serial_number is used for this and returns the product name
 160  * as returned from the FRU.
 161  * NOTE: This is only available for certain server cards
 162  */
 163
 164 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 165                 struct device_attribute *attr, char *buf)
 166 {
 167         struct drm_device *ddev = dev_get_drvdata(dev);
 168         struct amdgpu_device *adev = drm_to_adev(ddev);
 169
 170         return sysfs_emit(buf, "%s\n", adev->product_name);
 171 }
 172
 173 static DEVICE_ATTR(product_name, S_IRUGO,
 174                 amdgpu_device_get_product_name, NULL);
 175
 176 /**
 177  * DOC: product_number
 178  *
 179  * The amdgpu driver provides a sysfs API for reporting the part number
 180  * for the device
 181  * The file serial_number is used for this and returns the part number
 182  * as returned from the FRU.
 183  * NOTE: This is only available for certain server cards
 184  */
 185
 186 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 187                 struct device_attribute *attr, char *buf)
 188 {
 189         struct drm_device *ddev = dev_get_drvdata(dev);
 190         struct amdgpu_device *adev = drm_to_adev(ddev);
 191
 192         return sysfs_emit(buf, "%s\n", adev->product_number);
 193 }
 194
 195 static DEVICE_ATTR(product_number, S_IRUGO,
 196                 amdgpu_device_get_product_number, NULL);
 197
 198 /**
 199  * DOC: serial_number
 200  *
 201  * The amdgpu driver provides a sysfs API for reporting the serial number
 202  * for the device
 203  * The file serial_number is used for this and returns the serial number
 204  * as returned from the FRU.
 205  * NOTE: This is only available for certain server cards
 206  */
 207
 208 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 209                 struct device_attribute *attr, char *buf)
 210 {
 211         struct drm_device *ddev = dev_get_drvdata(dev);
 212         struct amdgpu_device *adev = drm_to_adev(ddev);
 213
 214         return sysfs_emit(buf, "%s\n", adev->serial);
 215 }
 216
 217 static DEVICE_ATTR(serial_number, S_IRUGO,
 218                 amdgpu_device_get_serial_number, NULL);
 219
 220 /**
 221  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 222  *
 223  * @dev: drm_device pointer
 224  *
 225  * Returns true if the device is a dGPU with ATPX power control,
 226  * otherwise return false.
 227  */
 228 bool amdgpu_device_supports_px(struct drm_device *dev)
 229 {
 230         struct amdgpu_device *adev = drm_to_adev(dev);
 231
 232         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 233                 return true;
 234         return false;
 235 }
 236
 237 /**
 238  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 239  *
 240  * @dev: drm_device pointer
 241  *
 242  * Returns true if the device is a dGPU with ACPI power control,
 243  * otherwise return false.
 244  */
 245 bool amdgpu_device_supports_boco(struct drm_device *dev)
 246 {
 247         struct amdgpu_device *adev = drm_to_adev(dev);
 248
 249         if (adev->has_pr3 ||
 250             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 251                 return true;
 252         return false;
 253 }
 254
 255 /**
 256  * amdgpu_device_supports_baco - Does the device support BACO
 257  *
 258  * @dev: drm_device pointer
 259  *
 260  * Returns true if the device supporte BACO,
 261  * otherwise return false.
 262  */
 263 bool amdgpu_device_supports_baco(struct drm_device *dev)
 264 {
 265         struct amdgpu_device *adev = drm_to_adev(dev);
 266
 267         return amdgpu_asic_supports_baco(adev);
 268 }
 269
 270 /**
 271  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 272  * smart shift support
 273  *
 274  * @dev: drm_device pointer
 275  *
 276  * Returns true if the device is a dGPU with Smart Shift support,
 277  * otherwise returns false.
 278  */
 279 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 280 {
 281         return (amdgpu_device_supports_boco(dev) &&
 282                 amdgpu_acpi_is_power_shift_control_supported());
 283 }
 284
 285 /*
 286  * VRAM access helper functions
 287  */
 288
 289 /**
 290  * amdgpu_device_vram_access - read/write a buffer in vram
 291  *
 292  * @adev: amdgpu_device pointer
 293  * @pos: offset of the buffer in vram
 294  * @buf: virtual address of the buffer in system memory
 295  * @size: read/write size, sizeof(@buf) must > @size
 296  * @write: true - write to vram, otherwise - read from vram
 297  */
 298 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 299                                uint32_t *buf, size_t size, bool write)
 300 {
 301         unsigned long flags;
 302         uint32_t hi = ~0;
 303         uint64_t last;
 304         int idx;
 305
 306         if (!drm_dev_enter(&adev->ddev, &idx))
 307                 return;
 308
 309 #ifdef CONFIG_64BIT
 310         last = min(pos + size, adev->gmc.visible_vram_size);
 311         if (last > pos) {
 312                 void __iomem *addr = adev->mman.aper_base_kaddr + pos;
 313                 size_t count = last - pos;
 314
 315                 if (write) {
 316                         memcpy_toio(addr, buf, count);
 317                         mb();
 318                         amdgpu_device_flush_hdp(adev, NULL);
 319                 } else {
 320                         amdgpu_device_invalidate_hdp(adev, NULL);
 321                         mb();
 322                         memcpy_fromio(buf, addr, count);
 323                 }
 324
 325                 if (count == size)
 326                         goto exit;
 327
 328                 pos += count;
 329                 buf += count / 4;
 330                 size -= count;
 331         }
 332 #endif
 333
 334         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 335         for (last = pos + size; pos < last; pos += 4) {
 336                 uint32_t tmp = pos >> 31;
 337
 338                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 339                 if (tmp != hi) {
 340                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 341                         hi = tmp;
 342                 }
 343                 if (write)
 344                         WREG32_NO_KIQ(mmMM_DATA, *buf++);
 345                 else
 346                         *buf++ = RREG32_NO_KIQ(mmMM_DATA);
 347         }
 348         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 349
 350 #ifdef CONFIG_64BIT
 351 exit:
 352 #endif
 353         drm_dev_exit(idx);
 354 }
 355
 356 /*
 357  * register access helper functions.
 358  */
 359
 360 /* Check if hw access should be skipped because of hotplug or device error */
 361 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 362 {
 363         if (adev->no_hw_access)
 364                 return true;
 365
 366 #ifdef CONFIG_LOCKDEP
 367         /*
 368          * This is a bit complicated to understand, so worth a comment. What we assert
 369          * here is that the GPU reset is not running on another thread in parallel.
 370          *
 371          * For this we trylock the read side of the reset semaphore, if that succeeds
 372          * we know that the reset is not running in paralell.
 373          *
 374          * If the trylock fails we assert that we are either already holding the read
 375          * side of the lock or are the reset thread itself and hold the write side of
 376          * the lock.
 377          */
 378         if (in_task()) {
 379                 if (down_read_trylock(&adev->reset_sem))
 380                         up_read(&adev->reset_sem);
 381                 else
 382                         lockdep_assert_held(&adev->reset_sem);
 383         }
 384 #endif
 385         return false;
 386 }
 387
 388 /**
 389  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 390  *
 391  * @adev: amdgpu_device pointer
 392  * @reg: dword aligned register offset
 393  * @acc_flags: access flags which require special behavior
 394  *
 395  * Returns the 32 bit value from the offset specified.
 396  */
 397 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 398                             uint32_t reg, uint32_t acc_flags)
 399 {
 400         uint32_t ret;
 401
 402         if (amdgpu_device_skip_hw_access(adev))
 403                 return 0;
 404
 405         if ((reg * 4) < adev->rmmio_size) {
 406                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 407                     amdgpu_sriov_runtime(adev) &&
 408                     down_read_trylock(&adev->reset_sem)) {
 409                         ret = amdgpu_kiq_rreg(adev, reg);
 410                         up_read(&adev->reset_sem);
 411                 } else {
 412                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 413                 }
 414         } else {
 415                 ret = adev->pcie_rreg(adev, reg * 4);
 416         }
 417
 418         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 419
 420         return ret;
 421 }
 422
 423 /*
 424  * MMIO register read with bytes helper functions
 425  * @offset:bytes offset from MMIO start
 426  *
 427 */
 428
 429 /**
 430  * amdgpu_mm_rreg8 - read a memory mapped IO register
 431  *
 432  * @adev: amdgpu_device pointer
 433  * @offset: byte aligned register offset
 434  *
 435  * Returns the 8 bit value from the offset specified.
 436  */
 437 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 438 {
 439         if (amdgpu_device_skip_hw_access(adev))
 440                 return 0;
 441
 442         if (offset < adev->rmmio_size)
 443                 return (readb(adev->rmmio + offset));
 444         BUG();
 445 }
 446
 447 /*
 448  * MMIO register write with bytes helper functions
 449  * @offset:bytes offset from MMIO start
 450  * @value: the value want to be written to the register
 451  *
 452 */
 453 /**
 454  * amdgpu_mm_wreg8 - read a memory mapped IO register
 455  *
 456  * @adev: amdgpu_device pointer
 457  * @offset: byte aligned register offset
 458  * @value: 8 bit value to write
 459  *
 460  * Writes the value specified to the offset specified.
 461  */
 462 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 463 {
 464         if (amdgpu_device_skip_hw_access(adev))
 465                 return;
 466
 467         if (offset < adev->rmmio_size)
 468                 writeb(value, adev->rmmio + offset);
 469         else
 470                 BUG();
 471 }
 472
 473 /**
 474  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 475  *
 476  * @adev: amdgpu_device pointer
 477  * @reg: dword aligned register offset
 478  * @v: 32 bit value to write to the register
 479  * @acc_flags: access flags which require special behavior
 480  *
 481  * Writes the value specified to the offset specified.
 482  */
 483 void amdgpu_device_wreg(struct amdgpu_device *adev,
 484                         uint32_t reg, uint32_t v,
 485                         uint32_t acc_flags)
 486 {
 487         if (amdgpu_device_skip_hw_access(adev))
 488                 return;
 489
 490         if ((reg * 4) < adev->rmmio_size) {
 491                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 492                     amdgpu_sriov_runtime(adev) &&
 493                     down_read_trylock(&adev->reset_sem)) {
 494                         amdgpu_kiq_wreg(adev, reg, v);
 495                         up_read(&adev->reset_sem);
 496                 } else {
 497                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 498                 }
 499         } else {
 500                 adev->pcie_wreg(adev, reg * 4, v);
 501         }
 502
 503         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 504 }
 505
 506 /*
 507  * amdgpu_mm_wreg_mmio_rlc -  write register either with mmio or with RLC path if in range
 508  *
 509  * this function is invoked only the debugfs register access
 510  * */
 511 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 512                              uint32_t reg, uint32_t v)
 513 {
 514         if (amdgpu_device_skip_hw_access(adev))
 515                 return;
 516
 517         if (amdgpu_sriov_fullaccess(adev) &&
 518             adev->gfx.rlc.funcs &&
 519             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 520                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 521                         return adev->gfx.rlc.funcs->rlcg_wreg(adev, reg, v, 0, 0);
 522         } else {
 523                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 524         }
 525 }
 526
 527 /**
 528  * amdgpu_mm_rdoorbell - read a doorbell dword
 529  *
 530  * @adev: amdgpu_device pointer
 531  * @index: doorbell index
 532  *
 533  * Returns the value in the doorbell aperture at the
 534  * requested doorbell index (CIK).
 535  */
 536 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 537 {
 538         if (amdgpu_device_skip_hw_access(adev))
 539                 return 0;
 540
 541         if (index < adev->doorbell.num_doorbells) {
 542                 return readl(adev->doorbell.ptr + index);
 543         } else {
 544                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 545                 return 0;
 546         }
 547 }
 548
 549 /**
 550  * amdgpu_mm_wdoorbell - write a doorbell dword
 551  *
 552  * @adev: amdgpu_device pointer
 553  * @index: doorbell index
 554  * @v: value to write
 555  *
 556  * Writes @v to the doorbell aperture at the
 557  * requested doorbell index (CIK).
 558  */
 559 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 560 {
 561         if (amdgpu_device_skip_hw_access(adev))
 562                 return;
 563
 564         if (index < adev->doorbell.num_doorbells) {
 565                 writel(v, adev->doorbell.ptr + index);
 566         } else {
 567                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 568         }
 569 }
 570
 571 /**
 572  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 573  *
 574  * @adev: amdgpu_device pointer
 575  * @index: doorbell index
 576  *
 577  * Returns the value in the doorbell aperture at the
 578  * requested doorbell index (VEGA10+).
 579  */
 580 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 581 {
 582         if (amdgpu_device_skip_hw_access(adev))
 583                 return 0;
 584
 585         if (index < adev->doorbell.num_doorbells) {
 586                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 587         } else {
 588                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 589                 return 0;
 590         }
 591 }
 592
 593 /**
 594  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 595  *
 596  * @adev: amdgpu_device pointer
 597  * @index: doorbell index
 598  * @v: value to write
 599  *
 600  * Writes @v to the doorbell aperture at the
 601  * requested doorbell index (VEGA10+).
 602  */
 603 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 604 {
 605         if (amdgpu_device_skip_hw_access(adev))
 606                 return;
 607
 608         if (index < adev->doorbell.num_doorbells) {
 609                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 610         } else {
 611                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 612         }
 613 }
 614
 615 /**
 616  * amdgpu_device_indirect_rreg - read an indirect register
 617  *
 618  * @adev: amdgpu_device pointer
 619  * @pcie_index: mmio register offset
 620  * @pcie_data: mmio register offset
 621  * @reg_addr: indirect register address to read from
 622  *
 623  * Returns the value of indirect register @reg_addr
 624  */
 625 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 626                                 u32 pcie_index, u32 pcie_data,
 627                                 u32 reg_addr)
 628 {
 629         unsigned long flags;
 630         u32 r;
 631         void __iomem *pcie_index_offset;
 632         void __iomem *pcie_data_offset;
 633
 634         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 635         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 636         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 637
 638         writel(reg_addr, pcie_index_offset);
 639         readl(pcie_index_offset);
 640         r = readl(pcie_data_offset);
 641         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 642
 643         return r;
 644 }
 645
 646 /**
 647  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 648  *
 649  * @adev: amdgpu_device pointer
 650  * @pcie_index: mmio register offset
 651  * @pcie_data: mmio register offset
 652  * @reg_addr: indirect register address to read from
 653  *
 654  * Returns the value of indirect register @reg_addr
 655  */
 656 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 657                                   u32 pcie_index, u32 pcie_data,
 658                                   u32 reg_addr)
 659 {
 660         unsigned long flags;
 661         u64 r;
 662         void __iomem *pcie_index_offset;
 663         void __iomem *pcie_data_offset;
 664
 665         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 666         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 667         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 668
 669         /* read low 32 bits */
 670         writel(reg_addr, pcie_index_offset);
 671         readl(pcie_index_offset);
 672         r = readl(pcie_data_offset);
 673         /* read high 32 bits */
 674         writel(reg_addr + 4, pcie_index_offset);
 675         readl(pcie_index_offset);
 676         r |= ((u64)readl(pcie_data_offset) << 32);
 677         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 678
 679         return r;
 680 }
 681
 682 /**
 683  * amdgpu_device_indirect_wreg - write an indirect register address
 684  *
 685  * @adev: amdgpu_device pointer
 686  * @pcie_index: mmio register offset
 687  * @pcie_data: mmio register offset
 688  * @reg_addr: indirect register offset
 689  * @reg_data: indirect register data
 690  *
 691  */
 692 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 693                                  u32 pcie_index, u32 pcie_data,
 694                                  u32 reg_addr, u32 reg_data)
 695 {
 696         unsigned long flags;
 697         void __iomem *pcie_index_offset;
 698         void __iomem *pcie_data_offset;
 699
 700         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 701         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 702         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 703
 704         writel(reg_addr, pcie_index_offset);
 705         readl(pcie_index_offset);
 706         writel(reg_data, pcie_data_offset);
 707         readl(pcie_data_offset);
 708         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 709 }
 710
 711 /**
 712  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 713  *
 714  * @adev: amdgpu_device pointer
 715  * @pcie_index: mmio register offset
 716  * @pcie_data: mmio register offset
 717  * @reg_addr: indirect register offset
 718  * @reg_data: indirect register data
 719  *
 720  */
 721 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 722                                    u32 pcie_index, u32 pcie_data,
 723                                    u32 reg_addr, u64 reg_data)
 724 {
 725         unsigned long flags;
 726         void __iomem *pcie_index_offset;
 727         void __iomem *pcie_data_offset;
 728
 729         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 730         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 731         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 732
 733         /* write low 32 bits */
 734         writel(reg_addr, pcie_index_offset);
 735         readl(pcie_index_offset);
 736         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 737         readl(pcie_data_offset);
 738         /* write high 32 bits */
 739         writel(reg_addr + 4, pcie_index_offset);
 740         readl(pcie_index_offset);
 741         writel((u32)(reg_data >> 32), pcie_data_offset);
 742         readl(pcie_data_offset);
 743         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 744 }
 745
 746 /**
 747  * amdgpu_invalid_rreg - dummy reg read function
 748  *
 749  * @adev: amdgpu_device pointer
 750  * @reg: offset of register
 751  *
 752  * Dummy register read function.  Used for register blocks
 753  * that certain asics don't have (all asics).
 754  * Returns the value in the register.
 755  */
 756 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 757 {
 758         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 759         BUG();
 760         return 0;
 761 }
 762
 763 /**
 764  * amdgpu_invalid_wreg - dummy reg write function
 765  *
 766  * @adev: amdgpu_device pointer
 767  * @reg: offset of register
 768  * @v: value to write to the register
 769  *
 770  * Dummy register read function.  Used for register blocks
 771  * that certain asics don't have (all asics).
 772  */
 773 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 774 {
 775         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 776                   reg, v);
 777         BUG();
 778 }
 779
 780 /**
 781  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 782  *
 783  * @adev: amdgpu_device pointer
 784  * @reg: offset of register
 785  *
 786  * Dummy register read function.  Used for register blocks
 787  * that certain asics don't have (all asics).
 788  * Returns the value in the register.
 789  */
 790 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 791 {
 792         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 793         BUG();
 794         return 0;
 795 }
 796
 797 /**
 798  * amdgpu_invalid_wreg64 - dummy reg write function
 799  *
 800  * @adev: amdgpu_device pointer
 801  * @reg: offset of register
 802  * @v: value to write to the register
 803  *
 804  * Dummy register read function.  Used for register blocks
 805  * that certain asics don't have (all asics).
 806  */
 807 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 808 {
 809         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 810                   reg, v);
 811         BUG();
 812 }
 813
 814 /**
 815  * amdgpu_block_invalid_rreg - dummy reg read function
 816  *
 817  * @adev: amdgpu_device pointer
 818  * @block: offset of instance
 819  * @reg: offset of register
 820  *
 821  * Dummy register read function.  Used for register blocks
 822  * that certain asics don't have (all asics).
 823  * Returns the value in the register.
 824  */
 825 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 826                                           uint32_t block, uint32_t reg)
 827 {
 828         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 829                   reg, block);
 830         BUG();
 831         return 0;
 832 }
 833
 834 /**
 835  * amdgpu_block_invalid_wreg - dummy reg write function
 836  *
 837  * @adev: amdgpu_device pointer
 838  * @block: offset of instance
 839  * @reg: offset of register
 840  * @v: value to write to the register
 841  *
 842  * Dummy register read function.  Used for register blocks
 843  * that certain asics don't have (all asics).
 844  */
 845 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 846                                       uint32_t block,
 847                                       uint32_t reg, uint32_t v)
 848 {
 849         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 850                   reg, block, v);
 851         BUG();
 852 }
 853
 854 /**
 855  * amdgpu_device_asic_init - Wrapper for atom asic_init
 856  *
 857  * @adev: amdgpu_device pointer
 858  *
 859  * Does any asic specific work and then calls atom asic init.
 860  */
 861 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 862 {
 863         amdgpu_asic_pre_asic_init(adev);
 864
 865         return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 866 }
 867
 868 /**
 869  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 870  *
 871  * @adev: amdgpu_device pointer
 872  *
 873  * Allocates a scratch page of VRAM for use by various things in the
 874  * driver.
 875  */
 876 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 877 {
 878         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 879                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 880                                        &adev->vram_scratch.robj,
 881                                        &adev->vram_scratch.gpu_addr,
 882                                        (void **)&adev->vram_scratch.ptr);
 883 }
 884
 885 /**
 886  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 887  *
 888  * @adev: amdgpu_device pointer
 889  *
 890  * Frees the VRAM scratch page.
 891  */
 892 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 893 {
 894         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 895 }
 896
 897 /**
 898  * amdgpu_device_program_register_sequence - program an array of registers.
 899  *
 900  * @adev: amdgpu_device pointer
 901  * @registers: pointer to the register array
 902  * @array_size: size of the register array
 903  *
 904  * Programs an array or registers with and and or masks.
 905  * This is a helper for setting golden registers.
 906  */
 907 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 908                                              const u32 *registers,
 909                                              const u32 array_size)
 910 {
 911         u32 tmp, reg, and_mask, or_mask;
 912         int i;
 913
 914         if (array_size % 3)
 915                 return;
 916
 917         for (i = 0; i < array_size; i +=3) {
 918                 reg = registers[i + 0];
 919                 and_mask = registers[i + 1];
 920                 or_mask = registers[i + 2];
 921
 922                 if (and_mask == 0xffffffff) {
 923                         tmp = or_mask;
 924                 } else {
 925                         tmp = RREG32(reg);
 926                         tmp &= ~and_mask;
 927                         if (adev->family >= AMDGPU_FAMILY_AI)
 928                                 tmp |= (or_mask & and_mask);
 929                         else
 930                                 tmp |= or_mask;
 931                 }
 932                 WREG32(reg, tmp);
 933         }
 934 }
 935
 936 /**
 937  * amdgpu_device_pci_config_reset - reset the GPU
 938  *
 939  * @adev: amdgpu_device pointer
 940  *
 941  * Resets the GPU using the pci config reset sequence.
 942  * Only applicable to asics prior to vega10.
 943  */
 944 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
 945 {
 946         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
 947 }
 948
 949 /**
 950  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
 951  *
 952  * @adev: amdgpu_device pointer
 953  *
 954  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
 955  */
 956 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
 957 {
 958         return pci_reset_function(adev->pdev);
 959 }
 960
 961 /*
 962  * GPU doorbell aperture helpers function.
 963  */
 964 /**
 965  * amdgpu_device_doorbell_init - Init doorbell driver information.
 966  *
 967  * @adev: amdgpu_device pointer
 968  *
 969  * Init doorbell driver information (CIK)
 970  * Returns 0 on success, error on failure.
 971  */
 972 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
 973 {
 974
 975         /* No doorbell on SI hardware generation */
 976         if (adev->asic_type < CHIP_BONAIRE) {
 977                 adev->doorbell.base = 0;
 978                 adev->doorbell.size = 0;
 979                 adev->doorbell.num_doorbells = 0;
 980                 adev->doorbell.ptr = NULL;
 981                 return 0;
 982         }
 983
 984         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
 985                 return -EINVAL;
 986
 987         amdgpu_asic_init_doorbell_index(adev);
 988
 989         /* doorbell bar mapping */
 990         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
 991         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
 992
 993         adev->doorbell.num_doorbells = min_t(u32, adev->doorbell.size / sizeof(u32),
 994                                              adev->doorbell_index.max_assignment+1);
 995         if (adev->doorbell.num_doorbells == 0)
 996                 return -EINVAL;
 997
 998         /* For Vega, reserve and map two pages on doorbell BAR since SDMA
 999          * paging queue doorbell use the second page. The
1000          * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1001          * doorbells are in the first page. So with paging queue enabled,
1002          * the max num_doorbells should + 1 page (0x400 in dword)
1003          */
1004         if (adev->asic_type >= CHIP_VEGA10)
1005                 adev->doorbell.num_doorbells += 0x400;
1006
1007         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1008                                      adev->doorbell.num_doorbells *
1009                                      sizeof(u32));
1010         if (adev->doorbell.ptr == NULL)
1011                 return -ENOMEM;
1012
1013         return 0;
1014 }
1015
1016 /**
1017  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1018  *
1019  * @adev: amdgpu_device pointer
1020  *
1021  * Tear down doorbell driver information (CIK)
1022  */
1023 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1024 {
1025         iounmap(adev->doorbell.ptr);
1026         adev->doorbell.ptr = NULL;
1027 }
1028
1029
1030
1031 /*
1032  * amdgpu_device_wb_*()
1033  * Writeback is the method by which the GPU updates special pages in memory
1034  * with the status of certain GPU events (fences, ring pointers,etc.).
1035  */
1036
1037 /**
1038  * amdgpu_device_wb_fini - Disable Writeback and free memory
1039  *
1040  * @adev: amdgpu_device pointer
1041  *
1042  * Disables Writeback and frees the Writeback memory (all asics).
1043  * Used at driver shutdown.
1044  */
1045 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1046 {
1047         if (adev->wb.wb_obj) {
1048                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1049                                       &adev->wb.gpu_addr,
1050                                       (void **)&adev->wb.wb);
1051                 adev->wb.wb_obj = NULL;
1052         }
1053 }
1054
1055 /**
1056  * amdgpu_device_wb_init- Init Writeback driver info and allocate memory
1057  *
1058  * @adev: amdgpu_device pointer
1059  *
1060  * Initializes writeback and allocates writeback memory (all asics).
1061  * Used at driver startup.
1062  * Returns 0 on success or an -error on failure.
1063  */
1064 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1065 {
1066         int r;
1067
1068         if (adev->wb.wb_obj == NULL) {
1069                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1070                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1071                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1072                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1073                                             (void **)&adev->wb.wb);
1074                 if (r) {
1075                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1076                         return r;
1077                 }
1078
1079                 adev->wb.num_wb = AMDGPU_MAX_WB;
1080                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1081
1082                 /* clear wb memory */
1083                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1084         }
1085
1086         return 0;
1087 }
1088
1089 /**
1090  * amdgpu_device_wb_get - Allocate a wb entry
1091  *
1092  * @adev: amdgpu_device pointer
1093  * @wb: wb index
1094  *
1095  * Allocate a wb slot for use by the driver (all asics).
1096  * Returns 0 on success or -EINVAL on failure.
1097  */
1098 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1099 {
1100         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1101
1102         if (offset < adev->wb.num_wb) {
1103                 __set_bit(offset, adev->wb.used);
1104                 *wb = offset << 3; /* convert to dw offset */
1105                 return 0;
1106         } else {
1107                 return -EINVAL;
1108         }
1109 }
1110
1111 /**
1112  * amdgpu_device_wb_free - Free a wb entry
1113  *
1114  * @adev: amdgpu_device pointer
1115  * @wb: wb index
1116  *
1117  * Free a wb slot allocated for use by the driver (all asics)
1118  */
1119 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1120 {
1121         wb >>= 3;
1122         if (wb < adev->wb.num_wb)
1123                 __clear_bit(wb, adev->wb.used);
1124 }
1125
1126 /**
1127  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1128  *
1129  * @adev: amdgpu_device pointer
1130  *
1131  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1132  * to fail, but if any of the BARs is not accessible after the size we abort
1133  * driver loading by returning -ENODEV.
1134  */
1135 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1136 {
1137         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1138         struct pci_bus *root;
1139         struct resource *res;
1140         unsigned i;
1141         u16 cmd;
1142         int r;
1143
1144         /* Bypass for VF */
1145         if (amdgpu_sriov_vf(adev))
1146                 return 0;
1147
1148         /* skip if the bios has already enabled large BAR */
1149         if (adev->gmc.real_vram_size &&
1150             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1151                 return 0;
1152
1153         /* Check if the root BUS has 64bit memory resources */
1154         root = adev->pdev->bus;
1155         while (root->parent)
1156                 root = root->parent;
1157
1158         pci_bus_for_each_resource(root, res, i) {
1159                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1160                     res->start > 0x100000000ull)
1161                         break;
1162         }
1163
1164         /* Trying to resize is pointless without a root hub window above 4GB */
1165         if (!res)
1166                 return 0;
1167
1168         /* Limit the BAR size to what is available */
1169         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1170                         rbar_size);
1171
1172         /* Disable memory decoding while we change the BAR addresses and size */
1173         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1174         pci_write_config_word(adev->pdev, PCI_COMMAND,
1175                               cmd & ~PCI_COMMAND_MEMORY);
1176
1177         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1178         amdgpu_device_doorbell_fini(adev);
1179         if (adev->asic_type >= CHIP_BONAIRE)
1180                 pci_release_resource(adev->pdev, 2);
1181
1182         pci_release_resource(adev->pdev, 0);
1183
1184         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1185         if (r == -ENOSPC)
1186                 DRM_INFO("Not enough PCI address space for a large BAR.");
1187         else if (r && r != -ENOTSUPP)
1188                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1189
1190         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1191
1192         /* When the doorbell or fb BAR isn't available we have no chance of
1193          * using the device.
1194          */
1195         r = amdgpu_device_doorbell_init(adev);
1196         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1197                 return -ENODEV;
1198
1199         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1200
1201         return 0;
1202 }
1203
1204 /*
1205  * GPU helpers function.
1206  */
1207 /**
1208  * amdgpu_device_need_post - check if the hw need post or not
1209  *
1210  * @adev: amdgpu_device pointer
1211  *
1212  * Check if the asic has been initialized (all asics) at driver startup
1213  * or post is needed if  hw reset is performed.
1214  * Returns true if need or false if not.
1215  */
1216 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1217 {
1218         uint32_t reg;
1219
1220         if (amdgpu_sriov_vf(adev))
1221                 return false;
1222
1223         if (amdgpu_passthrough(adev)) {
1224                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1225                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1226                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1227                  * vpost executed for smc version below 22.15
1228                  */
1229                 if (adev->asic_type == CHIP_FIJI) {
1230                         int err;
1231                         uint32_t fw_ver;
1232                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1233                         /* force vPost if error occured */
1234                         if (err)
1235                                 return true;
1236
1237                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1238                         if (fw_ver < 0x00160e00)
1239                                 return true;
1240                 }
1241         }
1242
1243         /* Don't post if we need to reset whole hive on init */
1244         if (adev->gmc.xgmi.pending_reset)
1245                 return false;
1246
1247         if (adev->has_hw_reset) {
1248                 adev->has_hw_reset = false;
1249                 return true;
1250         }
1251
1252         /* bios scratch used on CIK+ */
1253         if (adev->asic_type >= CHIP_BONAIRE)
1254                 return amdgpu_atombios_scratch_need_asic_init(adev);
1255
1256         /* check MEM_SIZE for older asics */
1257         reg = amdgpu_asic_get_config_memsize(adev);
1258
1259         if ((reg != 0) && (reg != 0xffffffff))
1260                 return false;
1261
1262         return true;
1263 }
1264
1265 /* if we get transitioned to only one device, take VGA back */
1266 /**
1267  * amdgpu_device_vga_set_decode - enable/disable vga decode
1268  *
1269  * @cookie: amdgpu_device pointer
1270  * @state: enable/disable vga decode
1271  *
1272  * Enable/disable vga decode (all asics).
1273  * Returns VGA resource flags.
1274  */
1275 static unsigned int amdgpu_device_vga_set_decode(void *cookie, bool state)
1276 {
1277         struct amdgpu_device *adev = cookie;
1278         amdgpu_asic_set_vga_state(adev, state);
1279         if (state)
1280                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1281                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1282         else
1283                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1284 }
1285
1286 /**
1287  * amdgpu_device_check_block_size - validate the vm block size
1288  *
1289  * @adev: amdgpu_device pointer
1290  *
1291  * Validates the vm block size specified via module parameter.
1292  * The vm block size defines number of bits in page table versus page directory,
1293  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1294  * page table and the remaining bits are in the page directory.
1295  */
1296 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1297 {
1298         /* defines number of bits in page table versus page directory,
1299          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1300          * page table and the remaining bits are in the page directory */
1301         if (amdgpu_vm_block_size == -1)
1302                 return;
1303
1304         if (amdgpu_vm_block_size < 9) {
1305                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1306                          amdgpu_vm_block_size);
1307                 amdgpu_vm_block_size = -1;
1308         }
1309 }
1310
1311 /**
1312  * amdgpu_device_check_vm_size - validate the vm size
1313  *
1314  * @adev: amdgpu_device pointer
1315  *
1316  * Validates the vm size in GB specified via module parameter.
1317  * The VM size is the size of the GPU virtual memory space in GB.
1318  */
1319 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1320 {
1321         /* no need to check the default value */
1322         if (amdgpu_vm_size == -1)
1323                 return;
1324
1325         if (amdgpu_vm_size < 1) {
1326                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1327                          amdgpu_vm_size);
1328                 amdgpu_vm_size = -1;
1329         }
1330 }
1331
1332 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1333 {
1334         struct sysinfo si;
1335         bool is_os_64 = (sizeof(void *) == 8);
1336         uint64_t total_memory;
1337         uint64_t dram_size_seven_GB = 0x1B8000000;
1338         uint64_t dram_size_three_GB = 0xB8000000;
1339
1340         if (amdgpu_smu_memory_pool_size == 0)
1341                 return;
1342
1343         if (!is_os_64) {
1344                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1345                 goto def_value;
1346         }
1347         si_meminfo(&si);
1348         total_memory = (uint64_t)si.totalram * si.mem_unit;
1349
1350         if ((amdgpu_smu_memory_pool_size == 1) ||
1351                 (amdgpu_smu_memory_pool_size == 2)) {
1352                 if (total_memory < dram_size_three_GB)
1353                         goto def_value1;
1354         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1355                 (amdgpu_smu_memory_pool_size == 8)) {
1356                 if (total_memory < dram_size_seven_GB)
1357                         goto def_value1;
1358         } else {
1359                 DRM_WARN("Smu memory pool size not supported\n");
1360                 goto def_value;
1361         }
1362         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1363
1364         return;
1365
1366 def_value1:
1367         DRM_WARN("No enough system memory\n");
1368 def_value:
1369         adev->pm.smu_prv_buffer_size = 0;
1370 }
1371
1372 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1373 {
1374         if (!(adev->flags & AMD_IS_APU) ||
1375             adev->asic_type < CHIP_RAVEN)
1376                 return 0;
1377
1378         switch (adev->asic_type) {
1379         case CHIP_RAVEN:
1380                 if (adev->pdev->device == 0x15dd)
1381                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1382                 if (adev->pdev->device == 0x15d8)
1383                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1384                 break;
1385         case CHIP_RENOIR:
1386                 if ((adev->pdev->device == 0x1636) ||
1387                     (adev->pdev->device == 0x164c))
1388                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1389                 else
1390                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1391                 break;
1392         case CHIP_VANGOGH:
1393                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1394                 break;
1395         case CHIP_YELLOW_CARP:
1396                 break;
1397         default:
1398                 return -EINVAL;
1399         }
1400
1401         return 0;
1402 }
1403
1404 /**
1405  * amdgpu_device_check_arguments - validate module params
1406  *
1407  * @adev: amdgpu_device pointer
1408  *
1409  * Validates certain module parameters and updates
1410  * the associated values used by the driver (all asics).
1411  */
1412 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1413 {
1414         if (amdgpu_sched_jobs < 4) {
1415                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1416                          amdgpu_sched_jobs);
1417                 amdgpu_sched_jobs = 4;
1418         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1419                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1420                          amdgpu_sched_jobs);
1421                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1422         }
1423
1424         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1425                 /* gart size must be greater or equal to 32M */
1426                 dev_warn(adev->dev, "gart size (%d) too small\n",
1427                          amdgpu_gart_size);
1428                 amdgpu_gart_size = -1;
1429         }
1430
1431         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1432                 /* gtt size must be greater or equal to 32M */
1433                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1434                                  amdgpu_gtt_size);
1435                 amdgpu_gtt_size = -1;
1436         }
1437
1438         /* valid range is between 4 and 9 inclusive */
1439         if (amdgpu_vm_fragment_size != -1 &&
1440             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1441                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1442                 amdgpu_vm_fragment_size = -1;
1443         }
1444
1445         if (amdgpu_sched_hw_submission < 2) {
1446                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1447                          amdgpu_sched_hw_submission);
1448                 amdgpu_sched_hw_submission = 2;
1449         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1450                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1451                          amdgpu_sched_hw_submission);
1452                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1453         }
1454
1455         amdgpu_device_check_smu_prv_buffer_size(adev);
1456
1457         amdgpu_device_check_vm_size(adev);
1458
1459         amdgpu_device_check_block_size(adev);
1460
1461         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1462
1463         amdgpu_gmc_tmz_set(adev);
1464
1465         amdgpu_gmc_noretry_set(adev);
1466
1467         return 0;
1468 }
1469
1470 /**
1471  * amdgpu_switcheroo_set_state - set switcheroo state
1472  *
1473  * @pdev: pci dev pointer
1474  * @state: vga_switcheroo state
1475  *
1476  * Callback for the switcheroo driver.  Suspends or resumes the
1477  * the asics before or after it is powered up using ACPI methods.
1478  */
1479 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1480                                         enum vga_switcheroo_state state)
1481 {
1482         struct drm_device *dev = pci_get_drvdata(pdev);
1483         int r;
1484
1485         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1486                 return;
1487
1488         if (state == VGA_SWITCHEROO_ON) {
1489                 pr_info("switched on\n");
1490                 /* don't suspend or resume card normally */
1491                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1492
1493                 pci_set_power_state(pdev, PCI_D0);
1494                 amdgpu_device_load_pci_state(pdev);
1495                 r = pci_enable_device(pdev);
1496                 if (r)
1497                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1498                 amdgpu_device_resume(dev, true);
1499
1500                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1501         } else {
1502                 pr_info("switched off\n");
1503                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1504                 amdgpu_device_suspend(dev, true);
1505                 amdgpu_device_cache_pci_state(pdev);
1506                 /* Shut down the device */
1507                 pci_disable_device(pdev);
1508                 pci_set_power_state(pdev, PCI_D3cold);
1509                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1510         }
1511 }
1512
1513 /**
1514  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1515  *
1516  * @pdev: pci dev pointer
1517  *
1518  * Callback for the switcheroo driver.  Check of the switcheroo
1519  * state can be changed.
1520  * Returns true if the state can be changed, false if not.
1521  */
1522 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1523 {
1524         struct drm_device *dev = pci_get_drvdata(pdev);
1525
1526         /*
1527         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1528         * locking inversion with the driver load path. And the access here is
1529         * completely racy anyway. So don't bother with locking for now.
1530         */
1531         return atomic_read(&dev->open_count) == 0;
1532 }
1533
1534 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1535         .set_gpu_state = amdgpu_switcheroo_set_state,
1536         .reprobe = NULL,
1537         .can_switch = amdgpu_switcheroo_can_switch,
1538 };
1539
1540 /**
1541  * amdgpu_device_ip_set_clockgating_state - set the CG state
1542  *
1543  * @dev: amdgpu_device pointer
1544  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1545  * @state: clockgating state (gate or ungate)
1546  *
1547  * Sets the requested clockgating state for all instances of
1548  * the hardware IP specified.
1549  * Returns the error code from the last instance.
1550  */
1551 int amdgpu_device_ip_set_clockgating_state(void *dev,
1552                                            enum amd_ip_block_type block_type,
1553                                            enum amd_clockgating_state state)
1554 {
1555         struct amdgpu_device *adev = dev;
1556         int i, r = 0;
1557
1558         for (i = 0; i < adev->num_ip_blocks; i++) {
1559                 if (!adev->ip_blocks[i].status.valid)
1560                         continue;
1561                 if (adev->ip_blocks[i].version->type != block_type)
1562                         continue;
1563                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1564                         continue;
1565                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1566                         (void *)adev, state);
1567                 if (r)
1568                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1569                                   adev->ip_blocks[i].version->funcs->name, r);
1570         }
1571         return r;
1572 }
1573
1574 /**
1575  * amdgpu_device_ip_set_powergating_state - set the PG state
1576  *
1577  * @dev: amdgpu_device pointer
1578  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1579  * @state: powergating state (gate or ungate)
1580  *
1581  * Sets the requested powergating state for all instances of
1582  * the hardware IP specified.
1583  * Returns the error code from the last instance.
1584  */
1585 int amdgpu_device_ip_set_powergating_state(void *dev,
1586                                            enum amd_ip_block_type block_type,
1587                                            enum amd_powergating_state state)
1588 {
1589         struct amdgpu_device *adev = dev;
1590         int i, r = 0;
1591
1592         for (i = 0; i < adev->num_ip_blocks; i++) {
1593                 if (!adev->ip_blocks[i].status.valid)
1594                         continue;
1595                 if (adev->ip_blocks[i].version->type != block_type)
1596                         continue;
1597                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1598                         continue;
1599                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1600                         (void *)adev, state);
1601                 if (r)
1602                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1603                                   adev->ip_blocks[i].version->funcs->name, r);
1604         }
1605         return r;
1606 }
1607
1608 /**
1609  * amdgpu_device_ip_get_clockgating_state - get the CG state
1610  *
1611  * @adev: amdgpu_device pointer
1612  * @flags: clockgating feature flags
1613  *
1614  * Walks the list of IPs on the device and updates the clockgating
1615  * flags for each IP.
1616  * Updates @flags with the feature flags for each hardware IP where
1617  * clockgating is enabled.
1618  */
1619 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1620                                             u32 *flags)
1621 {
1622         int i;
1623
1624         for (i = 0; i < adev->num_ip_blocks; i++) {
1625                 if (!adev->ip_blocks[i].status.valid)
1626                         continue;
1627                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1628                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1629         }
1630 }
1631
1632 /**
1633  * amdgpu_device_ip_wait_for_idle - wait for idle
1634  *
1635  * @adev: amdgpu_device pointer
1636  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1637  *
1638  * Waits for the request hardware IP to be idle.
1639  * Returns 0 for success or a negative error code on failure.
1640  */
1641 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1642                                    enum amd_ip_block_type block_type)
1643 {
1644         int i, r;
1645
1646         for (i = 0; i < adev->num_ip_blocks; i++) {
1647                 if (!adev->ip_blocks[i].status.valid)
1648                         continue;
1649                 if (adev->ip_blocks[i].version->type == block_type) {
1650                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1651                         if (r)
1652                                 return r;
1653                         break;
1654                 }
1655         }
1656         return 0;
1657
1658 }
1659
1660 /**
1661  * amdgpu_device_ip_is_idle - is the hardware IP idle
1662  *
1663  * @adev: amdgpu_device pointer
1664  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1665  *
1666  * Check if the hardware IP is idle or not.
1667  * Returns true if it the IP is idle, false if not.
1668  */
1669 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1670                               enum amd_ip_block_type block_type)
1671 {
1672         int i;
1673
1674         for (i = 0; i < adev->num_ip_blocks; i++) {
1675                 if (!adev->ip_blocks[i].status.valid)
1676                         continue;
1677                 if (adev->ip_blocks[i].version->type == block_type)
1678                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1679         }
1680         return true;
1681
1682 }
1683
1684 /**
1685  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1686  *
1687  * @adev: amdgpu_device pointer
1688  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1689  *
1690  * Returns a pointer to the hardware IP block structure
1691  * if it exists for the asic, otherwise NULL.
1692  */
1693 struct amdgpu_ip_block *
1694 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1695                               enum amd_ip_block_type type)
1696 {
1697         int i;
1698
1699         for (i = 0; i < adev->num_ip_blocks; i++)
1700                 if (adev->ip_blocks[i].version->type == type)
1701                         return &adev->ip_blocks[i];
1702
1703         return NULL;
1704 }
1705
1706 /**
1707  * amdgpu_device_ip_block_version_cmp
1708  *
1709  * @adev: amdgpu_device pointer
1710  * @type: enum amd_ip_block_type
1711  * @major: major version
1712  * @minor: minor version
1713  *
1714  * return 0 if equal or greater
1715  * return 1 if smaller or the ip_block doesn't exist
1716  */
1717 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1718                                        enum amd_ip_block_type type,
1719                                        u32 major, u32 minor)
1720 {
1721         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1722
1723         if (ip_block && ((ip_block->version->major > major) ||
1724                         ((ip_block->version->major == major) &&
1725                         (ip_block->version->minor >= minor))))
1726                 return 0;
1727
1728         return 1;
1729 }
1730
1731 /**
1732  * amdgpu_device_ip_block_add
1733  *
1734  * @adev: amdgpu_device pointer
1735  * @ip_block_version: pointer to the IP to add
1736  *
1737  * Adds the IP block driver information to the collection of IPs
1738  * on the asic.
1739  */
1740 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1741                                const struct amdgpu_ip_block_version *ip_block_version)
1742 {
1743         if (!ip_block_version)
1744                 return -EINVAL;
1745
1746         switch (ip_block_version->type) {
1747         case AMD_IP_BLOCK_TYPE_VCN:
1748                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1749                         return 0;
1750                 break;
1751         case AMD_IP_BLOCK_TYPE_JPEG:
1752                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1753                         return 0;
1754                 break;
1755         default:
1756                 break;
1757         }
1758
1759         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1760                   ip_block_version->funcs->name);
1761
1762         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1763
1764         return 0;
1765 }
1766
1767 /**
1768  * amdgpu_device_enable_virtual_display - enable virtual display feature
1769  *
1770  * @adev: amdgpu_device pointer
1771  *
1772  * Enabled the virtual display feature if the user has enabled it via
1773  * the module parameter virtual_display.  This feature provides a virtual
1774  * display hardware on headless boards or in virtualized environments.
1775  * This function parses and validates the configuration string specified by
1776  * the user and configues the virtual display configuration (number of
1777  * virtual connectors, crtcs, etc.) specified.
1778  */
1779 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1780 {
1781         adev->enable_virtual_display = false;
1782
1783         if (amdgpu_virtual_display) {
1784                 const char *pci_address_name = pci_name(adev->pdev);
1785                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1786
1787                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1788                 pciaddstr_tmp = pciaddstr;
1789                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1790                         pciaddname = strsep(&pciaddname_tmp, ",");
1791                         if (!strcmp("all", pciaddname)
1792                             || !strcmp(pci_address_name, pciaddname)) {
1793                                 long num_crtc;
1794                                 int res = -1;
1795
1796                                 adev->enable_virtual_display = true;
1797
1798                                 if (pciaddname_tmp)
1799                                         res = kstrtol(pciaddname_tmp, 10,
1800                                                       &num_crtc);
1801
1802                                 if (!res) {
1803                                         if (num_crtc < 1)
1804                                                 num_crtc = 1;
1805                                         if (num_crtc > 6)
1806                                                 num_crtc = 6;
1807                                         adev->mode_info.num_crtc = num_crtc;
1808                                 } else {
1809                                         adev->mode_info.num_crtc = 1;
1810                                 }
1811                                 break;
1812                         }
1813                 }
1814
1815                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1816                          amdgpu_virtual_display, pci_address_name,
1817                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1818
1819                 kfree(pciaddstr);
1820         }
1821 }
1822
1823 /**
1824  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1825  *
1826  * @adev: amdgpu_device pointer
1827  *
1828  * Parses the asic configuration parameters specified in the gpu info
1829  * firmware and makes them availale to the driver for use in configuring
1830  * the asic.
1831  * Returns 0 on success, -EINVAL on failure.
1832  */
1833 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1834 {
1835         const char *chip_name;
1836         char fw_name[40];
1837         int err;
1838         const struct gpu_info_firmware_header_v1_0 *hdr;
1839
1840         adev->firmware.gpu_info_fw = NULL;
1841
1842         if (adev->mman.discovery_bin) {
1843                 amdgpu_discovery_get_gfx_info(adev);
1844
1845                 /*
1846                  * FIXME: The bounding box is still needed by Navi12, so
1847                  * temporarily read it from gpu_info firmware. Should be droped
1848                  * when DAL no longer needs it.
1849                  */
1850                 if (adev->asic_type != CHIP_NAVI12)
1851                         return 0;
1852         }
1853
1854         switch (adev->asic_type) {
1855 #ifdef CONFIG_DRM_AMDGPU_SI
1856         case CHIP_VERDE:
1857         case CHIP_TAHITI:
1858         case CHIP_PITCAIRN:
1859         case CHIP_OLAND:
1860         case CHIP_HAINAN:
1861 #endif
1862 #ifdef CONFIG_DRM_AMDGPU_CIK
1863         case CHIP_BONAIRE:
1864         case CHIP_HAWAII:
1865         case CHIP_KAVERI:
1866         case CHIP_KABINI:
1867         case CHIP_MULLINS:
1868 #endif
1869         case CHIP_TOPAZ:
1870         case CHIP_TONGA:
1871         case CHIP_FIJI:
1872         case CHIP_POLARIS10:
1873         case CHIP_POLARIS11:
1874         case CHIP_POLARIS12:
1875         case CHIP_VEGAM:
1876         case CHIP_CARRIZO:
1877         case CHIP_STONEY:
1878         case CHIP_VEGA20:
1879         case CHIP_ALDEBARAN:
1880         case CHIP_SIENNA_CICHLID:
1881         case CHIP_NAVY_FLOUNDER:
1882         case CHIP_DIMGREY_CAVEFISH:
1883         case CHIP_BEIGE_GOBY:
1884         default:
1885                 return 0;
1886         case CHIP_VEGA10:
1887                 chip_name = "vega10";
1888                 break;
1889         case CHIP_VEGA12:
1890                 chip_name = "vega12";
1891                 break;
1892         case CHIP_RAVEN:
1893                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1894                         chip_name = "raven2";
1895                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1896                         chip_name = "picasso";
1897                 else
1898                         chip_name = "raven";
1899                 break;
1900         case CHIP_ARCTURUS:
1901                 chip_name = "arcturus";
1902                 break;
1903         case CHIP_RENOIR:
1904                 if (adev->apu_flags & AMD_APU_IS_RENOIR)
1905                         chip_name = "renoir";
1906                 else
1907                         chip_name = "green_sardine";
1908                 break;
1909         case CHIP_NAVI10:
1910                 chip_name = "navi10";
1911                 break;
1912         case CHIP_NAVI14:
1913                 chip_name = "navi14";
1914                 break;
1915         case CHIP_NAVI12:
1916                 chip_name = "navi12";
1917                 break;
1918         case CHIP_VANGOGH:
1919                 chip_name = "vangogh";
1920                 break;
1921         case CHIP_YELLOW_CARP:
1922                 chip_name = "yellow_carp";
1923                 break;
1924         }
1925
1926         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1927         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1928         if (err) {
1929                 dev_err(adev->dev,
1930                         "Failed to load gpu_info firmware \"%s\"\n",
1931                         fw_name);
1932                 goto out;
1933         }
1934         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1935         if (err) {
1936                 dev_err(adev->dev,
1937                         "Failed to validate gpu_info firmware \"%s\"\n",
1938                         fw_name);
1939                 goto out;
1940         }
1941
1942         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
1943         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
1944
1945         switch (hdr->version_major) {
1946         case 1:
1947         {
1948                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
1949                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
1950                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1951
1952                 /*
1953                  * Should be droped when DAL no longer needs it.
1954                  */
1955                 if (adev->asic_type == CHIP_NAVI12)
1956                         goto parse_soc_bounding_box;
1957
1958                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
1959                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
1960                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
1961                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
1962                 adev->gfx.config.max_texture_channel_caches =
1963                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
1964                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
1965                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
1966                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
1967                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
1968                 adev->gfx.config.double_offchip_lds_buf =
1969                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
1970                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
1971                 adev->gfx.cu_info.max_waves_per_simd =
1972                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
1973                 adev->gfx.cu_info.max_scratch_slots_per_cu =
1974                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
1975                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
1976                 if (hdr->version_minor >= 1) {
1977                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
1978                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
1979                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1980                         adev->gfx.config.num_sc_per_sh =
1981                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
1982                         adev->gfx.config.num_packer_per_sc =
1983                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
1984                 }
1985
1986 parse_soc_bounding_box:
1987                 /*
1988                  * soc bounding box info is not integrated in disocovery table,
1989                  * we always need to parse it from gpu info firmware if needed.
1990                  */
1991                 if (hdr->version_minor == 2) {
1992                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
1993                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
1994                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
1995                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
1996                 }
1997                 break;
1998         }
1999         default:
2000                 dev_err(adev->dev,
2001                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2002                 err = -EINVAL;
2003                 goto out;
2004         }
2005 out:
2006         return err;
2007 }
2008
2009 /**
2010  * amdgpu_device_ip_early_init - run early init for hardware IPs
2011  *
2012  * @adev: amdgpu_device pointer
2013  *
2014  * Early initialization pass for hardware IPs.  The hardware IPs that make
2015  * up each asic are discovered each IP's early_init callback is run.  This
2016  * is the first stage in initializing the asic.
2017  * Returns 0 on success, negative error code on failure.
2018  */
2019 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2020 {
2021         int i, r;
2022
2023         amdgpu_device_enable_virtual_display(adev);
2024
2025         if (amdgpu_sriov_vf(adev)) {
2026                 r = amdgpu_virt_request_full_gpu(adev, true);
2027                 if (r)
2028                         return r;
2029         }
2030
2031         switch (adev->asic_type) {
2032 #ifdef CONFIG_DRM_AMDGPU_SI
2033         case CHIP_VERDE:
2034         case CHIP_TAHITI:
2035         case CHIP_PITCAIRN:
2036         case CHIP_OLAND:
2037         case CHIP_HAINAN:
2038                 adev->family = AMDGPU_FAMILY_SI;
2039                 r = si_set_ip_blocks(adev);
2040                 if (r)
2041                         return r;
2042                 break;
2043 #endif
2044 #ifdef CONFIG_DRM_AMDGPU_CIK
2045         case CHIP_BONAIRE:
2046         case CHIP_HAWAII:
2047         case CHIP_KAVERI:
2048         case CHIP_KABINI:
2049         case CHIP_MULLINS:
2050                 if (adev->flags & AMD_IS_APU)
2051                         adev->family = AMDGPU_FAMILY_KV;
2052                 else
2053                         adev->family = AMDGPU_FAMILY_CI;
2054
2055                 r = cik_set_ip_blocks(adev);
2056                 if (r)
2057                         return r;
2058                 break;
2059 #endif
2060         case CHIP_TOPAZ:
2061         case CHIP_TONGA:
2062         case CHIP_FIJI:
2063         case CHIP_POLARIS10:
2064         case CHIP_POLARIS11:
2065         case CHIP_POLARIS12:
2066         case CHIP_VEGAM:
2067         case CHIP_CARRIZO:
2068         case CHIP_STONEY:
2069                 if (adev->flags & AMD_IS_APU)
2070                         adev->family = AMDGPU_FAMILY_CZ;
2071                 else
2072                         adev->family = AMDGPU_FAMILY_VI;
2073
2074                 r = vi_set_ip_blocks(adev);
2075                 if (r)
2076                         return r;
2077                 break;
2078         case CHIP_VEGA10:
2079         case CHIP_VEGA12:
2080         case CHIP_VEGA20:
2081         case CHIP_RAVEN:
2082         case CHIP_ARCTURUS:
2083         case CHIP_RENOIR:
2084         case CHIP_ALDEBARAN:
2085                 if (adev->flags & AMD_IS_APU)
2086                         adev->family = AMDGPU_FAMILY_RV;
2087                 else
2088                         adev->family = AMDGPU_FAMILY_AI;
2089
2090                 r = soc15_set_ip_blocks(adev);
2091                 if (r)
2092                         return r;
2093                 break;
2094         case  CHIP_NAVI10:
2095         case  CHIP_NAVI14:
2096         case  CHIP_NAVI12:
2097         case  CHIP_SIENNA_CICHLID:
2098         case  CHIP_NAVY_FLOUNDER:
2099         case  CHIP_DIMGREY_CAVEFISH:
2100         case  CHIP_BEIGE_GOBY:
2101         case CHIP_VANGOGH:
2102         case CHIP_YELLOW_CARP:
2103                 if (adev->asic_type == CHIP_VANGOGH)
2104                         adev->family = AMDGPU_FAMILY_VGH;
2105                 else if (adev->asic_type == CHIP_YELLOW_CARP)
2106                         adev->family = AMDGPU_FAMILY_YC;
2107                 else
2108                         adev->family = AMDGPU_FAMILY_NV;
2109
2110                 r = nv_set_ip_blocks(adev);
2111                 if (r)
2112                         return r;
2113                 break;
2114         default:
2115                 /* FIXME: not supported yet */
2116                 return -EINVAL;
2117         }
2118
2119         amdgpu_amdkfd_device_probe(adev);
2120
2121         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2122         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2123                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2124         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2125                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2126
2127         for (i = 0; i < adev->num_ip_blocks; i++) {
2128                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2129                         DRM_ERROR("disabled ip block: %d <%s>\n",
2130                                   i, adev->ip_blocks[i].version->funcs->name);
2131                         adev->ip_blocks[i].status.valid = false;
2132                 } else {
2133                         if (adev->ip_blocks[i].version->funcs->early_init) {
2134                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2135                                 if (r == -ENOENT) {
2136                                         adev->ip_blocks[i].status.valid = false;
2137                                 } else if (r) {
2138                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2139                                                   adev->ip_blocks[i].version->funcs->name, r);
2140                                         return r;
2141                                 } else {
2142                                         adev->ip_blocks[i].status.valid = true;
2143                                 }
2144                         } else {
2145                                 adev->ip_blocks[i].status.valid = true;
2146                         }
2147                 }
2148                 /* get the vbios after the asic_funcs are set up */
2149                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2150                         r = amdgpu_device_parse_gpu_info_fw(adev);
2151                         if (r)
2152                                 return r;
2153
2154                         /* Read BIOS */
2155                         if (!amdgpu_get_bios(adev))
2156                                 return -EINVAL;
2157
2158                         r = amdgpu_atombios_init(adev);
2159                         if (r) {
2160                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2161                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2162                                 return r;
2163                         }
2164
2165                         /*get pf2vf msg info at it's earliest time*/
2166                         if (amdgpu_sriov_vf(adev))
2167                                 amdgpu_virt_init_data_exchange(adev);
2168
2169                 }
2170         }
2171
2172         adev->cg_flags &= amdgpu_cg_mask;
2173         adev->pg_flags &= amdgpu_pg_mask;
2174
2175         return 0;
2176 }
2177
2178 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2179 {
2180         int i, r;
2181
2182         for (i = 0; i < adev->num_ip_blocks; i++) {
2183                 if (!adev->ip_blocks[i].status.sw)
2184                         continue;
2185                 if (adev->ip_blocks[i].status.hw)
2186                         continue;
2187                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2188                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2189                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2190                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2191                         if (r) {
2192                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2193                                           adev->ip_blocks[i].version->funcs->name, r);
2194                                 return r;
2195                         }
2196                         adev->ip_blocks[i].status.hw = true;
2197                 }
2198         }
2199
2200         return 0;
2201 }
2202
2203 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2204 {
2205         int i, r;
2206
2207         for (i = 0; i < adev->num_ip_blocks; i++) {
2208                 if (!adev->ip_blocks[i].status.sw)
2209                         continue;
2210                 if (adev->ip_blocks[i].status.hw)
2211                         continue;
2212                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2213                 if (r) {
2214                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2215                                   adev->ip_blocks[i].version->funcs->name, r);
2216                         return r;
2217                 }
2218                 adev->ip_blocks[i].status.hw = true;
2219         }
2220
2221         return 0;
2222 }
2223
2224 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2225 {
2226         int r = 0;
2227         int i;
2228         uint32_t smu_version;
2229
2230         if (adev->asic_type >= CHIP_VEGA10) {
2231                 for (i = 0; i < adev->num_ip_blocks; i++) {
2232                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2233                                 continue;
2234
2235                         if (!adev->ip_blocks[i].status.sw)
2236                                 continue;
2237
2238                         /* no need to do the fw loading again if already done*/
2239                         if (adev->ip_blocks[i].status.hw == true)
2240                                 break;
2241
2242                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2243                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2244                                 if (r) {
2245                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2246                                                           adev->ip_blocks[i].version->funcs->name, r);
2247                                         return r;
2248                                 }
2249                         } else {
2250                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2251                                 if (r) {
2252                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2253                                                           adev->ip_blocks[i].version->funcs->name, r);
2254                                         return r;
2255                                 }
2256                         }
2257
2258                         adev->ip_blocks[i].status.hw = true;
2259                         break;
2260                 }
2261         }
2262
2263         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2264                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2265
2266         return r;
2267 }
2268
2269 /**
2270  * amdgpu_device_ip_init - run init for hardware IPs
2271  *
2272  * @adev: amdgpu_device pointer
2273  *
2274  * Main initialization pass for hardware IPs.  The list of all the hardware
2275  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2276  * are run.  sw_init initializes the software state associated with each IP
2277  * and hw_init initializes the hardware associated with each IP.
2278  * Returns 0 on success, negative error code on failure.
2279  */
2280 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2281 {
2282         int i, r;
2283
2284         r = amdgpu_ras_init(adev);
2285         if (r)
2286                 return r;
2287
2288         for (i = 0; i < adev->num_ip_blocks; i++) {
2289                 if (!adev->ip_blocks[i].status.valid)
2290                         continue;
2291                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2292                 if (r) {
2293                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2294                                   adev->ip_blocks[i].version->funcs->name, r);
2295                         goto init_failed;
2296                 }
2297                 adev->ip_blocks[i].status.sw = true;
2298
2299                 /* need to do gmc hw init early so we can allocate gpu mem */
2300                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2301                         r = amdgpu_device_vram_scratch_init(adev);
2302                         if (r) {
2303                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2304                                 goto init_failed;
2305                         }
2306                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2307                         if (r) {
2308                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2309                                 goto init_failed;
2310                         }
2311                         r = amdgpu_device_wb_init(adev);
2312                         if (r) {
2313                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2314                                 goto init_failed;
2315                         }
2316                         adev->ip_blocks[i].status.hw = true;
2317
2318                         /* right after GMC hw init, we create CSA */
2319                         if (amdgpu_mcbp || amdgpu_sriov_vf(adev)) {
2320                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2321                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2322                                                                 AMDGPU_CSA_SIZE);
2323                                 if (r) {
2324                                         DRM_ERROR("allocate CSA failed %d\n", r);
2325                                         goto init_failed;
2326                                 }
2327                         }
2328                 }
2329         }
2330
2331         if (amdgpu_sriov_vf(adev))
2332                 amdgpu_virt_init_data_exchange(adev);
2333
2334         r = amdgpu_ib_pool_init(adev);
2335         if (r) {
2336                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2337                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2338                 goto init_failed;
2339         }
2340
2341         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2342         if (r)
2343                 goto init_failed;
2344
2345         r = amdgpu_device_ip_hw_init_phase1(adev);
2346         if (r)
2347                 goto init_failed;
2348
2349         r = amdgpu_device_fw_loading(adev);
2350         if (r)
2351                 goto init_failed;
2352
2353         r = amdgpu_device_ip_hw_init_phase2(adev);
2354         if (r)
2355                 goto init_failed;
2356
2357         /*
2358          * retired pages will be loaded from eeprom and reserved here,
2359          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2360          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2361          * for I2C communication which only true at this point.
2362          *
2363          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2364          * failure from bad gpu situation and stop amdgpu init process
2365          * accordingly. For other failed cases, it will still release all
2366          * the resource and print error message, rather than returning one
2367          * negative value to upper level.
2368          *
2369          * Note: theoretically, this should be called before all vram allocations
2370          * to protect retired page from abusing
2371          */
2372         r = amdgpu_ras_recovery_init(adev);
2373         if (r)
2374                 goto init_failed;
2375
2376         if (adev->gmc.xgmi.num_physical_nodes > 1)
2377                 amdgpu_xgmi_add_device(adev);
2378
2379         /* Don't init kfd if whole hive need to be reset during init */
2380         if (!adev->gmc.xgmi.pending_reset)
2381                 amdgpu_amdkfd_device_init(adev);
2382
2383         amdgpu_fru_get_product_info(adev);
2384
2385 init_failed:
2386         if (amdgpu_sriov_vf(adev))
2387                 amdgpu_virt_release_full_gpu(adev, true);
2388
2389         return r;
2390 }
2391
2392 /**
2393  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2394  *
2395  * @adev: amdgpu_device pointer
2396  *
2397  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2398  * this function before a GPU reset.  If the value is retained after a
2399  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2400  */
2401 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2402 {
2403         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2404 }
2405
2406 /**
2407  * amdgpu_device_check_vram_lost - check if vram is valid
2408  *
2409  * @adev: amdgpu_device pointer
2410  *
2411  * Checks the reset magic value written to the gart pointer in VRAM.
2412  * The driver calls this after a GPU reset to see if the contents of
2413  * VRAM is lost or now.
2414  * returns true if vram is lost, false if not.
2415  */
2416 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2417 {
2418         if (memcmp(adev->gart.ptr, adev->reset_magic,
2419                         AMDGPU_RESET_MAGIC_NUM))
2420                 return true;
2421
2422         if (!amdgpu_in_reset(adev))
2423                 return false;
2424
2425         /*
2426          * For all ASICs with baco/mode1 reset, the VRAM is
2427          * always assumed to be lost.
2428          */
2429         switch (amdgpu_asic_reset_method(adev)) {
2430         case AMD_RESET_METHOD_BACO:
2431         case AMD_RESET_METHOD_MODE1:
2432                 return true;
2433         default:
2434                 return false;
2435         }
2436 }
2437
2438 /**
2439  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2440  *
2441  * @adev: amdgpu_device pointer
2442  * @state: clockgating state (gate or ungate)
2443  *
2444  * The list of all the hardware IPs that make up the asic is walked and the
2445  * set_clockgating_state callbacks are run.
2446  * Late initialization pass enabling clockgating for hardware IPs.
2447  * Fini or suspend, pass disabling clockgating for hardware IPs.
2448  * Returns 0 on success, negative error code on failure.
2449  */
2450
2451 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2452                                enum amd_clockgating_state state)
2453 {
2454         int i, j, r;
2455
2456         if (amdgpu_emu_mode == 1)
2457                 return 0;
2458
2459         for (j = 0; j < adev->num_ip_blocks; j++) {
2460                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2461                 if (!adev->ip_blocks[i].status.late_initialized)
2462                         continue;
2463                 /* skip CG for GFX on S0ix */
2464                 if (adev->in_s0ix &&
2465                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2466                         continue;
2467                 /* skip CG for VCE/UVD, it's handled specially */
2468                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2469                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2470                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2471                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2472                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2473                         /* enable clockgating to save power */
2474                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2475                                                                                      state);
2476                         if (r) {
2477                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2478                                           adev->ip_blocks[i].version->funcs->name, r);
2479                                 return r;
2480                         }
2481                 }
2482         }
2483
2484         return 0;
2485 }
2486
2487 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2488                                enum amd_powergating_state state)
2489 {
2490         int i, j, r;
2491
2492         if (amdgpu_emu_mode == 1)
2493                 return 0;
2494
2495         for (j = 0; j < adev->num_ip_blocks; j++) {
2496                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2497                 if (!adev->ip_blocks[i].status.late_initialized)
2498                         continue;
2499                 /* skip PG for GFX on S0ix */
2500                 if (adev->in_s0ix &&
2501                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2502                         continue;
2503                 /* skip CG for VCE/UVD, it's handled specially */
2504                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2505                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2506                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2507                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2508                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2509                         /* enable powergating to save power */
2510                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2511                                                                                         state);
2512                         if (r) {
2513                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2514                                           adev->ip_blocks[i].version->funcs->name, r);
2515                                 return r;
2516                         }
2517                 }
2518         }
2519         return 0;
2520 }
2521
2522 static int amdgpu_device_enable_mgpu_fan_boost(void)
2523 {
2524         struct amdgpu_gpu_instance *gpu_ins;
2525         struct amdgpu_device *adev;
2526         int i, ret = 0;
2527
2528         mutex_lock(&mgpu_info.mutex);
2529
2530         /*
2531          * MGPU fan boost feature should be enabled
2532          * only when there are two or more dGPUs in
2533          * the system
2534          */
2535         if (mgpu_info.num_dgpu < 2)
2536                 goto out;
2537
2538         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2539                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2540                 adev = gpu_ins->adev;
2541                 if (!(adev->flags & AMD_IS_APU) &&
2542                     !gpu_ins->mgpu_fan_enabled) {
2543                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2544                         if (ret)
2545                                 break;
2546
2547                         gpu_ins->mgpu_fan_enabled = 1;
2548                 }
2549         }
2550
2551 out:
2552         mutex_unlock(&mgpu_info.mutex);
2553
2554         return ret;
2555 }
2556
2557 /**
2558  * amdgpu_device_ip_late_init - run late init for hardware IPs
2559  *
2560  * @adev: amdgpu_device pointer
2561  *
2562  * Late initialization pass for hardware IPs.  The list of all the hardware
2563  * IPs that make up the asic is walked and the late_init callbacks are run.
2564  * late_init covers any special initialization that an IP requires
2565  * after all of the have been initialized or something that needs to happen
2566  * late in the init process.
2567  * Returns 0 on success, negative error code on failure.
2568  */
2569 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2570 {
2571         struct amdgpu_gpu_instance *gpu_instance;
2572         int i = 0, r;
2573
2574         for (i = 0; i < adev->num_ip_blocks; i++) {
2575                 if (!adev->ip_blocks[i].status.hw)
2576                         continue;
2577                 if (adev->ip_blocks[i].version->funcs->late_init) {
2578                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2579                         if (r) {
2580                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2581                                           adev->ip_blocks[i].version->funcs->name, r);
2582                                 return r;
2583                         }
2584                 }
2585                 adev->ip_blocks[i].status.late_initialized = true;
2586         }
2587
2588         amdgpu_ras_set_error_query_ready(adev, true);
2589
2590         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2591         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2592
2593         amdgpu_device_fill_reset_magic(adev);
2594
2595         r = amdgpu_device_enable_mgpu_fan_boost();
2596         if (r)
2597                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2598
2599         /* For XGMI + passthrough configuration on arcturus, enable light SBR */
2600         if (adev->asic_type == CHIP_ARCTURUS &&
2601             amdgpu_passthrough(adev) &&
2602             adev->gmc.xgmi.num_physical_nodes > 1)
2603                 smu_set_light_sbr(&adev->smu, true);
2604
2605         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2606                 mutex_lock(&mgpu_info.mutex);
2607
2608                 /*
2609                  * Reset device p-state to low as this was booted with high.
2610                  *
2611                  * This should be performed only after all devices from the same
2612                  * hive get initialized.
2613                  *
2614                  * However, it's unknown how many device in the hive in advance.
2615                  * As this is counted one by one during devices initializations.
2616                  *
2617                  * So, we wait for all XGMI interlinked devices initialized.
2618                  * This may bring some delays as those devices may come from
2619                  * different hives. But that should be OK.
2620                  */
2621                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2622                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2623                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2624                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2625                                         continue;
2626
2627                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2628                                                 AMDGPU_XGMI_PSTATE_MIN);
2629                                 if (r) {
2630                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2631                                         break;
2632                                 }
2633                         }
2634                 }
2635
2636                 mutex_unlock(&mgpu_info.mutex);
2637         }
2638
2639         return 0;
2640 }
2641
2642 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2643 {
2644         int i, r;
2645
2646         for (i = 0; i < adev->num_ip_blocks; i++) {
2647                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2648                         continue;
2649
2650                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2651                 if (r) {
2652                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2653                                   adev->ip_blocks[i].version->funcs->name, r);
2654                 }
2655         }
2656
2657         amdgpu_amdkfd_suspend(adev, false);
2658
2659         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2660         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2661
2662         /* need to disable SMC first */
2663         for (i = 0; i < adev->num_ip_blocks; i++) {
2664                 if (!adev->ip_blocks[i].status.hw)
2665                         continue;
2666                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2667                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2668                         /* XXX handle errors */
2669                         if (r) {
2670                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2671                                           adev->ip_blocks[i].version->funcs->name, r);
2672                         }
2673                         adev->ip_blocks[i].status.hw = false;
2674                         break;
2675                 }
2676         }
2677
2678         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2679                 if (!adev->ip_blocks[i].status.hw)
2680                         continue;
2681
2682                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2683                 /* XXX handle errors */
2684                 if (r) {
2685                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2686                                   adev->ip_blocks[i].version->funcs->name, r);
2687                 }
2688
2689                 adev->ip_blocks[i].status.hw = false;
2690         }
2691
2692         return 0;
2693 }
2694
2695 /**
2696  * amdgpu_device_ip_fini - run fini for hardware IPs
2697  *
2698  * @adev: amdgpu_device pointer
2699  *
2700  * Main teardown pass for hardware IPs.  The list of all the hardware
2701  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2702  * are run.  hw_fini tears down the hardware associated with each IP
2703  * and sw_fini tears down any software state associated with each IP.
2704  * Returns 0 on success, negative error code on failure.
2705  */
2706 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2707 {
2708         int i, r;
2709
2710         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2711                 amdgpu_virt_release_ras_err_handler_data(adev);
2712
2713         amdgpu_ras_pre_fini(adev);
2714
2715         if (adev->gmc.xgmi.num_physical_nodes > 1)
2716                 amdgpu_xgmi_remove_device(adev);
2717
2718         amdgpu_amdkfd_device_fini_sw(adev);
2719
2720         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2721                 if (!adev->ip_blocks[i].status.sw)
2722                         continue;
2723
2724                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2725                         amdgpu_ucode_free_bo(adev);
2726                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2727                         amdgpu_device_wb_fini(adev);
2728                         amdgpu_device_vram_scratch_fini(adev);
2729                         amdgpu_ib_pool_fini(adev);
2730                 }
2731
2732                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2733                 /* XXX handle errors */
2734                 if (r) {
2735                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2736                                   adev->ip_blocks[i].version->funcs->name, r);
2737                 }
2738                 adev->ip_blocks[i].status.sw = false;
2739                 adev->ip_blocks[i].status.valid = false;
2740         }
2741
2742         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2743                 if (!adev->ip_blocks[i].status.late_initialized)
2744                         continue;
2745                 if (adev->ip_blocks[i].version->funcs->late_fini)
2746                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2747                 adev->ip_blocks[i].status.late_initialized = false;
2748         }
2749
2750         amdgpu_ras_fini(adev);
2751
2752         if (amdgpu_sriov_vf(adev))
2753                 if (amdgpu_virt_release_full_gpu(adev, false))
2754                         DRM_ERROR("failed to release exclusive mode on fini\n");
2755
2756         return 0;
2757 }
2758
2759 /**
2760  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2761  *
2762  * @work: work_struct.
2763  */
2764 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2765 {
2766         struct amdgpu_device *adev =
2767                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2768         int r;
2769
2770         r = amdgpu_ib_ring_tests(adev);
2771         if (r)
2772                 DRM_ERROR("ib ring test failed (%d).\n", r);
2773 }
2774
2775 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2776 {
2777         struct amdgpu_device *adev =
2778                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2779
2780         mutex_lock(&adev->gfx.gfx_off_mutex);
2781         if (!adev->gfx.gfx_off_state && !adev->gfx.gfx_off_req_count) {
2782                 if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2783                         adev->gfx.gfx_off_state = true;
2784         }
2785         mutex_unlock(&adev->gfx.gfx_off_mutex);
2786 }
2787
2788 /**
2789  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2790  *
2791  * @adev: amdgpu_device pointer
2792  *
2793  * Main suspend function for hardware IPs.  The list of all the hardware
2794  * IPs that make up the asic is walked, clockgating is disabled and the
2795  * suspend callbacks are run.  suspend puts the hardware and software state
2796  * in each IP into a state suitable for suspend.
2797  * Returns 0 on success, negative error code on failure.
2798  */
2799 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2800 {
2801         int i, r;
2802
2803         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2804         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2805
2806         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2807                 if (!adev->ip_blocks[i].status.valid)
2808                         continue;
2809
2810                 /* displays are handled separately */
2811                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2812                         continue;
2813
2814                 /* XXX handle errors */
2815                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2816                 /* XXX handle errors */
2817                 if (r) {
2818                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2819                                   adev->ip_blocks[i].version->funcs->name, r);
2820                         return r;
2821                 }
2822
2823                 adev->ip_blocks[i].status.hw = false;
2824         }
2825
2826         return 0;
2827 }
2828
2829 /**
2830  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2831  *
2832  * @adev: amdgpu_device pointer
2833  *
2834  * Main suspend function for hardware IPs.  The list of all the hardware
2835  * IPs that make up the asic is walked, clockgating is disabled and the
2836  * suspend callbacks are run.  suspend puts the hardware and software state
2837  * in each IP into a state suitable for suspend.
2838  * Returns 0 on success, negative error code on failure.
2839  */
2840 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2841 {
2842         int i, r;
2843
2844         if (adev->in_s0ix)
2845                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D3Entry);
2846
2847         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2848                 if (!adev->ip_blocks[i].status.valid)
2849                         continue;
2850                 /* displays are handled in phase1 */
2851                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
2852                         continue;
2853                 /* PSP lost connection when err_event_athub occurs */
2854                 if (amdgpu_ras_intr_triggered() &&
2855                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
2856                         adev->ip_blocks[i].status.hw = false;
2857                         continue;
2858                 }
2859
2860                 /* skip unnecessary suspend if we do not initialize them yet */
2861                 if (adev->gmc.xgmi.pending_reset &&
2862                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
2863                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
2864                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2865                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
2866                         adev->ip_blocks[i].status.hw = false;
2867                         continue;
2868                 }
2869
2870                 /* skip suspend of gfx and psp for S0ix
2871                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
2872                  * like at runtime. PSP is also part of the always on hardware
2873                  * so no need to suspend it.
2874                  */
2875                 if (adev->in_s0ix &&
2876                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
2877                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
2878                         continue;
2879
2880                 /* XXX handle errors */
2881                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2882                 /* XXX handle errors */
2883                 if (r) {
2884                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2885                                   adev->ip_blocks[i].version->funcs->name, r);
2886                 }
2887                 adev->ip_blocks[i].status.hw = false;
2888                 /* handle putting the SMC in the appropriate state */
2889                 if(!amdgpu_sriov_vf(adev)){
2890                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2891                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
2892                                 if (r) {
2893                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
2894                                                         adev->mp1_state, r);
2895                                         return r;
2896                                 }
2897                         }
2898                 }
2899         }
2900
2901         return 0;
2902 }
2903
2904 /**
2905  * amdgpu_device_ip_suspend - run suspend for hardware IPs
2906  *
2907  * @adev: amdgpu_device pointer
2908  *
2909  * Main suspend function for hardware IPs.  The list of all the hardware
2910  * IPs that make up the asic is walked, clockgating is disabled and the
2911  * suspend callbacks are run.  suspend puts the hardware and software state
2912  * in each IP into a state suitable for suspend.
2913  * Returns 0 on success, negative error code on failure.
2914  */
2915 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
2916 {
2917         int r;
2918
2919         if (amdgpu_sriov_vf(adev)) {
2920                 amdgpu_virt_fini_data_exchange(adev);
2921                 amdgpu_virt_request_full_gpu(adev, false);
2922         }
2923
2924         r = amdgpu_device_ip_suspend_phase1(adev);
2925         if (r)
2926                 return r;
2927         r = amdgpu_device_ip_suspend_phase2(adev);
2928
2929         if (amdgpu_sriov_vf(adev))
2930                 amdgpu_virt_release_full_gpu(adev, false);
2931
2932         return r;
2933 }
2934
2935 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
2936 {
2937         int i, r;
2938
2939         static enum amd_ip_block_type ip_order[] = {
2940                 AMD_IP_BLOCK_TYPE_GMC,
2941                 AMD_IP_BLOCK_TYPE_COMMON,
2942                 AMD_IP_BLOCK_TYPE_PSP,
2943                 AMD_IP_BLOCK_TYPE_IH,
2944         };
2945
2946         for (i = 0; i < adev->num_ip_blocks; i++) {
2947                 int j;
2948                 struct amdgpu_ip_block *block;
2949
2950                 block = &adev->ip_blocks[i];
2951                 block->status.hw = false;
2952
2953                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
2954
2955                         if (block->version->type != ip_order[j] ||
2956                                 !block->status.valid)
2957                                 continue;
2958
2959                         r = block->version->funcs->hw_init(adev);
2960                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
2961                         if (r)
2962                                 return r;
2963                         block->status.hw = true;
2964                 }
2965         }
2966
2967         return 0;
2968 }
2969
2970 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
2971 {
2972         int i, r;
2973
2974         static enum amd_ip_block_type ip_order[] = {
2975                 AMD_IP_BLOCK_TYPE_SMC,
2976                 AMD_IP_BLOCK_TYPE_DCE,
2977                 AMD_IP_BLOCK_TYPE_GFX,
2978                 AMD_IP_BLOCK_TYPE_SDMA,
2979                 AMD_IP_BLOCK_TYPE_UVD,
2980                 AMD_IP_BLOCK_TYPE_VCE,
2981                 AMD_IP_BLOCK_TYPE_VCN
2982         };
2983
2984         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
2985                 int j;
2986                 struct amdgpu_ip_block *block;
2987
2988                 for (j = 0; j < adev->num_ip_blocks; j++) {
2989                         block = &adev->ip_blocks[j];
2990
2991                         if (block->version->type != ip_order[i] ||
2992                                 !block->status.valid ||
2993                                 block->status.hw)
2994                                 continue;
2995
2996                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
2997                                 r = block->version->funcs->resume(adev);
2998                         else
2999                                 r = block->version->funcs->hw_init(adev);
3000
3001                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3002                         if (r)
3003                                 return r;
3004                         block->status.hw = true;
3005                 }
3006         }
3007
3008         return 0;
3009 }
3010
3011 /**
3012  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3013  *
3014  * @adev: amdgpu_device pointer
3015  *
3016  * First resume function for hardware IPs.  The list of all the hardware
3017  * IPs that make up the asic is walked and the resume callbacks are run for
3018  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3019  * after a suspend and updates the software state as necessary.  This
3020  * function is also used for restoring the GPU after a GPU reset.
3021  * Returns 0 on success, negative error code on failure.
3022  */
3023 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3024 {
3025         int i, r;
3026
3027         for (i = 0; i < adev->num_ip_blocks; i++) {
3028                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3029                         continue;
3030                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3031                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3032                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
3033
3034                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3035                         if (r) {
3036                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3037                                           adev->ip_blocks[i].version->funcs->name, r);
3038                                 return r;
3039                         }
3040                         adev->ip_blocks[i].status.hw = true;
3041                 }
3042         }
3043
3044         return 0;
3045 }
3046
3047 /**
3048  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3049  *
3050  * @adev: amdgpu_device pointer
3051  *
3052  * First resume function for hardware IPs.  The list of all the hardware
3053  * IPs that make up the asic is walked and the resume callbacks are run for
3054  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3055  * functional state after a suspend and updates the software state as
3056  * necessary.  This function is also used for restoring the GPU after a GPU
3057  * reset.
3058  * Returns 0 on success, negative error code on failure.
3059  */
3060 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3061 {
3062         int i, r;
3063
3064         for (i = 0; i < adev->num_ip_blocks; i++) {
3065                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3066                         continue;
3067                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3068                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3069                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3070                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3071                         continue;
3072                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3073                 if (r) {
3074                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3075                                   adev->ip_blocks[i].version->funcs->name, r);
3076                         return r;
3077                 }
3078                 adev->ip_blocks[i].status.hw = true;
3079         }
3080
3081         return 0;
3082 }
3083
3084 /**
3085  * amdgpu_device_ip_resume - run resume for hardware IPs
3086  *
3087  * @adev: amdgpu_device pointer
3088  *
3089  * Main resume function for hardware IPs.  The hardware IPs
3090  * are split into two resume functions because they are
3091  * are also used in in recovering from a GPU reset and some additional
3092  * steps need to be take between them.  In this case (S3/S4) they are
3093  * run sequentially.
3094  * Returns 0 on success, negative error code on failure.
3095  */
3096 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3097 {
3098         int r;
3099
3100         r = amdgpu_device_ip_resume_phase1(adev);
3101         if (r)
3102                 return r;
3103
3104         r = amdgpu_device_fw_loading(adev);
3105         if (r)
3106                 return r;
3107
3108         r = amdgpu_device_ip_resume_phase2(adev);
3109
3110         return r;
3111 }
3112
3113 /**
3114  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3115  *
3116  * @adev: amdgpu_device pointer
3117  *
3118  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3119  */
3120 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3121 {
3122         if (amdgpu_sriov_vf(adev)) {
3123                 if (adev->is_atom_fw) {
3124                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3125                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3126                 } else {
3127                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3128                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3129                 }
3130
3131                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3132                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3133         }
3134 }
3135
3136 /**
3137  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3138  *
3139  * @asic_type: AMD asic type
3140  *
3141  * Check if there is DC (new modesetting infrastructre) support for an asic.
3142  * returns true if DC has support, false if not.
3143  */
3144 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3145 {
3146         switch (asic_type) {
3147 #if defined(CONFIG_DRM_AMD_DC)
3148 #if defined(CONFIG_DRM_AMD_DC_SI)
3149         case CHIP_TAHITI:
3150         case CHIP_PITCAIRN:
3151         case CHIP_VERDE:
3152         case CHIP_OLAND:
3153 #endif
3154         case CHIP_BONAIRE:
3155         case CHIP_KAVERI:
3156         case CHIP_KABINI:
3157         case CHIP_MULLINS:
3158                 /*
3159                  * We have systems in the wild with these ASICs that require
3160                  * LVDS and VGA support which is not supported with DC.
3161                  *
3162                  * Fallback to the non-DC driver here by default so as not to
3163                  * cause regressions.
3164                  */
3165                 return amdgpu_dc > 0;
3166         case CHIP_HAWAII:
3167         case CHIP_CARRIZO:
3168         case CHIP_STONEY:
3169         case CHIP_POLARIS10:
3170         case CHIP_POLARIS11:
3171         case CHIP_POLARIS12:
3172         case CHIP_VEGAM:
3173         case CHIP_TONGA:
3174         case CHIP_FIJI:
3175         case CHIP_VEGA10:
3176         case CHIP_VEGA12:
3177         case CHIP_VEGA20:
3178 #if defined(CONFIG_DRM_AMD_DC_DCN)
3179         case CHIP_RAVEN:
3180         case CHIP_NAVI10:
3181         case CHIP_NAVI14:
3182         case CHIP_NAVI12:
3183         case CHIP_RENOIR:
3184         case CHIP_SIENNA_CICHLID:
3185         case CHIP_NAVY_FLOUNDER:
3186         case CHIP_DIMGREY_CAVEFISH:
3187         case CHIP_BEIGE_GOBY:
3188         case CHIP_VANGOGH:
3189         case CHIP_YELLOW_CARP:
3190 #endif
3191                 return amdgpu_dc != 0;
3192 #endif
3193         default:
3194                 if (amdgpu_dc > 0)
3195                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3196                                          "but isn't supported by ASIC, ignoring\n");
3197                 return false;
3198         }
3199 }
3200
3201 /**
3202  * amdgpu_device_has_dc_support - check if dc is supported
3203  *
3204  * @adev: amdgpu_device pointer
3205  *
3206  * Returns true for supported, false for not supported
3207  */
3208 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3209 {
3210         if (amdgpu_sriov_vf(adev) ||
3211             adev->enable_virtual_display ||
3212             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3213                 return false;
3214
3215         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3216 }
3217
3218 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3219 {
3220         struct amdgpu_device *adev =
3221                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3222         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3223
3224         /* It's a bug to not have a hive within this function */
3225         if (WARN_ON(!hive))
3226                 return;
3227
3228         /*
3229          * Use task barrier to synchronize all xgmi reset works across the
3230          * hive. task_barrier_enter and task_barrier_exit will block
3231          * until all the threads running the xgmi reset works reach
3232          * those points. task_barrier_full will do both blocks.
3233          */
3234         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3235
3236                 task_barrier_enter(&hive->tb);
3237                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3238
3239                 if (adev->asic_reset_res)
3240                         goto fail;
3241
3242                 task_barrier_exit(&hive->tb);
3243                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3244
3245                 if (adev->asic_reset_res)
3246                         goto fail;
3247
3248                 if (adev->mmhub.ras_funcs &&
3249                     adev->mmhub.ras_funcs->reset_ras_error_count)
3250                         adev->mmhub.ras_funcs->reset_ras_error_count(adev);
3251         } else {
3252
3253                 task_barrier_full(&hive->tb);
3254                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3255         }
3256
3257 fail:
3258         if (adev->asic_reset_res)
3259                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3260                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3261         amdgpu_put_xgmi_hive(hive);
3262 }
3263
3264 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3265 {
3266         char *input = amdgpu_lockup_timeout;
3267         char *timeout_setting = NULL;
3268         int index = 0;
3269         long timeout;
3270         int ret = 0;
3271
3272         /*
3273          * By default timeout for non compute jobs is 10000
3274          * and 60000 for compute jobs.
3275          * In SR-IOV or passthrough mode, timeout for compute
3276          * jobs are 60000 by default.
3277          */
3278         adev->gfx_timeout = msecs_to_jiffies(10000);
3279         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3280         if (amdgpu_sriov_vf(adev))
3281                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3282                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3283         else
3284                 adev->compute_timeout =  msecs_to_jiffies(60000);
3285
3286         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3287                 while ((timeout_setting = strsep(&input, ",")) &&
3288                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3289                         ret = kstrtol(timeout_setting, 0, &timeout);
3290                         if (ret)
3291                                 return ret;
3292
3293                         if (timeout == 0) {
3294                                 index++;
3295                                 continue;
3296                         } else if (timeout < 0) {
3297                                 timeout = MAX_SCHEDULE_TIMEOUT;
3298                         } else {
3299                                 timeout = msecs_to_jiffies(timeout);
3300                         }
3301
3302                         switch (index++) {
3303                         case 0:
3304                                 adev->gfx_timeout = timeout;
3305                                 break;
3306                         case 1:
3307                                 adev->compute_timeout = timeout;
3308                                 break;
3309                         case 2:
3310                                 adev->sdma_timeout = timeout;
3311                                 break;
3312                         case 3:
3313                                 adev->video_timeout = timeout;
3314                                 break;
3315                         default:
3316                                 break;
3317                         }
3318                 }
3319                 /*
3320                  * There is only one value specified and
3321                  * it should apply to all non-compute jobs.
3322                  */
3323                 if (index == 1) {
3324                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3325                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3326                                 adev->compute_timeout = adev->gfx_timeout;
3327                 }
3328         }
3329
3330         return ret;
3331 }
3332
3333 static const struct attribute *amdgpu_dev_attributes[] = {
3334         &dev_attr_product_name.attr,
3335         &dev_attr_product_number.attr,
3336         &dev_attr_serial_number.attr,
3337         &dev_attr_pcie_replay_count.attr,
3338         NULL
3339 };
3340
3341 /**
3342  * amdgpu_device_init - initialize the driver
3343  *
3344  * @adev: amdgpu_device pointer
3345  * @flags: driver flags
3346  *
3347  * Initializes the driver info and hw (all asics).
3348  * Returns 0 for success or an error on failure.
3349  * Called at driver startup.
3350  */
3351 int amdgpu_device_init(struct amdgpu_device *adev,
3352                        uint32_t flags)
3353 {
3354         struct drm_device *ddev = adev_to_drm(adev);
3355         struct pci_dev *pdev = adev->pdev;
3356         int r, i;
3357         bool px = false;
3358         u32 max_MBps;
3359
3360         adev->shutdown = false;
3361         adev->flags = flags;
3362
3363         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3364                 adev->asic_type = amdgpu_force_asic_type;
3365         else
3366                 adev->asic_type = flags & AMD_ASIC_MASK;
3367
3368         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3369         if (amdgpu_emu_mode == 1)
3370                 adev->usec_timeout *= 10;
3371         adev->gmc.gart_size = 512 * 1024 * 1024;
3372         adev->accel_working = false;
3373         adev->num_rings = 0;
3374         adev->mman.buffer_funcs = NULL;
3375         adev->mman.buffer_funcs_ring = NULL;
3376         adev->vm_manager.vm_pte_funcs = NULL;
3377         adev->vm_manager.vm_pte_num_scheds = 0;
3378         adev->gmc.gmc_funcs = NULL;
3379         adev->harvest_ip_mask = 0x0;
3380         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3381         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3382
3383         adev->smc_rreg = &amdgpu_invalid_rreg;
3384         adev->smc_wreg = &amdgpu_invalid_wreg;
3385         adev->pcie_rreg = &amdgpu_invalid_rreg;
3386         adev->pcie_wreg = &amdgpu_invalid_wreg;
3387         adev->pciep_rreg = &amdgpu_invalid_rreg;
3388         adev->pciep_wreg = &amdgpu_invalid_wreg;
3389         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3390         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3391         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3392         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3393         adev->didt_rreg = &amdgpu_invalid_rreg;
3394         adev->didt_wreg = &amdgpu_invalid_wreg;
3395         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3396         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3397         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3398         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3399
3400         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3401                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3402                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3403
3404         /* mutex initialization are all done here so we
3405          * can recall function without having locking issues */
3406         mutex_init(&adev->firmware.mutex);
3407         mutex_init(&adev->pm.mutex);
3408         mutex_init(&adev->gfx.gpu_clock_mutex);
3409         mutex_init(&adev->srbm_mutex);
3410         mutex_init(&adev->gfx.pipe_reserve_mutex);
3411         mutex_init(&adev->gfx.gfx_off_mutex);
3412         mutex_init(&adev->grbm_idx_mutex);
3413         mutex_init(&adev->mn_lock);
3414         mutex_init(&adev->virt.vf_errors.lock);
3415         hash_init(adev->mn_hash);
3416         atomic_set(&adev->in_gpu_reset, 0);
3417         init_rwsem(&adev->reset_sem);
3418         mutex_init(&adev->psp.mutex);
3419         mutex_init(&adev->notifier_lock);
3420
3421         r = amdgpu_device_init_apu_flags(adev);
3422         if (r)
3423                 return r;
3424
3425         r = amdgpu_device_check_arguments(adev);
3426         if (r)
3427                 return r;
3428
3429         spin_lock_init(&adev->mmio_idx_lock);
3430         spin_lock_init(&adev->smc_idx_lock);
3431         spin_lock_init(&adev->pcie_idx_lock);
3432         spin_lock_init(&adev->uvd_ctx_idx_lock);
3433         spin_lock_init(&adev->didt_idx_lock);
3434         spin_lock_init(&adev->gc_cac_idx_lock);
3435         spin_lock_init(&adev->se_cac_idx_lock);
3436         spin_lock_init(&adev->audio_endpt_idx_lock);
3437         spin_lock_init(&adev->mm_stats.lock);
3438
3439         INIT_LIST_HEAD(&adev->shadow_list);
3440         mutex_init(&adev->shadow_list_lock);
3441
3442         INIT_LIST_HEAD(&adev->reset_list);
3443
3444         INIT_DELAYED_WORK(&adev->delayed_init_work,
3445                           amdgpu_device_delayed_init_work_handler);
3446         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3447                           amdgpu_device_delay_enable_gfx_off);
3448
3449         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3450
3451         adev->gfx.gfx_off_req_count = 1;
3452         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3453
3454         atomic_set(&adev->throttling_logging_enabled, 1);
3455         /*
3456          * If throttling continues, logging will be performed every minute
3457          * to avoid log flooding. "-1" is subtracted since the thermal
3458          * throttling interrupt comes every second. Thus, the total logging
3459          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3460          * for throttling interrupt) = 60 seconds.
3461          */
3462         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3463         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3464
3465         /* Registers mapping */
3466         /* TODO: block userspace mapping of io register */
3467         if (adev->asic_type >= CHIP_BONAIRE) {
3468                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3469                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3470         } else {
3471                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3472                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3473         }
3474
3475         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3476         if (adev->rmmio == NULL) {
3477                 return -ENOMEM;
3478         }
3479         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3480         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3481
3482         /* enable PCIE atomic ops */
3483         r = pci_enable_atomic_ops_to_root(adev->pdev,
3484                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3485                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3486         if (r) {
3487                 adev->have_atomics_support = false;
3488                 DRM_INFO("PCIE atomic ops is not supported\n");
3489         } else {
3490                 adev->have_atomics_support = true;
3491         }
3492
3493         amdgpu_device_get_pcie_info(adev);
3494
3495         if (amdgpu_mcbp)
3496                 DRM_INFO("MCBP is enabled\n");
3497
3498         if (amdgpu_mes && adev->asic_type >= CHIP_NAVI10)
3499                 adev->enable_mes = true;
3500
3501         /* detect hw virtualization here */
3502         amdgpu_detect_virtualization(adev);
3503
3504         r = amdgpu_device_get_job_timeout_settings(adev);
3505         if (r) {
3506                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3507                 return r;
3508         }
3509
3510         /* early init functions */
3511         r = amdgpu_device_ip_early_init(adev);
3512         if (r)
3513                 return r;
3514
3515         /* doorbell bar mapping and doorbell index init*/
3516         amdgpu_device_doorbell_init(adev);
3517
3518         if (amdgpu_emu_mode == 1) {
3519                 /* post the asic on emulation mode */
3520                 emu_soc_asic_init(adev);
3521                 goto fence_driver_init;
3522         }
3523
3524         amdgpu_reset_init(adev);
3525
3526         /* detect if we are with an SRIOV vbios */
3527         amdgpu_device_detect_sriov_bios(adev);
3528
3529         /* check if we need to reset the asic
3530          *  E.g., driver was not cleanly unloaded previously, etc.
3531          */
3532         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3533                 if (adev->gmc.xgmi.num_physical_nodes) {
3534                         dev_info(adev->dev, "Pending hive reset.\n");
3535                         adev->gmc.xgmi.pending_reset = true;
3536                         /* Only need to init necessary block for SMU to handle the reset */
3537                         for (i = 0; i < adev->num_ip_blocks; i++) {
3538                                 if (!adev->ip_blocks[i].status.valid)
3539                                         continue;
3540                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3541                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3542                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3543                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3544                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3545                                                 adev->ip_blocks[i].version->funcs->name);
3546                                         adev->ip_blocks[i].status.hw = true;
3547                                 }
3548                         }
3549                 } else {
3550                         r = amdgpu_asic_reset(adev);
3551                         if (r) {
3552                                 dev_err(adev->dev, "asic reset on init failed\n");
3553                                 goto failed;
3554                         }
3555                 }
3556         }
3557
3558         pci_enable_pcie_error_reporting(adev->pdev);
3559
3560         /* Post card if necessary */
3561         if (amdgpu_device_need_post(adev)) {
3562                 if (!adev->bios) {
3563                         dev_err(adev->dev, "no vBIOS found\n");
3564                         r = -EINVAL;
3565                         goto failed;
3566                 }
3567                 DRM_INFO("GPU posting now...\n");
3568                 r = amdgpu_device_asic_init(adev);
3569                 if (r) {
3570                         dev_err(adev->dev, "gpu post error!\n");
3571                         goto failed;
3572                 }
3573         }
3574
3575         if (adev->is_atom_fw) {
3576                 /* Initialize clocks */
3577                 r = amdgpu_atomfirmware_get_clock_info(adev);
3578                 if (r) {
3579                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3580                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3581                         goto failed;
3582                 }
3583         } else {
3584                 /* Initialize clocks */
3585                 r = amdgpu_atombios_get_clock_info(adev);
3586                 if (r) {
3587                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3588                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3589                         goto failed;
3590                 }
3591                 /* init i2c buses */
3592                 if (!amdgpu_device_has_dc_support(adev))
3593                         amdgpu_atombios_i2c_init(adev);
3594         }
3595
3596 fence_driver_init:
3597         /* Fence driver */
3598         r = amdgpu_fence_driver_init(adev);
3599         if (r) {
3600                 dev_err(adev->dev, "amdgpu_fence_driver_init failed\n");
3601                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3602                 goto failed;
3603         }
3604
3605         /* init the mode config */
3606         drm_mode_config_init(adev_to_drm(adev));
3607
3608         r = amdgpu_device_ip_init(adev);
3609         if (r) {
3610                 /* failed in exclusive mode due to timeout */
3611                 if (amdgpu_sriov_vf(adev) &&
3612                     !amdgpu_sriov_runtime(adev) &&
3613                     amdgpu_virt_mmio_blocked(adev) &&
3614                     !amdgpu_virt_wait_reset(adev)) {
3615                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3616                         /* Don't send request since VF is inactive. */
3617                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3618                         adev->virt.ops = NULL;
3619                         r = -EAGAIN;
3620                         goto release_ras_con;
3621                 }
3622                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3623                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3624                 goto release_ras_con;
3625         }
3626
3627         dev_info(adev->dev,
3628                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3629                         adev->gfx.config.max_shader_engines,
3630                         adev->gfx.config.max_sh_per_se,
3631                         adev->gfx.config.max_cu_per_sh,
3632                         adev->gfx.cu_info.number);
3633
3634         adev->accel_working = true;
3635
3636         amdgpu_vm_check_compute_bug(adev);
3637
3638         /* Initialize the buffer migration limit. */
3639         if (amdgpu_moverate >= 0)
3640                 max_MBps = amdgpu_moverate;
3641         else
3642                 max_MBps = 8; /* Allow 8 MB/s. */
3643         /* Get a log2 for easy divisions. */
3644         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3645
3646         amdgpu_fbdev_init(adev);
3647
3648         r = amdgpu_pm_sysfs_init(adev);
3649         if (r) {
3650                 adev->pm_sysfs_en = false;
3651                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3652         } else
3653                 adev->pm_sysfs_en = true;
3654
3655         r = amdgpu_ucode_sysfs_init(adev);
3656         if (r) {
3657                 adev->ucode_sysfs_en = false;
3658                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3659         } else
3660                 adev->ucode_sysfs_en = true;
3661
3662         if ((amdgpu_testing & 1)) {
3663                 if (adev->accel_working)
3664                         amdgpu_test_moves(adev);
3665                 else
3666                         DRM_INFO("amdgpu: acceleration disabled, skipping move tests\n");
3667         }
3668         if (amdgpu_benchmarking) {
3669                 if (adev->accel_working)
3670                         amdgpu_benchmark(adev, amdgpu_benchmarking);
3671                 else
3672                         DRM_INFO("amdgpu: acceleration disabled, skipping benchmarks\n");
3673         }
3674
3675         /*
3676          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3677          * Otherwise the mgpu fan boost feature will be skipped due to the
3678          * gpu instance is counted less.
3679          */
3680         amdgpu_register_gpu_instance(adev);
3681
3682         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3683          * explicit gating rather than handling it automatically.
3684          */
3685         if (!adev->gmc.xgmi.pending_reset) {
3686                 r = amdgpu_device_ip_late_init(adev);
3687                 if (r) {
3688                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3689                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3690                         goto release_ras_con;
3691                 }
3692                 /* must succeed. */
3693                 amdgpu_ras_resume(adev);
3694                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3695                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3696         }
3697
3698         if (amdgpu_sriov_vf(adev))
3699                 flush_delayed_work(&adev->delayed_init_work);
3700
3701         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3702         if (r)
3703                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3704
3705         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3706                 r = amdgpu_pmu_init(adev);
3707         if (r)
3708                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3709
3710         /* Have stored pci confspace at hand for restore in sudden PCI error */
3711         if (amdgpu_device_cache_pci_state(adev->pdev))
3712                 pci_restore_state(pdev);
3713
3714         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3715         /* this will fail for cards that aren't VGA class devices, just
3716          * ignore it */
3717         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3718                 vga_client_register(adev->pdev, adev, NULL, amdgpu_device_vga_set_decode);
3719
3720         if (amdgpu_device_supports_px(ddev)) {
3721                 px = true;
3722                 vga_switcheroo_register_client(adev->pdev,
3723                                                &amdgpu_switcheroo_ops, px);
3724                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3725         }
3726
3727         if (adev->gmc.xgmi.pending_reset)
3728                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3729                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3730
3731         return 0;
3732
3733 release_ras_con:
3734         amdgpu_release_ras_context(adev);
3735
3736 failed:
3737         amdgpu_vf_error_trans_all(adev);
3738
3739         return r;
3740 }
3741
3742 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3743 {
3744         /* Clear all CPU mappings pointing to this device */
3745         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3746
3747         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3748         amdgpu_device_doorbell_fini(adev);
3749
3750         iounmap(adev->rmmio);
3751         adev->rmmio = NULL;
3752         if (adev->mman.aper_base_kaddr)
3753                 iounmap(adev->mman.aper_base_kaddr);
3754         adev->mman.aper_base_kaddr = NULL;
3755
3756         /* Memory manager related */
3757         if (!adev->gmc.xgmi.connected_to_cpu) {
3758                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3759                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3760         }
3761 }
3762
3763 /**
3764  * amdgpu_device_fini - tear down the driver
3765  *
3766  * @adev: amdgpu_device pointer
3767  *
3768  * Tear down the driver info (all asics).
3769  * Called at driver shutdown.
3770  */
3771 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3772 {
3773         dev_info(adev->dev, "amdgpu: finishing device.\n");
3774         flush_delayed_work(&adev->delayed_init_work);
3775         ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3776         adev->shutdown = true;
3777
3778         /* make sure IB test finished before entering exclusive mode
3779          * to avoid preemption on IB test
3780          * */
3781         if (amdgpu_sriov_vf(adev)) {
3782                 amdgpu_virt_request_full_gpu(adev, false);
3783                 amdgpu_virt_fini_data_exchange(adev);
3784         }
3785
3786         /* disable all interrupts */
3787         amdgpu_irq_disable_all(adev);
3788         if (adev->mode_info.mode_config_initialized){
3789                 if (!amdgpu_device_has_dc_support(adev))
3790                         drm_helper_force_disable_all(adev_to_drm(adev));
3791                 else
3792                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3793         }
3794         amdgpu_fence_driver_fini_hw(adev);
3795
3796         if (adev->pm_sysfs_en)
3797                 amdgpu_pm_sysfs_fini(adev);
3798         if (adev->ucode_sysfs_en)
3799                 amdgpu_ucode_sysfs_fini(adev);
3800         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
3801
3802         amdgpu_fbdev_fini(adev);
3803
3804         amdgpu_irq_fini_hw(adev);
3805
3806         amdgpu_device_ip_fini_early(adev);
3807
3808         amdgpu_gart_dummy_page_fini(adev);
3809
3810         amdgpu_device_unmap_mmio(adev);
3811 }
3812
3813 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
3814 {
3815         amdgpu_device_ip_fini(adev);
3816         amdgpu_fence_driver_fini_sw(adev);
3817         release_firmware(adev->firmware.gpu_info_fw);
3818         adev->firmware.gpu_info_fw = NULL;
3819         adev->accel_working = false;
3820
3821         amdgpu_reset_fini(adev);
3822
3823         /* free i2c buses */
3824         if (!amdgpu_device_has_dc_support(adev))
3825                 amdgpu_i2c_fini(adev);
3826
3827         if (amdgpu_emu_mode != 1)
3828                 amdgpu_atombios_fini(adev);
3829
3830         kfree(adev->bios);
3831         adev->bios = NULL;
3832         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
3833                 vga_switcheroo_unregister_client(adev->pdev);
3834                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
3835         }
3836         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3837                 vga_client_register(adev->pdev, NULL, NULL, NULL);
3838
3839         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3840                 amdgpu_pmu_fini(adev);
3841         if (adev->mman.discovery_bin)
3842                 amdgpu_discovery_fini(adev);
3843
3844         kfree(adev->pci_state);
3845
3846 }
3847
3848
3849 /*
3850  * Suspend & resume.
3851  */
3852 /**
3853  * amdgpu_device_suspend - initiate device suspend
3854  *
3855  * @dev: drm dev pointer
3856  * @fbcon : notify the fbdev of suspend
3857  *
3858  * Puts the hw in the suspend state (all asics).
3859  * Returns 0 for success or an error on failure.
3860  * Called at driver suspend.
3861  */
3862 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
3863 {
3864         struct amdgpu_device *adev = drm_to_adev(dev);
3865
3866         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3867                 return 0;
3868
3869         adev->in_suspend = true;
3870
3871         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
3872                 DRM_WARN("smart shift update failed\n");
3873
3874         drm_kms_helper_poll_disable(dev);
3875
3876         if (fbcon)
3877                 amdgpu_fbdev_set_suspend(adev, 1);
3878
3879         cancel_delayed_work_sync(&adev->delayed_init_work);
3880
3881         amdgpu_ras_suspend(adev);
3882
3883         amdgpu_device_ip_suspend_phase1(adev);
3884
3885         if (!adev->in_s0ix)
3886                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
3887
3888         /* evict vram memory */
3889         amdgpu_bo_evict_vram(adev);
3890
3891         amdgpu_fence_driver_suspend(adev);
3892
3893         amdgpu_device_ip_suspend_phase2(adev);
3894         /* evict remaining vram memory
3895          * This second call to evict vram is to evict the gart page table
3896          * using the CPU.
3897          */
3898         amdgpu_bo_evict_vram(adev);
3899
3900         return 0;
3901 }
3902
3903 /**
3904  * amdgpu_device_resume - initiate device resume
3905  *
3906  * @dev: drm dev pointer
3907  * @fbcon : notify the fbdev of resume
3908  *
3909  * Bring the hw back to operating state (all asics).
3910  * Returns 0 for success or an error on failure.
3911  * Called at driver resume.
3912  */
3913 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
3914 {
3915         struct amdgpu_device *adev = drm_to_adev(dev);
3916         int r = 0;
3917
3918         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
3919                 return 0;
3920
3921         if (adev->in_s0ix)
3922                 amdgpu_gfx_state_change_set(adev, sGpuChangeState_D0Entry);
3923
3924         /* post card */
3925         if (amdgpu_device_need_post(adev)) {
3926                 r = amdgpu_device_asic_init(adev);
3927                 if (r)
3928                         dev_err(adev->dev, "amdgpu asic init failed\n");
3929         }
3930
3931         r = amdgpu_device_ip_resume(adev);
3932         if (r) {
3933                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
3934                 return r;
3935         }
3936         amdgpu_fence_driver_resume(adev);
3937
3938
3939         r = amdgpu_device_ip_late_init(adev);
3940         if (r)
3941                 return r;
3942
3943         queue_delayed_work(system_wq, &adev->delayed_init_work,
3944                            msecs_to_jiffies(AMDGPU_RESUME_MS));
3945
3946         if (!adev->in_s0ix) {
3947                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
3948                 if (r)
3949                         return r;
3950         }
3951
3952         /* Make sure IB tests flushed */
3953         flush_delayed_work(&adev->delayed_init_work);
3954
3955         if (fbcon)
3956                 amdgpu_fbdev_set_suspend(adev, 0);
3957
3958         drm_kms_helper_poll_enable(dev);
3959
3960         amdgpu_ras_resume(adev);
3961
3962         /*
3963          * Most of the connector probing functions try to acquire runtime pm
3964          * refs to ensure that the GPU is powered on when connector polling is
3965          * performed. Since we're calling this from a runtime PM callback,
3966          * trying to acquire rpm refs will cause us to deadlock.
3967          *
3968          * Since we're guaranteed to be holding the rpm lock, it's safe to
3969          * temporarily disable the rpm helpers so this doesn't deadlock us.
3970          */
3971 #ifdef CONFIG_PM
3972         dev->dev->power.disable_depth++;
3973 #endif
3974         if (!amdgpu_device_has_dc_support(adev))
3975                 drm_helper_hpd_irq_event(dev);
3976         else
3977                 drm_kms_helper_hotplug_event(dev);
3978 #ifdef CONFIG_PM
3979         dev->dev->power.disable_depth--;
3980 #endif
3981         adev->in_suspend = false;
3982
3983         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
3984                 DRM_WARN("smart shift update failed\n");
3985
3986         return 0;
3987 }
3988
3989 /**
3990  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
3991  *
3992  * @adev: amdgpu_device pointer
3993  *
3994  * The list of all the hardware IPs that make up the asic is walked and
3995  * the check_soft_reset callbacks are run.  check_soft_reset determines
3996  * if the asic is still hung or not.
3997  * Returns true if any of the IPs are still in a hung state, false if not.
3998  */
3999 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4000 {
4001         int i;
4002         bool asic_hang = false;
4003
4004         if (amdgpu_sriov_vf(adev))
4005                 return true;
4006
4007         if (amdgpu_asic_need_full_reset(adev))
4008                 return true;
4009
4010         for (i = 0; i < adev->num_ip_blocks; i++) {
4011                 if (!adev->ip_blocks[i].status.valid)
4012                         continue;
4013                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4014                         adev->ip_blocks[i].status.hang =
4015                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4016                 if (adev->ip_blocks[i].status.hang) {
4017                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4018                         asic_hang = true;
4019                 }
4020         }
4021         return asic_hang;
4022 }
4023
4024 /**
4025  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4026  *
4027  * @adev: amdgpu_device pointer
4028  *
4029  * The list of all the hardware IPs that make up the asic is walked and the
4030  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4031  * handles any IP specific hardware or software state changes that are
4032  * necessary for a soft reset to succeed.
4033  * Returns 0 on success, negative error code on failure.
4034  */
4035 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4036 {
4037         int i, r = 0;
4038
4039         for (i = 0; i < adev->num_ip_blocks; i++) {
4040                 if (!adev->ip_blocks[i].status.valid)
4041                         continue;
4042                 if (adev->ip_blocks[i].status.hang &&
4043                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4044                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4045                         if (r)
4046                                 return r;
4047                 }
4048         }
4049
4050         return 0;
4051 }
4052
4053 /**
4054  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4055  *
4056  * @adev: amdgpu_device pointer
4057  *
4058  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4059  * reset is necessary to recover.
4060  * Returns true if a full asic reset is required, false if not.
4061  */
4062 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4063 {
4064         int i;
4065
4066         if (amdgpu_asic_need_full_reset(adev))
4067                 return true;
4068
4069         for (i = 0; i < adev->num_ip_blocks; i++) {
4070                 if (!adev->ip_blocks[i].status.valid)
4071                         continue;
4072                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4073                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4074                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4075                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4076                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4077                         if (adev->ip_blocks[i].status.hang) {
4078                                 dev_info(adev->dev, "Some block need full reset!\n");
4079                                 return true;
4080                         }
4081                 }
4082         }
4083         return false;
4084 }
4085
4086 /**
4087  * amdgpu_device_ip_soft_reset - do a soft reset
4088  *
4089  * @adev: amdgpu_device pointer
4090  *
4091  * The list of all the hardware IPs that make up the asic is walked and the
4092  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4093  * IP specific hardware or software state changes that are necessary to soft
4094  * reset the IP.
4095  * Returns 0 on success, negative error code on failure.
4096  */
4097 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4098 {
4099         int i, r = 0;
4100
4101         for (i = 0; i < adev->num_ip_blocks; i++) {
4102                 if (!adev->ip_blocks[i].status.valid)
4103                         continue;
4104                 if (adev->ip_blocks[i].status.hang &&
4105                     adev->ip_blocks[i].version->funcs->soft_reset) {
4106                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4107                         if (r)
4108                                 return r;
4109                 }
4110         }
4111
4112         return 0;
4113 }
4114
4115 /**
4116  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4117  *
4118  * @adev: amdgpu_device pointer
4119  *
4120  * The list of all the hardware IPs that make up the asic is walked and the
4121  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4122  * handles any IP specific hardware or software state changes that are
4123  * necessary after the IP has been soft reset.
4124  * Returns 0 on success, negative error code on failure.
4125  */
4126 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4127 {
4128         int i, r = 0;
4129
4130         for (i = 0; i < adev->num_ip_blocks; i++) {
4131                 if (!adev->ip_blocks[i].status.valid)
4132                         continue;
4133                 if (adev->ip_blocks[i].status.hang &&
4134                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4135                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4136                 if (r)
4137                         return r;
4138         }
4139
4140         return 0;
4141 }
4142
4143 /**
4144  * amdgpu_device_recover_vram - Recover some VRAM contents
4145  *
4146  * @adev: amdgpu_device pointer
4147  *
4148  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4149  * restore things like GPUVM page tables after a GPU reset where
4150  * the contents of VRAM might be lost.
4151  *
4152  * Returns:
4153  * 0 on success, negative error code on failure.
4154  */
4155 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4156 {
4157         struct dma_fence *fence = NULL, *next = NULL;
4158         struct amdgpu_bo *shadow;
4159         struct amdgpu_bo_vm *vmbo;
4160         long r = 1, tmo;
4161
4162         if (amdgpu_sriov_runtime(adev))
4163                 tmo = msecs_to_jiffies(8000);
4164         else
4165                 tmo = msecs_to_jiffies(100);
4166
4167         dev_info(adev->dev, "recover vram bo from shadow start\n");
4168         mutex_lock(&adev->shadow_list_lock);
4169         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4170                 shadow = &vmbo->bo;
4171                 /* No need to recover an evicted BO */
4172                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4173                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4174                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4175                         continue;
4176
4177                 r = amdgpu_bo_restore_shadow(shadow, &next);
4178                 if (r)
4179                         break;
4180
4181                 if (fence) {
4182                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4183                         dma_fence_put(fence);
4184                         fence = next;
4185                         if (tmo == 0) {
4186                                 r = -ETIMEDOUT;
4187                                 break;
4188                         } else if (tmo < 0) {
4189                                 r = tmo;
4190                                 break;
4191                         }
4192                 } else {
4193                         fence = next;
4194                 }
4195         }
4196         mutex_unlock(&adev->shadow_list_lock);
4197
4198         if (fence)
4199                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4200         dma_fence_put(fence);
4201
4202         if (r < 0 || tmo <= 0) {
4203                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4204                 return -EIO;
4205         }
4206
4207         dev_info(adev->dev, "recover vram bo from shadow done\n");
4208         return 0;
4209 }
4210
4211
4212 /**
4213  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4214  *
4215  * @adev: amdgpu_device pointer
4216  * @from_hypervisor: request from hypervisor
4217  *
4218  * do VF FLR and reinitialize Asic
4219  * return 0 means succeeded otherwise failed
4220  */
4221 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4222                                      bool from_hypervisor)
4223 {
4224         int r;
4225
4226         if (from_hypervisor)
4227                 r = amdgpu_virt_request_full_gpu(adev, true);
4228         else
4229                 r = amdgpu_virt_reset_gpu(adev);
4230         if (r)
4231                 return r;
4232
4233         amdgpu_amdkfd_pre_reset(adev);
4234
4235         /* Resume IP prior to SMC */
4236         r = amdgpu_device_ip_reinit_early_sriov(adev);
4237         if (r)
4238                 goto error;
4239
4240         amdgpu_virt_init_data_exchange(adev);
4241         /* we need recover gart prior to run SMC/CP/SDMA resume */
4242         amdgpu_gtt_mgr_recover(ttm_manager_type(&adev->mman.bdev, TTM_PL_TT));
4243
4244         r = amdgpu_device_fw_loading(adev);
4245         if (r)
4246                 return r;
4247
4248         /* now we are okay to resume SMC/CP/SDMA */
4249         r = amdgpu_device_ip_reinit_late_sriov(adev);
4250         if (r)
4251                 goto error;
4252
4253         amdgpu_irq_gpu_reset_resume_helper(adev);
4254         r = amdgpu_ib_ring_tests(adev);
4255         amdgpu_amdkfd_post_reset(adev);
4256
4257 error:
4258         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4259                 amdgpu_inc_vram_lost(adev);
4260                 r = amdgpu_device_recover_vram(adev);
4261         }
4262         amdgpu_virt_release_full_gpu(adev, true);
4263
4264         return r;
4265 }
4266
4267 /**
4268  * amdgpu_device_has_job_running - check if there is any job in mirror list
4269  *
4270  * @adev: amdgpu_device pointer
4271  *
4272  * check if there is any job in mirror list
4273  */
4274 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4275 {
4276         int i;
4277         struct drm_sched_job *job;
4278
4279         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4280                 struct amdgpu_ring *ring = adev->rings[i];
4281
4282                 if (!ring || !ring->sched.thread)
4283                         continue;
4284
4285                 spin_lock(&ring->sched.job_list_lock);
4286                 job = list_first_entry_or_null(&ring->sched.pending_list,
4287                                                struct drm_sched_job, list);
4288                 spin_unlock(&ring->sched.job_list_lock);
4289                 if (job)
4290                         return true;
4291         }
4292         return false;
4293 }
4294
4295 /**
4296  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4297  *
4298  * @adev: amdgpu_device pointer
4299  *
4300  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4301  * a hung GPU.
4302  */
4303 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4304 {
4305         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4306                 dev_info(adev->dev, "Timeout, but no hardware hang detected.\n");
4307                 return false;
4308         }
4309
4310         if (amdgpu_gpu_recovery == 0)
4311                 goto disabled;
4312
4313         if (amdgpu_sriov_vf(adev))
4314                 return true;
4315
4316         if (amdgpu_gpu_recovery == -1) {
4317                 switch (adev->asic_type) {
4318                 case CHIP_BONAIRE:
4319                 case CHIP_HAWAII:
4320                 case CHIP_TOPAZ:
4321                 case CHIP_TONGA:
4322                 case CHIP_FIJI:
4323                 case CHIP_POLARIS10:
4324                 case CHIP_POLARIS11:
4325                 case CHIP_POLARIS12:
4326                 case CHIP_VEGAM:
4327                 case CHIP_VEGA20:
4328                 case CHIP_VEGA10:
4329                 case CHIP_VEGA12:
4330                 case CHIP_RAVEN:
4331                 case CHIP_ARCTURUS:
4332                 case CHIP_RENOIR:
4333                 case CHIP_NAVI10:
4334                 case CHIP_NAVI14:
4335                 case CHIP_NAVI12:
4336                 case CHIP_SIENNA_CICHLID:
4337                 case CHIP_NAVY_FLOUNDER:
4338                 case CHIP_DIMGREY_CAVEFISH:
4339                 case CHIP_BEIGE_GOBY:
4340                 case CHIP_VANGOGH:
4341                 case CHIP_ALDEBARAN:
4342                         break;
4343                 default:
4344                         goto disabled;
4345                 }
4346         }
4347
4348         return true;
4349
4350 disabled:
4351                 dev_info(adev->dev, "GPU recovery disabled.\n");
4352                 return false;
4353 }
4354
4355 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4356 {
4357         u32 i;
4358         int ret = 0;
4359
4360         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4361
4362         dev_info(adev->dev, "GPU mode1 reset\n");
4363
4364         /* disable BM */
4365         pci_clear_master(adev->pdev);
4366
4367         amdgpu_device_cache_pci_state(adev->pdev);
4368
4369         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4370                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4371                 ret = amdgpu_dpm_mode1_reset(adev);
4372         } else {
4373                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4374                 ret = psp_gpu_reset(adev);
4375         }
4376
4377         if (ret)
4378                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4379
4380         amdgpu_device_load_pci_state(adev->pdev);
4381
4382         /* wait for asic to come out of reset */
4383         for (i = 0; i < adev->usec_timeout; i++) {
4384                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4385
4386                 if (memsize != 0xffffffff)
4387                         break;
4388                 udelay(1);
4389         }
4390
4391         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4392         return ret;
4393 }
4394
4395 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4396                                  struct amdgpu_reset_context *reset_context)
4397 {
4398         int i, r = 0;
4399         struct amdgpu_job *job = NULL;
4400         bool need_full_reset =
4401                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4402
4403         if (reset_context->reset_req_dev == adev)
4404                 job = reset_context->job;
4405
4406         /* no need to dump if device is not in good state during probe period */
4407         if (!adev->gmc.xgmi.pending_reset)
4408                 amdgpu_debugfs_wait_dump(adev);
4409
4410         if (amdgpu_sriov_vf(adev)) {
4411                 /* stop the data exchange thread */
4412                 amdgpu_virt_fini_data_exchange(adev);
4413         }
4414
4415         /* block all schedulers and reset given job's ring */
4416         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4417                 struct amdgpu_ring *ring = adev->rings[i];
4418
4419                 if (!ring || !ring->sched.thread)
4420                         continue;
4421
4422                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4423                 amdgpu_fence_driver_force_completion(ring);
4424         }
4425
4426         if(job)
4427                 drm_sched_increase_karma(&job->base);
4428
4429         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4430         /* If reset handler not implemented, continue; otherwise return */
4431         if (r == -ENOSYS)
4432                 r = 0;
4433         else
4434                 return r;
4435
4436         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4437         if (!amdgpu_sriov_vf(adev)) {
4438
4439                 if (!need_full_reset)
4440                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4441
4442                 if (!need_full_reset) {
4443                         amdgpu_device_ip_pre_soft_reset(adev);
4444                         r = amdgpu_device_ip_soft_reset(adev);
4445                         amdgpu_device_ip_post_soft_reset(adev);
4446                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4447                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4448                                 need_full_reset = true;
4449                         }
4450                 }
4451
4452                 if (need_full_reset)
4453                         r = amdgpu_device_ip_suspend(adev);
4454                 if (need_full_reset)
4455                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4456                 else
4457                         clear_bit(AMDGPU_NEED_FULL_RESET,
4458                                   &reset_context->flags);
4459         }
4460
4461         return r;
4462 }
4463
4464 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4465                          struct amdgpu_reset_context *reset_context)
4466 {
4467         struct amdgpu_device *tmp_adev = NULL;
4468         bool need_full_reset, skip_hw_reset, vram_lost = false;
4469         int r = 0;
4470
4471         /* Try reset handler method first */
4472         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4473                                     reset_list);
4474         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4475         /* If reset handler not implemented, continue; otherwise return */
4476         if (r == -ENOSYS)
4477                 r = 0;
4478         else
4479                 return r;
4480
4481         /* Reset handler not implemented, use the default method */
4482         need_full_reset =
4483                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4484         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4485
4486         /*
4487          * ASIC reset has to be done on all XGMI hive nodes ASAP
4488          * to allow proper links negotiation in FW (within 1 sec)
4489          */
4490         if (!skip_hw_reset && need_full_reset) {
4491                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4492                         /* For XGMI run all resets in parallel to speed up the process */
4493                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4494                                 tmp_adev->gmc.xgmi.pending_reset = false;
4495                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4496                                         r = -EALREADY;
4497                         } else
4498                                 r = amdgpu_asic_reset(tmp_adev);
4499
4500                         if (r) {
4501                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4502                                          r, adev_to_drm(tmp_adev)->unique);
4503                                 break;
4504                         }
4505                 }
4506
4507                 /* For XGMI wait for all resets to complete before proceed */
4508                 if (!r) {
4509                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4510                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4511                                         flush_work(&tmp_adev->xgmi_reset_work);
4512                                         r = tmp_adev->asic_reset_res;
4513                                         if (r)
4514                                                 break;
4515                                 }
4516                         }
4517                 }
4518         }
4519
4520         if (!r && amdgpu_ras_intr_triggered()) {
4521                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4522                         if (tmp_adev->mmhub.ras_funcs &&
4523                             tmp_adev->mmhub.ras_funcs->reset_ras_error_count)
4524                                 tmp_adev->mmhub.ras_funcs->reset_ras_error_count(tmp_adev);
4525                 }
4526
4527                 amdgpu_ras_intr_cleared();
4528         }
4529
4530         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4531                 if (need_full_reset) {
4532                         /* post card */
4533                         r = amdgpu_device_asic_init(tmp_adev);
4534                         if (r) {
4535                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4536                         } else {
4537                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4538                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4539                                 if (r)
4540                                         goto out;
4541
4542                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4543                                 if (vram_lost) {
4544                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4545                                         amdgpu_inc_vram_lost(tmp_adev);
4546                                 }
4547
4548                                 r = amdgpu_gtt_mgr_recover(ttm_manager_type(&tmp_adev->mman.bdev, TTM_PL_TT));
4549                                 if (r)
4550                                         goto out;
4551
4552                                 r = amdgpu_device_fw_loading(tmp_adev);
4553                                 if (r)
4554                                         return r;
4555
4556                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4557                                 if (r)
4558                                         goto out;
4559
4560                                 if (vram_lost)
4561                                         amdgpu_device_fill_reset_magic(tmp_adev);
4562
4563                                 /*
4564                                  * Add this ASIC as tracked as reset was already
4565                                  * complete successfully.
4566                                  */
4567                                 amdgpu_register_gpu_instance(tmp_adev);
4568
4569                                 if (!reset_context->hive &&
4570                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4571                                         amdgpu_xgmi_add_device(tmp_adev);
4572
4573                                 r = amdgpu_device_ip_late_init(tmp_adev);
4574                                 if (r)
4575                                         goto out;
4576
4577                                 amdgpu_fbdev_set_suspend(tmp_adev, 0);
4578
4579                                 /*
4580                                  * The GPU enters bad state once faulty pages
4581                                  * by ECC has reached the threshold, and ras
4582                                  * recovery is scheduled next. So add one check
4583                                  * here to break recovery if it indeed exceeds
4584                                  * bad page threshold, and remind user to
4585                                  * retire this GPU or setting one bigger
4586                                  * bad_page_threshold value to fix this once
4587                                  * probing driver again.
4588                                  */
4589                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4590                                         /* must succeed. */
4591                                         amdgpu_ras_resume(tmp_adev);
4592                                 } else {
4593                                         r = -EINVAL;
4594                                         goto out;
4595                                 }
4596
4597                                 /* Update PSP FW topology after reset */
4598                                 if (reset_context->hive &&
4599                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4600                                         r = amdgpu_xgmi_update_topology(
4601                                                 reset_context->hive, tmp_adev);
4602                         }
4603                 }
4604
4605 out:
4606                 if (!r) {
4607                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4608                         r = amdgpu_ib_ring_tests(tmp_adev);
4609                         if (r) {
4610                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
4611                                 need_full_reset = true;
4612                                 r = -EAGAIN;
4613                                 goto end;
4614                         }
4615                 }
4616
4617                 if (!r)
4618                         r = amdgpu_device_recover_vram(tmp_adev);
4619                 else
4620                         tmp_adev->asic_reset_res = r;
4621         }
4622
4623 end:
4624         if (need_full_reset)
4625                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4626         else
4627                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4628         return r;
4629 }
4630
4631 static bool amdgpu_device_lock_adev(struct amdgpu_device *adev,
4632                                 struct amdgpu_hive_info *hive)
4633 {
4634         if (atomic_cmpxchg(&adev->in_gpu_reset, 0, 1) != 0)
4635                 return false;
4636
4637         if (hive) {
4638                 down_write_nest_lock(&adev->reset_sem, &hive->hive_lock);
4639         } else {
4640                 down_write(&adev->reset_sem);
4641         }
4642
4643         switch (amdgpu_asic_reset_method(adev)) {
4644         case AMD_RESET_METHOD_MODE1:
4645                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
4646                 break;
4647         case AMD_RESET_METHOD_MODE2:
4648                 adev->mp1_state = PP_MP1_STATE_RESET;
4649                 break;
4650         default:
4651                 adev->mp1_state = PP_MP1_STATE_NONE;
4652                 break;
4653         }
4654
4655         return true;
4656 }
4657
4658 static void amdgpu_device_unlock_adev(struct amdgpu_device *adev)
4659 {
4660         amdgpu_vf_error_trans_all(adev);
4661         adev->mp1_state = PP_MP1_STATE_NONE;
4662         atomic_set(&adev->in_gpu_reset, 0);
4663         up_write(&adev->reset_sem);
4664 }
4665
4666 /*
4667  * to lockup a list of amdgpu devices in a hive safely, if not a hive
4668  * with multiple nodes, it will be similar as amdgpu_device_lock_adev.
4669  *
4670  * unlock won't require roll back.
4671  */
4672 static int amdgpu_device_lock_hive_adev(struct amdgpu_device *adev, struct amdgpu_hive_info *hive)
4673 {
4674         struct amdgpu_device *tmp_adev = NULL;
4675
4676         if (adev->gmc.xgmi.num_physical_nodes > 1) {
4677                 if (!hive) {
4678                         dev_err(adev->dev, "Hive is NULL while device has multiple xgmi nodes");
4679                         return -ENODEV;
4680                 }
4681                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4682                         if (!amdgpu_device_lock_adev(tmp_adev, hive))
4683                                 goto roll_back;
4684                 }
4685         } else if (!amdgpu_device_lock_adev(adev, hive))
4686                 return -EAGAIN;
4687
4688         return 0;
4689 roll_back:
4690         if (!list_is_first(&tmp_adev->gmc.xgmi.head, &hive->device_list)) {
4691                 /*
4692                  * if the lockup iteration break in the middle of a hive,
4693                  * it may means there may has a race issue,
4694                  * or a hive device locked up independently.
4695                  * we may be in trouble and may not, so will try to roll back
4696                  * the lock and give out a warnning.
4697                  */
4698                 dev_warn(tmp_adev->dev, "Hive lock iteration broke in the middle. Rolling back to unlock");
4699                 list_for_each_entry_continue_reverse(tmp_adev, &hive->device_list, gmc.xgmi.head) {
4700                         amdgpu_device_unlock_adev(tmp_adev);
4701                 }
4702         }
4703         return -EAGAIN;
4704 }
4705
4706 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
4707 {
4708         struct pci_dev *p = NULL;
4709
4710         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4711                         adev->pdev->bus->number, 1);
4712         if (p) {
4713                 pm_runtime_enable(&(p->dev));
4714                 pm_runtime_resume(&(p->dev));
4715         }
4716 }
4717
4718 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
4719 {
4720         enum amd_reset_method reset_method;
4721         struct pci_dev *p = NULL;
4722         u64 expires;
4723
4724         /*
4725          * For now, only BACO and mode1 reset are confirmed
4726          * to suffer the audio issue without proper suspended.
4727          */
4728         reset_method = amdgpu_asic_reset_method(adev);
4729         if ((reset_method != AMD_RESET_METHOD_BACO) &&
4730              (reset_method != AMD_RESET_METHOD_MODE1))
4731                 return -EINVAL;
4732
4733         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
4734                         adev->pdev->bus->number, 1);
4735         if (!p)
4736                 return -ENODEV;
4737
4738         expires = pm_runtime_autosuspend_expiration(&(p->dev));
4739         if (!expires)
4740                 /*
4741                  * If we cannot get the audio device autosuspend delay,
4742                  * a fixed 4S interval will be used. Considering 3S is
4743                  * the audio controller default autosuspend delay setting.
4744                  * 4S used here is guaranteed to cover that.
4745                  */
4746                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
4747
4748         while (!pm_runtime_status_suspended(&(p->dev))) {
4749                 if (!pm_runtime_suspend(&(p->dev)))
4750                         break;
4751
4752                 if (expires < ktime_get_mono_fast_ns()) {
4753                         dev_warn(adev->dev, "failed to suspend display audio\n");
4754                         /* TODO: abort the succeeding gpu reset? */
4755                         return -ETIMEDOUT;
4756                 }
4757         }
4758
4759         pm_runtime_disable(&(p->dev));
4760
4761         return 0;
4762 }
4763
4764 static void amdgpu_device_recheck_guilty_jobs(
4765         struct amdgpu_device *adev, struct list_head *device_list_handle,
4766         struct amdgpu_reset_context *reset_context)
4767 {
4768         int i, r = 0;
4769
4770         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4771                 struct amdgpu_ring *ring = adev->rings[i];
4772                 int ret = 0;
4773                 struct drm_sched_job *s_job;
4774
4775                 if (!ring || !ring->sched.thread)
4776                         continue;
4777
4778                 s_job = list_first_entry_or_null(&ring->sched.pending_list,
4779                                 struct drm_sched_job, list);
4780                 if (s_job == NULL)
4781                         continue;
4782
4783                 /* clear job's guilty and depend the folowing step to decide the real one */
4784                 drm_sched_reset_karma(s_job);
4785                 drm_sched_resubmit_jobs_ext(&ring->sched, 1);
4786
4787                 ret = dma_fence_wait_timeout(s_job->s_fence->parent, false, ring->sched.timeout);
4788                 if (ret == 0) { /* timeout */
4789                         DRM_ERROR("Found the real bad job! ring:%s, job_id:%llx\n",
4790                                                 ring->sched.name, s_job->id);
4791
4792                         /* set guilty */
4793                         drm_sched_increase_karma(s_job);
4794 retry:
4795                         /* do hw reset */
4796                         if (amdgpu_sriov_vf(adev)) {
4797                                 amdgpu_virt_fini_data_exchange(adev);
4798                                 r = amdgpu_device_reset_sriov(adev, false);
4799                                 if (r)
4800                                         adev->asic_reset_res = r;
4801                         } else {
4802                                 clear_bit(AMDGPU_SKIP_HW_RESET,
4803                                           &reset_context->flags);
4804                                 r = amdgpu_do_asic_reset(device_list_handle,
4805                                                          reset_context);
4806                                 if (r && r == -EAGAIN)
4807                                         goto retry;
4808                         }
4809
4810                         /*
4811                          * add reset counter so that the following
4812                          * resubmitted job could flush vmid
4813                          */
4814                         atomic_inc(&adev->gpu_reset_counter);
4815                         continue;
4816                 }
4817
4818                 /* got the hw fence, signal finished fence */
4819                 atomic_dec(ring->sched.score);
4820                 dma_fence_get(&s_job->s_fence->finished);
4821                 dma_fence_signal(&s_job->s_fence->finished);
4822                 dma_fence_put(&s_job->s_fence->finished);
4823
4824                 /* remove node from list and free the job */
4825                 spin_lock(&ring->sched.job_list_lock);
4826                 list_del_init(&s_job->list);
4827                 spin_unlock(&ring->sched.job_list_lock);
4828                 ring->sched.ops->free_job(s_job);
4829         }
4830 }
4831
4832 /**
4833  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
4834  *
4835  * @adev: amdgpu_device pointer
4836  * @job: which job trigger hang
4837  *
4838  * Attempt to reset the GPU if it has hung (all asics).
4839  * Attempt to do soft-reset or full-reset and reinitialize Asic
4840  * Returns 0 for success or an error on failure.
4841  */
4842
4843 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
4844                               struct amdgpu_job *job)
4845 {
4846         struct list_head device_list, *device_list_handle =  NULL;
4847         bool job_signaled = false;
4848         struct amdgpu_hive_info *hive = NULL;
4849         struct amdgpu_device *tmp_adev = NULL;
4850         int i, r = 0;
4851         bool need_emergency_restart = false;
4852         bool audio_suspended = false;
4853         int tmp_vram_lost_counter;
4854         struct amdgpu_reset_context reset_context;
4855
4856         memset(&reset_context, 0, sizeof(reset_context));
4857
4858         /*
4859          * Special case: RAS triggered and full reset isn't supported
4860          */
4861         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
4862
4863         /*
4864          * Flush RAM to disk so that after reboot
4865          * the user can read log and see why the system rebooted.
4866          */
4867         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
4868                 DRM_WARN("Emergency reboot.");
4869
4870                 ksys_sync_helper();
4871                 emergency_restart();
4872         }
4873
4874         dev_info(adev->dev, "GPU %s begin!\n",
4875                 need_emergency_restart ? "jobs stop":"reset");
4876
4877         /*
4878          * Here we trylock to avoid chain of resets executing from
4879          * either trigger by jobs on different adevs in XGMI hive or jobs on
4880          * different schedulers for same device while this TO handler is running.
4881          * We always reset all schedulers for device and all devices for XGMI
4882          * hive so that should take care of them too.
4883          */
4884         hive = amdgpu_get_xgmi_hive(adev);
4885         if (hive) {
4886                 if (atomic_cmpxchg(&hive->in_reset, 0, 1) != 0) {
4887                         DRM_INFO("Bailing on TDR for s_job:%llx, hive: %llx as another already in progress",
4888                                 job ? job->base.id : -1, hive->hive_id);
4889                         amdgpu_put_xgmi_hive(hive);
4890                         if (job)
4891                                 drm_sched_increase_karma(&job->base);
4892                         return 0;
4893                 }
4894                 mutex_lock(&hive->hive_lock);
4895         }
4896
4897         reset_context.method = AMD_RESET_METHOD_NONE;
4898         reset_context.reset_req_dev = adev;
4899         reset_context.job = job;
4900         reset_context.hive = hive;
4901         clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
4902
4903         /*
4904          * lock the device before we try to operate the linked list
4905          * if didn't get the device lock, don't touch the linked list since
4906          * others may iterating it.
4907          */
4908         r = amdgpu_device_lock_hive_adev(adev, hive);
4909         if (r) {
4910                 dev_info(adev->dev, "Bailing on TDR for s_job:%llx, as another already in progress",
4911                                         job ? job->base.id : -1);
4912
4913                 /* even we skipped this reset, still need to set the job to guilty */
4914                 if (job)
4915                         drm_sched_increase_karma(&job->base);
4916                 goto skip_recovery;
4917         }
4918
4919         /*
4920          * Build list of devices to reset.
4921          * In case we are in XGMI hive mode, resort the device list
4922          * to put adev in the 1st position.
4923          */
4924         INIT_LIST_HEAD(&device_list);
4925         if (adev->gmc.xgmi.num_physical_nodes > 1) {
4926                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head)
4927                         list_add_tail(&tmp_adev->reset_list, &device_list);
4928                 if (!list_is_first(&adev->reset_list, &device_list))
4929                         list_rotate_to_front(&adev->reset_list, &device_list);
4930                 device_list_handle = &device_list;
4931         } else {
4932                 list_add_tail(&adev->reset_list, &device_list);
4933                 device_list_handle = &device_list;
4934         }
4935
4936         /* block all schedulers and reset given job's ring */
4937         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4938                 /*
4939                  * Try to put the audio codec into suspend state
4940                  * before gpu reset started.
4941                  *
4942                  * Due to the power domain of the graphics device
4943                  * is shared with AZ power domain. Without this,
4944                  * we may change the audio hardware from behind
4945                  * the audio driver's back. That will trigger
4946                  * some audio codec errors.
4947                  */
4948                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
4949                         audio_suspended = true;
4950
4951                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
4952
4953                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
4954
4955                 if (!amdgpu_sriov_vf(tmp_adev))
4956                         amdgpu_amdkfd_pre_reset(tmp_adev);
4957
4958                 /*
4959                  * Mark these ASICs to be reseted as untracked first
4960                  * And add them back after reset completed
4961                  */
4962                 amdgpu_unregister_gpu_instance(tmp_adev);
4963
4964                 amdgpu_fbdev_set_suspend(tmp_adev, 1);
4965
4966                 /* disable ras on ALL IPs */
4967                 if (!need_emergency_restart &&
4968                       amdgpu_device_ip_need_full_reset(tmp_adev))
4969                         amdgpu_ras_suspend(tmp_adev);
4970
4971                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4972                         struct amdgpu_ring *ring = tmp_adev->rings[i];
4973
4974                         if (!ring || !ring->sched.thread)
4975                                 continue;
4976
4977                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
4978
4979                         if (need_emergency_restart)
4980                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
4981                 }
4982                 atomic_inc(&tmp_adev->gpu_reset_counter);
4983         }
4984
4985         if (need_emergency_restart)
4986                 goto skip_sched_resume;
4987
4988         /*
4989          * Must check guilty signal here since after this point all old
4990          * HW fences are force signaled.
4991          *
4992          * job->base holds a reference to parent fence
4993          */
4994         if (job && job->base.s_fence->parent &&
4995             dma_fence_is_signaled(job->base.s_fence->parent)) {
4996                 job_signaled = true;
4997                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
4998                 goto skip_hw_reset;
4999         }
5000
5001 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5002         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5003                 r = amdgpu_device_pre_asic_reset(tmp_adev, &reset_context);
5004                 /*TODO Should we stop ?*/
5005                 if (r) {
5006                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5007                                   r, adev_to_drm(tmp_adev)->unique);
5008                         tmp_adev->asic_reset_res = r;
5009                 }
5010         }
5011
5012         tmp_vram_lost_counter = atomic_read(&((adev)->vram_lost_counter));
5013         /* Actual ASIC resets if needed.*/
5014         /* TODO Implement XGMI hive reset logic for SRIOV */
5015         if (amdgpu_sriov_vf(adev)) {
5016                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5017                 if (r)
5018                         adev->asic_reset_res = r;
5019         } else {
5020                 r = amdgpu_do_asic_reset(device_list_handle, &reset_context);
5021                 if (r && r == -EAGAIN)
5022                         goto retry;
5023         }
5024
5025 skip_hw_reset:
5026
5027         /* Post ASIC reset for all devs .*/
5028         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5029
5030                 /*
5031                  * Sometimes a later bad compute job can block a good gfx job as gfx
5032                  * and compute ring share internal GC HW mutually. We add an additional
5033                  * guilty jobs recheck step to find the real guilty job, it synchronously
5034                  * submits and pends for the first job being signaled. If it gets timeout,
5035                  * we identify it as a real guilty job.
5036                  */
5037                 if (amdgpu_gpu_recovery == 2 &&
5038                         !(tmp_vram_lost_counter < atomic_read(&adev->vram_lost_counter)))
5039                         amdgpu_device_recheck_guilty_jobs(
5040                                 tmp_adev, device_list_handle, &reset_context);
5041
5042                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5043                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5044
5045                         if (!ring || !ring->sched.thread)
5046                                 continue;
5047
5048                         /* No point to resubmit jobs if we didn't HW reset*/
5049                         if (!tmp_adev->asic_reset_res && !job_signaled)
5050                                 drm_sched_resubmit_jobs(&ring->sched);
5051
5052                         drm_sched_start(&ring->sched, !tmp_adev->asic_reset_res);
5053                 }
5054
5055                 if (!amdgpu_device_has_dc_support(tmp_adev) && !job_signaled) {
5056                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5057                 }
5058
5059                 tmp_adev->asic_reset_res = 0;
5060
5061                 if (r) {
5062                         /* bad news, how to tell it to userspace ? */
5063                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5064                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5065                 } else {
5066                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5067                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5068                                 DRM_WARN("smart shift update failed\n");
5069                 }
5070         }
5071
5072 skip_sched_resume:
5073         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5074                 /* unlock kfd: SRIOV would do it separately */
5075                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5076                         amdgpu_amdkfd_post_reset(tmp_adev);
5077
5078                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5079                  * need to bring up kfd here if it's not be initialized before
5080                  */
5081                 if (!adev->kfd.init_complete)
5082                         amdgpu_amdkfd_device_init(adev);
5083
5084                 if (audio_suspended)
5085                         amdgpu_device_resume_display_audio(tmp_adev);
5086                 amdgpu_device_unlock_adev(tmp_adev);
5087         }
5088
5089 skip_recovery:
5090         if (hive) {
5091                 atomic_set(&hive->in_reset, 0);
5092                 mutex_unlock(&hive->hive_lock);
5093                 amdgpu_put_xgmi_hive(hive);
5094         }
5095
5096         if (r && r != -EAGAIN)
5097                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5098         return r;
5099 }
5100
5101 /**
5102  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5103  *
5104  * @adev: amdgpu_device pointer
5105  *
5106  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5107  * and lanes) of the slot the device is in. Handles APUs and
5108  * virtualized environments where PCIE config space may not be available.
5109  */
5110 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5111 {
5112         struct pci_dev *pdev;
5113         enum pci_bus_speed speed_cap, platform_speed_cap;
5114         enum pcie_link_width platform_link_width;
5115
5116         if (amdgpu_pcie_gen_cap)
5117                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5118
5119         if (amdgpu_pcie_lane_cap)
5120                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5121
5122         /* covers APUs as well */
5123         if (pci_is_root_bus(adev->pdev->bus)) {
5124                 if (adev->pm.pcie_gen_mask == 0)
5125                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5126                 if (adev->pm.pcie_mlw_mask == 0)
5127                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5128                 return;
5129         }
5130
5131         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5132                 return;
5133
5134         pcie_bandwidth_available(adev->pdev, NULL,
5135                                  &platform_speed_cap, &platform_link_width);
5136
5137         if (adev->pm.pcie_gen_mask == 0) {
5138                 /* asic caps */
5139                 pdev = adev->pdev;
5140                 speed_cap = pcie_get_speed_cap(pdev);
5141                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5142                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5143                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5144                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5145                 } else {
5146                         if (speed_cap == PCIE_SPEED_32_0GT)
5147                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5148                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5149                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5150                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5151                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5152                         else if (speed_cap == PCIE_SPEED_16_0GT)
5153                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5154                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5155                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5156                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5157                         else if (speed_cap == PCIE_SPEED_8_0GT)
5158                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5159                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5160                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5161                         else if (speed_cap == PCIE_SPEED_5_0GT)
5162                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5163                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5164                         else
5165                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5166                 }
5167                 /* platform caps */
5168                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5169                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5170                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5171                 } else {
5172                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5173                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5174                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5175                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5176                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5177                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5178                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5179                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5180                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5181                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5182                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5183                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5184                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5185                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5186                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5187                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5188                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5189                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5190                         else
5191                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5192
5193                 }
5194         }
5195         if (adev->pm.pcie_mlw_mask == 0) {
5196                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5197                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5198                 } else {
5199                         switch (platform_link_width) {
5200                         case PCIE_LNK_X32:
5201                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5202                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5203                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5204                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5205                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5206                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5207                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5208                                 break;
5209                         case PCIE_LNK_X16:
5210                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5211                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5212                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5213                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5214                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5215                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5216                                 break;
5217                         case PCIE_LNK_X12:
5218                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5219                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5220                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5221                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5222                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5223                                 break;
5224                         case PCIE_LNK_X8:
5225                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5226                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5227                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5228                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5229                                 break;
5230                         case PCIE_LNK_X4:
5231                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5232                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5233                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5234                                 break;
5235                         case PCIE_LNK_X2:
5236                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5237                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5238                                 break;
5239                         case PCIE_LNK_X1:
5240                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5241                                 break;
5242                         default:
5243                                 break;
5244                         }
5245                 }
5246         }
5247 }
5248
5249 int amdgpu_device_baco_enter(struct drm_device *dev)
5250 {
5251         struct amdgpu_device *adev = drm_to_adev(dev);
5252         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5253
5254         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5255                 return -ENOTSUPP;
5256
5257         if (ras && adev->ras_enabled &&
5258             adev->nbio.funcs->enable_doorbell_interrupt)
5259                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5260
5261         return amdgpu_dpm_baco_enter(adev);
5262 }
5263
5264 int amdgpu_device_baco_exit(struct drm_device *dev)
5265 {
5266         struct amdgpu_device *adev = drm_to_adev(dev);
5267         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5268         int ret = 0;
5269
5270         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5271                 return -ENOTSUPP;
5272
5273         ret = amdgpu_dpm_baco_exit(adev);
5274         if (ret)
5275                 return ret;
5276
5277         if (ras && adev->ras_enabled &&
5278             adev->nbio.funcs->enable_doorbell_interrupt)
5279                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5280
5281         return 0;
5282 }
5283
5284 static void amdgpu_cancel_all_tdr(struct amdgpu_device *adev)
5285 {
5286         int i;
5287
5288         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5289                 struct amdgpu_ring *ring = adev->rings[i];
5290
5291                 if (!ring || !ring->sched.thread)
5292                         continue;
5293
5294                 cancel_delayed_work_sync(&ring->sched.work_tdr);
5295         }
5296 }
5297
5298 /**
5299  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5300  * @pdev: PCI device struct
5301  * @state: PCI channel state
5302  *
5303  * Description: Called when a PCI error is detected.
5304  *
5305  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5306  */
5307 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5308 {
5309         struct drm_device *dev = pci_get_drvdata(pdev);
5310         struct amdgpu_device *adev = drm_to_adev(dev);
5311         int i;
5312
5313         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5314
5315         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5316                 DRM_WARN("No support for XGMI hive yet...");
5317                 return PCI_ERS_RESULT_DISCONNECT;
5318         }
5319
5320         switch (state) {
5321         case pci_channel_io_normal:
5322                 return PCI_ERS_RESULT_CAN_RECOVER;
5323         /* Fatal error, prepare for slot reset */
5324         case pci_channel_io_frozen:
5325                 /*
5326                  * Cancel and wait for all TDRs in progress if failing to
5327                  * set  adev->in_gpu_reset in amdgpu_device_lock_adev
5328                  *
5329                  * Locking adev->reset_sem will prevent any external access
5330                  * to GPU during PCI error recovery
5331                  */
5332                 while (!amdgpu_device_lock_adev(adev, NULL))
5333                         amdgpu_cancel_all_tdr(adev);
5334
5335                 /*
5336                  * Block any work scheduling as we do for regular GPU reset
5337                  * for the duration of the recovery
5338                  */
5339                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5340                         struct amdgpu_ring *ring = adev->rings[i];
5341
5342                         if (!ring || !ring->sched.thread)
5343                                 continue;
5344
5345                         drm_sched_stop(&ring->sched, NULL);
5346                 }
5347                 atomic_inc(&adev->gpu_reset_counter);
5348                 return PCI_ERS_RESULT_NEED_RESET;
5349         case pci_channel_io_perm_failure:
5350                 /* Permanent error, prepare for device removal */
5351                 return PCI_ERS_RESULT_DISCONNECT;
5352         }
5353
5354         return PCI_ERS_RESULT_NEED_RESET;
5355 }
5356
5357 /**
5358  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5359  * @pdev: pointer to PCI device
5360  */
5361 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5362 {
5363
5364         DRM_INFO("PCI error: mmio enabled callback!!\n");
5365
5366         /* TODO - dump whatever for debugging purposes */
5367
5368         /* This called only if amdgpu_pci_error_detected returns
5369          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5370          * works, no need to reset slot.
5371          */
5372
5373         return PCI_ERS_RESULT_RECOVERED;
5374 }
5375
5376 /**
5377  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5378  * @pdev: PCI device struct
5379  *
5380  * Description: This routine is called by the pci error recovery
5381  * code after the PCI slot has been reset, just before we
5382  * should resume normal operations.
5383  */
5384 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5385 {
5386         struct drm_device *dev = pci_get_drvdata(pdev);
5387         struct amdgpu_device *adev = drm_to_adev(dev);
5388         int r, i;
5389         struct amdgpu_reset_context reset_context;
5390         u32 memsize;
5391         struct list_head device_list;
5392
5393         DRM_INFO("PCI error: slot reset callback!!\n");
5394
5395         memset(&reset_context, 0, sizeof(reset_context));
5396
5397         INIT_LIST_HEAD(&device_list);
5398         list_add_tail(&adev->reset_list, &device_list);
5399
5400         /* wait for asic to come out of reset */
5401         msleep(500);
5402
5403         /* Restore PCI confspace */
5404         amdgpu_device_load_pci_state(pdev);
5405
5406         /* confirm  ASIC came out of reset */
5407         for (i = 0; i < adev->usec_timeout; i++) {
5408                 memsize = amdgpu_asic_get_config_memsize(adev);
5409
5410                 if (memsize != 0xffffffff)
5411                         break;
5412                 udelay(1);
5413         }
5414         if (memsize == 0xffffffff) {
5415                 r = -ETIME;
5416                 goto out;
5417         }
5418
5419         reset_context.method = AMD_RESET_METHOD_NONE;
5420         reset_context.reset_req_dev = adev;
5421         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5422         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5423
5424         adev->no_hw_access = true;
5425         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5426         adev->no_hw_access = false;
5427         if (r)
5428                 goto out;
5429
5430         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5431
5432 out:
5433         if (!r) {
5434                 if (amdgpu_device_cache_pci_state(adev->pdev))
5435                         pci_restore_state(adev->pdev);
5436
5437                 DRM_INFO("PCIe error recovery succeeded\n");
5438         } else {
5439                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5440                 amdgpu_device_unlock_adev(adev);
5441         }
5442
5443         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5444 }
5445
5446 /**
5447  * amdgpu_pci_resume() - resume normal ops after PCI reset
5448  * @pdev: pointer to PCI device
5449  *
5450  * Called when the error recovery driver tells us that its
5451  * OK to resume normal operation.
5452  */
5453 void amdgpu_pci_resume(struct pci_dev *pdev)
5454 {
5455         struct drm_device *dev = pci_get_drvdata(pdev);
5456         struct amdgpu_device *adev = drm_to_adev(dev);
5457         int i;
5458
5459
5460         DRM_INFO("PCI error: resume callback!!\n");
5461
5462         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5463                 struct amdgpu_ring *ring = adev->rings[i];
5464
5465                 if (!ring || !ring->sched.thread)
5466                         continue;
5467
5468
5469                 drm_sched_resubmit_jobs(&ring->sched);
5470                 drm_sched_start(&ring->sched, true);
5471         }
5472
5473         amdgpu_device_unlock_adev(adev);
5474 }
5475
5476 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5477 {
5478         struct drm_device *dev = pci_get_drvdata(pdev);
5479         struct amdgpu_device *adev = drm_to_adev(dev);
5480         int r;
5481
5482         r = pci_save_state(pdev);
5483         if (!r) {
5484                 kfree(adev->pci_state);
5485
5486                 adev->pci_state = pci_store_saved_state(pdev);
5487
5488                 if (!adev->pci_state) {
5489                         DRM_ERROR("Failed to store PCI saved state");
5490                         return false;
5491                 }
5492         } else {
5493                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5494                 return false;
5495         }
5496
5497         return true;
5498 }
5499
5500 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5501 {
5502         struct drm_device *dev = pci_get_drvdata(pdev);
5503         struct amdgpu_device *adev = drm_to_adev(dev);
5504         int r;
5505
5506         if (!adev->pci_state)
5507                 return false;
5508
5509         r = pci_load_saved_state(pdev, adev->pci_state);
5510
5511         if (!r) {
5512                 pci_restore_state(pdev);
5513         } else {
5514                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5515                 return false;
5516         }
5517
5518         return true;
5519 }
5520
5521 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5522                 struct amdgpu_ring *ring)
5523 {
5524 #ifdef CONFIG_X86_64
5525         if (adev->flags & AMD_IS_APU)
5526                 return;
5527 #endif
5528         if (adev->gmc.xgmi.connected_to_cpu)
5529                 return;
5530
5531         if (ring && ring->funcs->emit_hdp_flush)
5532                 amdgpu_ring_emit_hdp_flush(ring);
5533         else
5534                 amdgpu_asic_flush_hdp(adev, ring);
5535 }
5536
5537 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5538                 struct amdgpu_ring *ring)
5539 {
5540 #ifdef CONFIG_X86_64
5541         if (adev->flags & AMD_IS_APU)
5542                 return;
5543 #endif
5544         if (adev->gmc.xgmi.connected_to_cpu)
5545                 return;
5546
5547         amdgpu_asic_invalidate_hdp(adev, ring);
5548 }