drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/devcoredump.h>
  36 #include <generated/utsrelease.h>
  37 #include <linux/pci-p2pdma.h>
  38
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_fb_helper.h>
  41 #include <drm/drm_probe_helper.h>
  42 #include <drm/amdgpu_drm.h>
  43 #include <linux/vgaarb.h>
  44 #include <linux/vga_switcheroo.h>
  45 #include <linux/efi.h>
  46 #include "amdgpu.h"
  47 #include "amdgpu_trace.h"
  48 #include "amdgpu_i2c.h"
  49 #include "atom.h"
  50 #include "amdgpu_atombios.h"
  51 #include "amdgpu_atomfirmware.h"
  52 #include "amd_pcie.h"
  53 #ifdef CONFIG_DRM_AMDGPU_SI
  54 #include "si.h"
  55 #endif
  56 #ifdef CONFIG_DRM_AMDGPU_CIK
  57 #include "cik.h"
  58 #endif
  59 #include "vi.h"
  60 #include "soc15.h"
  61 #include "nv.h"
  62 #include "bif/bif_4_1_d.h"
  63 #include <linux/firmware.h>
  64 #include "amdgpu_vf_error.h"
  65
  66 #include "amdgpu_amdkfd.h"
  67 #include "amdgpu_pm.h"
  68
  69 #include "amdgpu_xgmi.h"
  70 #include "amdgpu_ras.h"
  71 #include "amdgpu_pmu.h"
  72 #include "amdgpu_fru_eeprom.h"
  73 #include "amdgpu_reset.h"
  74
  75 #include <linux/suspend.h>
  76 #include <drm/task_barrier.h>
  77 #include <linux/pm_runtime.h>
  78
  79 #include <drm/drm_drv.h>
  80
  81 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  82 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  83 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  84 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  85 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  86 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  87 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  88
  89 #define AMDGPU_RESUME_MS                2000
  90 #define AMDGPU_MAX_RETRY_LIMIT          2
  91 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  92
  93 const char *amdgpu_asic_name[] = {
  94         "TAHITI",
  95         "PITCAIRN",
  96         "VERDE",
  97         "OLAND",
  98         "HAINAN",
  99         "BONAIRE",
 100         "KAVERI",
 101         "KABINI",
 102         "HAWAII",
 103         "MULLINS",
 104         "TOPAZ",
 105         "TONGA",
 106         "FIJI",
 107         "CARRIZO",
 108         "STONEY",
 109         "POLARIS10",
 110         "POLARIS11",
 111         "POLARIS12",
 112         "VEGAM",
 113         "VEGA10",
 114         "VEGA12",
 115         "VEGA20",
 116         "RAVEN",
 117         "ARCTURUS",
 118         "RENOIR",
 119         "ALDEBARAN",
 120         "NAVI10",
 121         "CYAN_SKILLFISH",
 122         "NAVI14",
 123         "NAVI12",
 124         "SIENNA_CICHLID",
 125         "NAVY_FLOUNDER",
 126         "VANGOGH",
 127         "DIMGREY_CAVEFISH",
 128         "BEIGE_GOBY",
 129         "YELLOW_CARP",
 130         "IP DISCOVERY",
 131         "LAST",
 132 };
 133
 134 /**
 135  * DOC: pcie_replay_count
 136  *
 137  * The amdgpu driver provides a sysfs API for reporting the total number
 138  * of PCIe replays (NAKs)
 139  * The file pcie_replay_count is used for this and returns the total
 140  * number of replays as a sum of the NAKs generated and NAKs received
 141  */
 142
 143 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 144                 struct device_attribute *attr, char *buf)
 145 {
 146         struct drm_device *ddev = dev_get_drvdata(dev);
 147         struct amdgpu_device *adev = drm_to_adev(ddev);
 148         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 149
 150         return sysfs_emit(buf, "%llu\n", cnt);
 151 }
 152
 153 static DEVICE_ATTR(pcie_replay_count, S_IRUGO,
 154                 amdgpu_device_get_pcie_replay_count, NULL);
 155
 156 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 157
 158 /**
 159  * DOC: product_name
 160  *
 161  * The amdgpu driver provides a sysfs API for reporting the product name
 162  * for the device
 163  * The file serial_number is used for this and returns the product name
 164  * as returned from the FRU.
 165  * NOTE: This is only available for certain server cards
 166  */
 167
 168 static ssize_t amdgpu_device_get_product_name(struct device *dev,
 169                 struct device_attribute *attr, char *buf)
 170 {
 171         struct drm_device *ddev = dev_get_drvdata(dev);
 172         struct amdgpu_device *adev = drm_to_adev(ddev);
 173
 174         return sysfs_emit(buf, "%s\n", adev->product_name);
 175 }
 176
 177 static DEVICE_ATTR(product_name, S_IRUGO,
 178                 amdgpu_device_get_product_name, NULL);
 179
 180 /**
 181  * DOC: product_number
 182  *
 183  * The amdgpu driver provides a sysfs API for reporting the part number
 184  * for the device
 185  * The file serial_number is used for this and returns the part number
 186  * as returned from the FRU.
 187  * NOTE: This is only available for certain server cards
 188  */
 189
 190 static ssize_t amdgpu_device_get_product_number(struct device *dev,
 191                 struct device_attribute *attr, char *buf)
 192 {
 193         struct drm_device *ddev = dev_get_drvdata(dev);
 194         struct amdgpu_device *adev = drm_to_adev(ddev);
 195
 196         return sysfs_emit(buf, "%s\n", adev->product_number);
 197 }
 198
 199 static DEVICE_ATTR(product_number, S_IRUGO,
 200                 amdgpu_device_get_product_number, NULL);
 201
 202 /**
 203  * DOC: serial_number
 204  *
 205  * The amdgpu driver provides a sysfs API for reporting the serial number
 206  * for the device
 207  * The file serial_number is used for this and returns the serial number
 208  * as returned from the FRU.
 209  * NOTE: This is only available for certain server cards
 210  */
 211
 212 static ssize_t amdgpu_device_get_serial_number(struct device *dev,
 213                 struct device_attribute *attr, char *buf)
 214 {
 215         struct drm_device *ddev = dev_get_drvdata(dev);
 216         struct amdgpu_device *adev = drm_to_adev(ddev);
 217
 218         return sysfs_emit(buf, "%s\n", adev->serial);
 219 }
 220
 221 static DEVICE_ATTR(serial_number, S_IRUGO,
 222                 amdgpu_device_get_serial_number, NULL);
 223
 224 /**
 225  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 226  *
 227  * @dev: drm_device pointer
 228  *
 229  * Returns true if the device is a dGPU with ATPX power control,
 230  * otherwise return false.
 231  */
 232 bool amdgpu_device_supports_px(struct drm_device *dev)
 233 {
 234         struct amdgpu_device *adev = drm_to_adev(dev);
 235
 236         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 237                 return true;
 238         return false;
 239 }
 240
 241 /**
 242  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 243  *
 244  * @dev: drm_device pointer
 245  *
 246  * Returns true if the device is a dGPU with ACPI power control,
 247  * otherwise return false.
 248  */
 249 bool amdgpu_device_supports_boco(struct drm_device *dev)
 250 {
 251         struct amdgpu_device *adev = drm_to_adev(dev);
 252
 253         if (adev->has_pr3 ||
 254             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 255                 return true;
 256         return false;
 257 }
 258
 259 /**
 260  * amdgpu_device_supports_baco - Does the device support BACO
 261  *
 262  * @dev: drm_device pointer
 263  *
 264  * Returns true if the device supporte BACO,
 265  * otherwise return false.
 266  */
 267 bool amdgpu_device_supports_baco(struct drm_device *dev)
 268 {
 269         struct amdgpu_device *adev = drm_to_adev(dev);
 270
 271         return amdgpu_asic_supports_baco(adev);
 272 }
 273
 274 /**
 275  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 276  * smart shift support
 277  *
 278  * @dev: drm_device pointer
 279  *
 280  * Returns true if the device is a dGPU with Smart Shift support,
 281  * otherwise returns false.
 282  */
 283 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 284 {
 285         return (amdgpu_device_supports_boco(dev) &&
 286                 amdgpu_acpi_is_power_shift_control_supported());
 287 }
 288
 289 /*
 290  * VRAM access helper functions
 291  */
 292
 293 /**
 294  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 295  *
 296  * @adev: amdgpu_device pointer
 297  * @pos: offset of the buffer in vram
 298  * @buf: virtual address of the buffer in system memory
 299  * @size: read/write size, sizeof(@buf) must > @size
 300  * @write: true - write to vram, otherwise - read from vram
 301  */
 302 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 303                              void *buf, size_t size, bool write)
 304 {
 305         unsigned long flags;
 306         uint32_t hi = ~0, tmp = 0;
 307         uint32_t *data = buf;
 308         uint64_t last;
 309         int idx;
 310
 311         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 312                 return;
 313
 314         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 315
 316         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 317         for (last = pos + size; pos < last; pos += 4) {
 318                 tmp = pos >> 31;
 319
 320                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 321                 if (tmp != hi) {
 322                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 323                         hi = tmp;
 324                 }
 325                 if (write)
 326                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 327                 else
 328                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 329         }
 330
 331         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 332         drm_dev_exit(idx);
 333 }
 334
 335 /**
 336  * amdgpu_device_aper_access - access vram by vram aperature
 337  *
 338  * @adev: amdgpu_device pointer
 339  * @pos: offset of the buffer in vram
 340  * @buf: virtual address of the buffer in system memory
 341  * @size: read/write size, sizeof(@buf) must > @size
 342  * @write: true - write to vram, otherwise - read from vram
 343  *
 344  * The return value means how many bytes have been transferred.
 345  */
 346 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 347                                  void *buf, size_t size, bool write)
 348 {
 349 #ifdef CONFIG_64BIT
 350         void __iomem *addr;
 351         size_t count = 0;
 352         uint64_t last;
 353
 354         if (!adev->mman.aper_base_kaddr)
 355                 return 0;
 356
 357         last = min(pos + size, adev->gmc.visible_vram_size);
 358         if (last > pos) {
 359                 addr = adev->mman.aper_base_kaddr + pos;
 360                 count = last - pos;
 361
 362                 if (write) {
 363                         memcpy_toio(addr, buf, count);
 364                         mb();
 365                         amdgpu_device_flush_hdp(adev, NULL);
 366                 } else {
 367                         amdgpu_device_invalidate_hdp(adev, NULL);
 368                         mb();
 369                         memcpy_fromio(buf, addr, count);
 370                 }
 371
 372         }
 373
 374         return count;
 375 #else
 376         return 0;
 377 #endif
 378 }
 379
 380 /**
 381  * amdgpu_device_vram_access - read/write a buffer in vram
 382  *
 383  * @adev: amdgpu_device pointer
 384  * @pos: offset of the buffer in vram
 385  * @buf: virtual address of the buffer in system memory
 386  * @size: read/write size, sizeof(@buf) must > @size
 387  * @write: true - write to vram, otherwise - read from vram
 388  */
 389 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 390                                void *buf, size_t size, bool write)
 391 {
 392         size_t count;
 393
 394         /* try to using vram apreature to access vram first */
 395         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 396         size -= count;
 397         if (size) {
 398                 /* using MM to access rest vram */
 399                 pos += count;
 400                 buf += count;
 401                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 402         }
 403 }
 404
 405 /*
 406  * register access helper functions.
 407  */
 408
 409 /* Check if hw access should be skipped because of hotplug or device error */
 410 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 411 {
 412         if (adev->no_hw_access)
 413                 return true;
 414
 415 #ifdef CONFIG_LOCKDEP
 416         /*
 417          * This is a bit complicated to understand, so worth a comment. What we assert
 418          * here is that the GPU reset is not running on another thread in parallel.
 419          *
 420          * For this we trylock the read side of the reset semaphore, if that succeeds
 421          * we know that the reset is not running in paralell.
 422          *
 423          * If the trylock fails we assert that we are either already holding the read
 424          * side of the lock or are the reset thread itself and hold the write side of
 425          * the lock.
 426          */
 427         if (in_task()) {
 428                 if (down_read_trylock(&adev->reset_domain->sem))
 429                         up_read(&adev->reset_domain->sem);
 430                 else
 431                         lockdep_assert_held(&adev->reset_domain->sem);
 432         }
 433 #endif
 434         return false;
 435 }
 436
 437 /**
 438  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 439  *
 440  * @adev: amdgpu_device pointer
 441  * @reg: dword aligned register offset
 442  * @acc_flags: access flags which require special behavior
 443  *
 444  * Returns the 32 bit value from the offset specified.
 445  */
 446 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 447                             uint32_t reg, uint32_t acc_flags)
 448 {
 449         uint32_t ret;
 450
 451         if (amdgpu_device_skip_hw_access(adev))
 452                 return 0;
 453
 454         if ((reg * 4) < adev->rmmio_size) {
 455                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 456                     amdgpu_sriov_runtime(adev) &&
 457                     down_read_trylock(&adev->reset_domain->sem)) {
 458                         ret = amdgpu_kiq_rreg(adev, reg);
 459                         up_read(&adev->reset_domain->sem);
 460                 } else {
 461                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 462                 }
 463         } else {
 464                 ret = adev->pcie_rreg(adev, reg * 4);
 465         }
 466
 467         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 468
 469         return ret;
 470 }
 471
 472 /*
 473  * MMIO register read with bytes helper functions
 474  * @offset:bytes offset from MMIO start
 475  *
 476 */
 477
 478 /**
 479  * amdgpu_mm_rreg8 - read a memory mapped IO register
 480  *
 481  * @adev: amdgpu_device pointer
 482  * @offset: byte aligned register offset
 483  *
 484  * Returns the 8 bit value from the offset specified.
 485  */
 486 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 487 {
 488         if (amdgpu_device_skip_hw_access(adev))
 489                 return 0;
 490
 491         if (offset < adev->rmmio_size)
 492                 return (readb(adev->rmmio + offset));
 493         BUG();
 494 }
 495
 496 /*
 497  * MMIO register write with bytes helper functions
 498  * @offset:bytes offset from MMIO start
 499  * @value: the value want to be written to the register
 500  *
 501 */
 502 /**
 503  * amdgpu_mm_wreg8 - read a memory mapped IO register
 504  *
 505  * @adev: amdgpu_device pointer
 506  * @offset: byte aligned register offset
 507  * @value: 8 bit value to write
 508  *
 509  * Writes the value specified to the offset specified.
 510  */
 511 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 512 {
 513         if (amdgpu_device_skip_hw_access(adev))
 514                 return;
 515
 516         if (offset < adev->rmmio_size)
 517                 writeb(value, adev->rmmio + offset);
 518         else
 519                 BUG();
 520 }
 521
 522 /**
 523  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 524  *
 525  * @adev: amdgpu_device pointer
 526  * @reg: dword aligned register offset
 527  * @v: 32 bit value to write to the register
 528  * @acc_flags: access flags which require special behavior
 529  *
 530  * Writes the value specified to the offset specified.
 531  */
 532 void amdgpu_device_wreg(struct amdgpu_device *adev,
 533                         uint32_t reg, uint32_t v,
 534                         uint32_t acc_flags)
 535 {
 536         if (amdgpu_device_skip_hw_access(adev))
 537                 return;
 538
 539         if ((reg * 4) < adev->rmmio_size) {
 540                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 541                     amdgpu_sriov_runtime(adev) &&
 542                     down_read_trylock(&adev->reset_domain->sem)) {
 543                         amdgpu_kiq_wreg(adev, reg, v);
 544                         up_read(&adev->reset_domain->sem);
 545                 } else {
 546                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 547                 }
 548         } else {
 549                 adev->pcie_wreg(adev, reg * 4, v);
 550         }
 551
 552         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 553 }
 554
 555 /**
 556  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 557  *
 558  * @adev: amdgpu_device pointer
 559  * @reg: mmio/rlc register
 560  * @v: value to write
 561  *
 562  * this function is invoked only for the debugfs register access
 563  */
 564 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 565                              uint32_t reg, uint32_t v)
 566 {
 567         if (amdgpu_device_skip_hw_access(adev))
 568                 return;
 569
 570         if (amdgpu_sriov_fullaccess(adev) &&
 571             adev->gfx.rlc.funcs &&
 572             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 573                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 574                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0);
 575         } else if ((reg * 4) >= adev->rmmio_size) {
 576                 adev->pcie_wreg(adev, reg * 4, v);
 577         } else {
 578                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 579         }
 580 }
 581
 582 /**
 583  * amdgpu_mm_rdoorbell - read a doorbell dword
 584  *
 585  * @adev: amdgpu_device pointer
 586  * @index: doorbell index
 587  *
 588  * Returns the value in the doorbell aperture at the
 589  * requested doorbell index (CIK).
 590  */
 591 u32 amdgpu_mm_rdoorbell(struct amdgpu_device *adev, u32 index)
 592 {
 593         if (amdgpu_device_skip_hw_access(adev))
 594                 return 0;
 595
 596         if (index < adev->doorbell.num_doorbells) {
 597                 return readl(adev->doorbell.ptr + index);
 598         } else {
 599                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 600                 return 0;
 601         }
 602 }
 603
 604 /**
 605  * amdgpu_mm_wdoorbell - write a doorbell dword
 606  *
 607  * @adev: amdgpu_device pointer
 608  * @index: doorbell index
 609  * @v: value to write
 610  *
 611  * Writes @v to the doorbell aperture at the
 612  * requested doorbell index (CIK).
 613  */
 614 void amdgpu_mm_wdoorbell(struct amdgpu_device *adev, u32 index, u32 v)
 615 {
 616         if (amdgpu_device_skip_hw_access(adev))
 617                 return;
 618
 619         if (index < adev->doorbell.num_doorbells) {
 620                 writel(v, adev->doorbell.ptr + index);
 621         } else {
 622                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 623         }
 624 }
 625
 626 /**
 627  * amdgpu_mm_rdoorbell64 - read a doorbell Qword
 628  *
 629  * @adev: amdgpu_device pointer
 630  * @index: doorbell index
 631  *
 632  * Returns the value in the doorbell aperture at the
 633  * requested doorbell index (VEGA10+).
 634  */
 635 u64 amdgpu_mm_rdoorbell64(struct amdgpu_device *adev, u32 index)
 636 {
 637         if (amdgpu_device_skip_hw_access(adev))
 638                 return 0;
 639
 640         if (index < adev->doorbell.num_doorbells) {
 641                 return atomic64_read((atomic64_t *)(adev->doorbell.ptr + index));
 642         } else {
 643                 DRM_ERROR("reading beyond doorbell aperture: 0x%08x!\n", index);
 644                 return 0;
 645         }
 646 }
 647
 648 /**
 649  * amdgpu_mm_wdoorbell64 - write a doorbell Qword
 650  *
 651  * @adev: amdgpu_device pointer
 652  * @index: doorbell index
 653  * @v: value to write
 654  *
 655  * Writes @v to the doorbell aperture at the
 656  * requested doorbell index (VEGA10+).
 657  */
 658 void amdgpu_mm_wdoorbell64(struct amdgpu_device *adev, u32 index, u64 v)
 659 {
 660         if (amdgpu_device_skip_hw_access(adev))
 661                 return;
 662
 663         if (index < adev->doorbell.num_doorbells) {
 664                 atomic64_set((atomic64_t *)(adev->doorbell.ptr + index), v);
 665         } else {
 666                 DRM_ERROR("writing beyond doorbell aperture: 0x%08x!\n", index);
 667         }
 668 }
 669
 670 /**
 671  * amdgpu_device_indirect_rreg - read an indirect register
 672  *
 673  * @adev: amdgpu_device pointer
 674  * @pcie_index: mmio register offset
 675  * @pcie_data: mmio register offset
 676  * @reg_addr: indirect register address to read from
 677  *
 678  * Returns the value of indirect register @reg_addr
 679  */
 680 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 681                                 u32 pcie_index, u32 pcie_data,
 682                                 u32 reg_addr)
 683 {
 684         unsigned long flags;
 685         u32 r;
 686         void __iomem *pcie_index_offset;
 687         void __iomem *pcie_data_offset;
 688
 689         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 690         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 691         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 692
 693         writel(reg_addr, pcie_index_offset);
 694         readl(pcie_index_offset);
 695         r = readl(pcie_data_offset);
 696         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 697
 698         return r;
 699 }
 700
 701 /**
 702  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 703  *
 704  * @adev: amdgpu_device pointer
 705  * @pcie_index: mmio register offset
 706  * @pcie_data: mmio register offset
 707  * @reg_addr: indirect register address to read from
 708  *
 709  * Returns the value of indirect register @reg_addr
 710  */
 711 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 712                                   u32 pcie_index, u32 pcie_data,
 713                                   u32 reg_addr)
 714 {
 715         unsigned long flags;
 716         u64 r;
 717         void __iomem *pcie_index_offset;
 718         void __iomem *pcie_data_offset;
 719
 720         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 721         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 722         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 723
 724         /* read low 32 bits */
 725         writel(reg_addr, pcie_index_offset);
 726         readl(pcie_index_offset);
 727         r = readl(pcie_data_offset);
 728         /* read high 32 bits */
 729         writel(reg_addr + 4, pcie_index_offset);
 730         readl(pcie_index_offset);
 731         r |= ((u64)readl(pcie_data_offset) << 32);
 732         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 733
 734         return r;
 735 }
 736
 737 /**
 738  * amdgpu_device_indirect_wreg - write an indirect register address
 739  *
 740  * @adev: amdgpu_device pointer
 741  * @pcie_index: mmio register offset
 742  * @pcie_data: mmio register offset
 743  * @reg_addr: indirect register offset
 744  * @reg_data: indirect register data
 745  *
 746  */
 747 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 748                                  u32 pcie_index, u32 pcie_data,
 749                                  u32 reg_addr, u32 reg_data)
 750 {
 751         unsigned long flags;
 752         void __iomem *pcie_index_offset;
 753         void __iomem *pcie_data_offset;
 754
 755         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 756         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 757         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 758
 759         writel(reg_addr, pcie_index_offset);
 760         readl(pcie_index_offset);
 761         writel(reg_data, pcie_data_offset);
 762         readl(pcie_data_offset);
 763         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 764 }
 765
 766 /**
 767  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 768  *
 769  * @adev: amdgpu_device pointer
 770  * @pcie_index: mmio register offset
 771  * @pcie_data: mmio register offset
 772  * @reg_addr: indirect register offset
 773  * @reg_data: indirect register data
 774  *
 775  */
 776 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 777                                    u32 pcie_index, u32 pcie_data,
 778                                    u32 reg_addr, u64 reg_data)
 779 {
 780         unsigned long flags;
 781         void __iomem *pcie_index_offset;
 782         void __iomem *pcie_data_offset;
 783
 784         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 785         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 786         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 787
 788         /* write low 32 bits */
 789         writel(reg_addr, pcie_index_offset);
 790         readl(pcie_index_offset);
 791         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 792         readl(pcie_data_offset);
 793         /* write high 32 bits */
 794         writel(reg_addr + 4, pcie_index_offset);
 795         readl(pcie_index_offset);
 796         writel((u32)(reg_data >> 32), pcie_data_offset);
 797         readl(pcie_data_offset);
 798         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 799 }
 800
 801 /**
 802  * amdgpu_invalid_rreg - dummy reg read function
 803  *
 804  * @adev: amdgpu_device pointer
 805  * @reg: offset of register
 806  *
 807  * Dummy register read function.  Used for register blocks
 808  * that certain asics don't have (all asics).
 809  * Returns the value in the register.
 810  */
 811 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
 812 {
 813         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
 814         BUG();
 815         return 0;
 816 }
 817
 818 /**
 819  * amdgpu_invalid_wreg - dummy reg write function
 820  *
 821  * @adev: amdgpu_device pointer
 822  * @reg: offset of register
 823  * @v: value to write to the register
 824  *
 825  * Dummy register read function.  Used for register blocks
 826  * that certain asics don't have (all asics).
 827  */
 828 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
 829 {
 830         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
 831                   reg, v);
 832         BUG();
 833 }
 834
 835 /**
 836  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
 837  *
 838  * @adev: amdgpu_device pointer
 839  * @reg: offset of register
 840  *
 841  * Dummy register read function.  Used for register blocks
 842  * that certain asics don't have (all asics).
 843  * Returns the value in the register.
 844  */
 845 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
 846 {
 847         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
 848         BUG();
 849         return 0;
 850 }
 851
 852 /**
 853  * amdgpu_invalid_wreg64 - dummy reg write function
 854  *
 855  * @adev: amdgpu_device pointer
 856  * @reg: offset of register
 857  * @v: value to write to the register
 858  *
 859  * Dummy register read function.  Used for register blocks
 860  * that certain asics don't have (all asics).
 861  */
 862 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
 863 {
 864         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
 865                   reg, v);
 866         BUG();
 867 }
 868
 869 /**
 870  * amdgpu_block_invalid_rreg - dummy reg read function
 871  *
 872  * @adev: amdgpu_device pointer
 873  * @block: offset of instance
 874  * @reg: offset of register
 875  *
 876  * Dummy register read function.  Used for register blocks
 877  * that certain asics don't have (all asics).
 878  * Returns the value in the register.
 879  */
 880 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
 881                                           uint32_t block, uint32_t reg)
 882 {
 883         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
 884                   reg, block);
 885         BUG();
 886         return 0;
 887 }
 888
 889 /**
 890  * amdgpu_block_invalid_wreg - dummy reg write function
 891  *
 892  * @adev: amdgpu_device pointer
 893  * @block: offset of instance
 894  * @reg: offset of register
 895  * @v: value to write to the register
 896  *
 897  * Dummy register read function.  Used for register blocks
 898  * that certain asics don't have (all asics).
 899  */
 900 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
 901                                       uint32_t block,
 902                                       uint32_t reg, uint32_t v)
 903 {
 904         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
 905                   reg, block, v);
 906         BUG();
 907 }
 908
 909 /**
 910  * amdgpu_device_asic_init - Wrapper for atom asic_init
 911  *
 912  * @adev: amdgpu_device pointer
 913  *
 914  * Does any asic specific work and then calls atom asic init.
 915  */
 916 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
 917 {
 918         amdgpu_asic_pre_asic_init(adev);
 919
 920         if (adev->ip_versions[GC_HWIP][0] >= IP_VERSION(11, 0, 0))
 921                 return amdgpu_atomfirmware_asic_init(adev, true);
 922         else
 923                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
 924 }
 925
 926 /**
 927  * amdgpu_device_vram_scratch_init - allocate the VRAM scratch page
 928  *
 929  * @adev: amdgpu_device pointer
 930  *
 931  * Allocates a scratch page of VRAM for use by various things in the
 932  * driver.
 933  */
 934 static int amdgpu_device_vram_scratch_init(struct amdgpu_device *adev)
 935 {
 936         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE,
 937                                        PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM,
 938                                        &adev->vram_scratch.robj,
 939                                        &adev->vram_scratch.gpu_addr,
 940                                        (void **)&adev->vram_scratch.ptr);
 941 }
 942
 943 /**
 944  * amdgpu_device_vram_scratch_fini - Free the VRAM scratch page
 945  *
 946  * @adev: amdgpu_device pointer
 947  *
 948  * Frees the VRAM scratch page.
 949  */
 950 static void amdgpu_device_vram_scratch_fini(struct amdgpu_device *adev)
 951 {
 952         amdgpu_bo_free_kernel(&adev->vram_scratch.robj, NULL, NULL);
 953 }
 954
 955 /**
 956  * amdgpu_device_program_register_sequence - program an array of registers.
 957  *
 958  * @adev: amdgpu_device pointer
 959  * @registers: pointer to the register array
 960  * @array_size: size of the register array
 961  *
 962  * Programs an array or registers with and and or masks.
 963  * This is a helper for setting golden registers.
 964  */
 965 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
 966                                              const u32 *registers,
 967                                              const u32 array_size)
 968 {
 969         u32 tmp, reg, and_mask, or_mask;
 970         int i;
 971
 972         if (array_size % 3)
 973                 return;
 974
 975         for (i = 0; i < array_size; i +=3) {
 976                 reg = registers[i + 0];
 977                 and_mask = registers[i + 1];
 978                 or_mask = registers[i + 2];
 979
 980                 if (and_mask == 0xffffffff) {
 981                         tmp = or_mask;
 982                 } else {
 983                         tmp = RREG32(reg);
 984                         tmp &= ~and_mask;
 985                         if (adev->family >= AMDGPU_FAMILY_AI)
 986                                 tmp |= (or_mask & and_mask);
 987                         else
 988                                 tmp |= or_mask;
 989                 }
 990                 WREG32(reg, tmp);
 991         }
 992 }
 993
 994 /**
 995  * amdgpu_device_pci_config_reset - reset the GPU
 996  *
 997  * @adev: amdgpu_device pointer
 998  *
 999  * Resets the GPU using the pci config reset sequence.
1000  * Only applicable to asics prior to vega10.
1001  */
1002 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1003 {
1004         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1005 }
1006
1007 /**
1008  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1009  *
1010  * @adev: amdgpu_device pointer
1011  *
1012  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1013  */
1014 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1015 {
1016         return pci_reset_function(adev->pdev);
1017 }
1018
1019 /*
1020  * GPU doorbell aperture helpers function.
1021  */
1022 /**
1023  * amdgpu_device_doorbell_init - Init doorbell driver information.
1024  *
1025  * @adev: amdgpu_device pointer
1026  *
1027  * Init doorbell driver information (CIK)
1028  * Returns 0 on success, error on failure.
1029  */
1030 static int amdgpu_device_doorbell_init(struct amdgpu_device *adev)
1031 {
1032
1033         /* No doorbell on SI hardware generation */
1034         if (adev->asic_type < CHIP_BONAIRE) {
1035                 adev->doorbell.base = 0;
1036                 adev->doorbell.size = 0;
1037                 adev->doorbell.num_doorbells = 0;
1038                 adev->doorbell.ptr = NULL;
1039                 return 0;
1040         }
1041
1042         if (pci_resource_flags(adev->pdev, 2) & IORESOURCE_UNSET)
1043                 return -EINVAL;
1044
1045         amdgpu_asic_init_doorbell_index(adev);
1046
1047         /* doorbell bar mapping */
1048         adev->doorbell.base = pci_resource_start(adev->pdev, 2);
1049         adev->doorbell.size = pci_resource_len(adev->pdev, 2);
1050
1051         if (adev->enable_mes) {
1052                 adev->doorbell.num_doorbells =
1053                         adev->doorbell.size / sizeof(u32);
1054         } else {
1055                 adev->doorbell.num_doorbells =
1056                         min_t(u32, adev->doorbell.size / sizeof(u32),
1057                               adev->doorbell_index.max_assignment+1);
1058                 if (adev->doorbell.num_doorbells == 0)
1059                         return -EINVAL;
1060
1061                 /* For Vega, reserve and map two pages on doorbell BAR since SDMA
1062                  * paging queue doorbell use the second page. The
1063                  * AMDGPU_DOORBELL64_MAX_ASSIGNMENT definition assumes all the
1064                  * doorbells are in the first page. So with paging queue enabled,
1065                  * the max num_doorbells should + 1 page (0x400 in dword)
1066                  */
1067                 if (adev->asic_type >= CHIP_VEGA10)
1068                         adev->doorbell.num_doorbells += 0x400;
1069         }
1070
1071         adev->doorbell.ptr = ioremap(adev->doorbell.base,
1072                                      adev->doorbell.num_doorbells *
1073                                      sizeof(u32));
1074         if (adev->doorbell.ptr == NULL)
1075                 return -ENOMEM;
1076
1077         return 0;
1078 }
1079
1080 /**
1081  * amdgpu_device_doorbell_fini - Tear down doorbell driver information.
1082  *
1083  * @adev: amdgpu_device pointer
1084  *
1085  * Tear down doorbell driver information (CIK)
1086  */
1087 static void amdgpu_device_doorbell_fini(struct amdgpu_device *adev)
1088 {
1089         iounmap(adev->doorbell.ptr);
1090         adev->doorbell.ptr = NULL;
1091 }
1092
1093
1094
1095 /*
1096  * amdgpu_device_wb_*()
1097  * Writeback is the method by which the GPU updates special pages in memory
1098  * with the status of certain GPU events (fences, ring pointers,etc.).
1099  */
1100
1101 /**
1102  * amdgpu_device_wb_fini - Disable Writeback and free memory
1103  *
1104  * @adev: amdgpu_device pointer
1105  *
1106  * Disables Writeback and frees the Writeback memory (all asics).
1107  * Used at driver shutdown.
1108  */
1109 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1110 {
1111         if (adev->wb.wb_obj) {
1112                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1113                                       &adev->wb.gpu_addr,
1114                                       (void **)&adev->wb.wb);
1115                 adev->wb.wb_obj = NULL;
1116         }
1117 }
1118
1119 /**
1120  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1121  *
1122  * @adev: amdgpu_device pointer
1123  *
1124  * Initializes writeback and allocates writeback memory (all asics).
1125  * Used at driver startup.
1126  * Returns 0 on success or an -error on failure.
1127  */
1128 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1129 {
1130         int r;
1131
1132         if (adev->wb.wb_obj == NULL) {
1133                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1134                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1135                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1136                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1137                                             (void **)&adev->wb.wb);
1138                 if (r) {
1139                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1140                         return r;
1141                 }
1142
1143                 adev->wb.num_wb = AMDGPU_MAX_WB;
1144                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1145
1146                 /* clear wb memory */
1147                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1148         }
1149
1150         return 0;
1151 }
1152
1153 /**
1154  * amdgpu_device_wb_get - Allocate a wb entry
1155  *
1156  * @adev: amdgpu_device pointer
1157  * @wb: wb index
1158  *
1159  * Allocate a wb slot for use by the driver (all asics).
1160  * Returns 0 on success or -EINVAL on failure.
1161  */
1162 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1163 {
1164         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1165
1166         if (offset < adev->wb.num_wb) {
1167                 __set_bit(offset, adev->wb.used);
1168                 *wb = offset << 3; /* convert to dw offset */
1169                 return 0;
1170         } else {
1171                 return -EINVAL;
1172         }
1173 }
1174
1175 /**
1176  * amdgpu_device_wb_free - Free a wb entry
1177  *
1178  * @adev: amdgpu_device pointer
1179  * @wb: wb index
1180  *
1181  * Free a wb slot allocated for use by the driver (all asics)
1182  */
1183 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1184 {
1185         wb >>= 3;
1186         if (wb < adev->wb.num_wb)
1187                 __clear_bit(wb, adev->wb.used);
1188 }
1189
1190 /**
1191  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1192  *
1193  * @adev: amdgpu_device pointer
1194  *
1195  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1196  * to fail, but if any of the BARs is not accessible after the size we abort
1197  * driver loading by returning -ENODEV.
1198  */
1199 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1200 {
1201         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1202         struct pci_bus *root;
1203         struct resource *res;
1204         unsigned i;
1205         u16 cmd;
1206         int r;
1207
1208         /* Bypass for VF */
1209         if (amdgpu_sriov_vf(adev))
1210                 return 0;
1211
1212         /* skip if the bios has already enabled large BAR */
1213         if (adev->gmc.real_vram_size &&
1214             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1215                 return 0;
1216
1217         /* Check if the root BUS has 64bit memory resources */
1218         root = adev->pdev->bus;
1219         while (root->parent)
1220                 root = root->parent;
1221
1222         pci_bus_for_each_resource(root, res, i) {
1223                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1224                     res->start > 0x100000000ull)
1225                         break;
1226         }
1227
1228         /* Trying to resize is pointless without a root hub window above 4GB */
1229         if (!res)
1230                 return 0;
1231
1232         /* Limit the BAR size to what is available */
1233         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1234                         rbar_size);
1235
1236         /* Disable memory decoding while we change the BAR addresses and size */
1237         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1238         pci_write_config_word(adev->pdev, PCI_COMMAND,
1239                               cmd & ~PCI_COMMAND_MEMORY);
1240
1241         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1242         amdgpu_device_doorbell_fini(adev);
1243         if (adev->asic_type >= CHIP_BONAIRE)
1244                 pci_release_resource(adev->pdev, 2);
1245
1246         pci_release_resource(adev->pdev, 0);
1247
1248         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1249         if (r == -ENOSPC)
1250                 DRM_INFO("Not enough PCI address space for a large BAR.");
1251         else if (r && r != -ENOTSUPP)
1252                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1253
1254         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1255
1256         /* When the doorbell or fb BAR isn't available we have no chance of
1257          * using the device.
1258          */
1259         r = amdgpu_device_doorbell_init(adev);
1260         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1261                 return -ENODEV;
1262
1263         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1264
1265         return 0;
1266 }
1267
1268 /*
1269  * GPU helpers function.
1270  */
1271 /**
1272  * amdgpu_device_need_post - check if the hw need post or not
1273  *
1274  * @adev: amdgpu_device pointer
1275  *
1276  * Check if the asic has been initialized (all asics) at driver startup
1277  * or post is needed if  hw reset is performed.
1278  * Returns true if need or false if not.
1279  */
1280 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1281 {
1282         uint32_t reg;
1283
1284         if (amdgpu_sriov_vf(adev))
1285                 return false;
1286
1287         if (amdgpu_passthrough(adev)) {
1288                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1289                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1290                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1291                  * vpost executed for smc version below 22.15
1292                  */
1293                 if (adev->asic_type == CHIP_FIJI) {
1294                         int err;
1295                         uint32_t fw_ver;
1296                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1297                         /* force vPost if error occured */
1298                         if (err)
1299                                 return true;
1300
1301                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1302                         if (fw_ver < 0x00160e00)
1303                                 return true;
1304                 }
1305         }
1306
1307         /* Don't post if we need to reset whole hive on init */
1308         if (adev->gmc.xgmi.pending_reset)
1309                 return false;
1310
1311         if (adev->has_hw_reset) {
1312                 adev->has_hw_reset = false;
1313                 return true;
1314         }
1315
1316         /* bios scratch used on CIK+ */
1317         if (adev->asic_type >= CHIP_BONAIRE)
1318                 return amdgpu_atombios_scratch_need_asic_init(adev);
1319
1320         /* check MEM_SIZE for older asics */
1321         reg = amdgpu_asic_get_config_memsize(adev);
1322
1323         if ((reg != 0) && (reg != 0xffffffff))
1324                 return false;
1325
1326         return true;
1327 }
1328
1329 /**
1330  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1331  *
1332  * @adev: amdgpu_device pointer
1333  *
1334  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1335  * be set for this device.
1336  *
1337  * Returns true if it should be used or false if not.
1338  */
1339 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1340 {
1341         switch (amdgpu_aspm) {
1342         case -1:
1343                 break;
1344         case 0:
1345                 return false;
1346         case 1:
1347                 return true;
1348         default:
1349                 return false;
1350         }
1351         return pcie_aspm_enabled(adev->pdev);
1352 }
1353
1354 /* if we get transitioned to only one device, take VGA back */
1355 /**
1356  * amdgpu_device_vga_set_decode - enable/disable vga decode
1357  *
1358  * @pdev: PCI device pointer
1359  * @state: enable/disable vga decode
1360  *
1361  * Enable/disable vga decode (all asics).
1362  * Returns VGA resource flags.
1363  */
1364 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1365                 bool state)
1366 {
1367         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1368         amdgpu_asic_set_vga_state(adev, state);
1369         if (state)
1370                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1371                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1372         else
1373                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1374 }
1375
1376 /**
1377  * amdgpu_device_check_block_size - validate the vm block size
1378  *
1379  * @adev: amdgpu_device pointer
1380  *
1381  * Validates the vm block size specified via module parameter.
1382  * The vm block size defines number of bits in page table versus page directory,
1383  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1384  * page table and the remaining bits are in the page directory.
1385  */
1386 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1387 {
1388         /* defines number of bits in page table versus page directory,
1389          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1390          * page table and the remaining bits are in the page directory */
1391         if (amdgpu_vm_block_size == -1)
1392                 return;
1393
1394         if (amdgpu_vm_block_size < 9) {
1395                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1396                          amdgpu_vm_block_size);
1397                 amdgpu_vm_block_size = -1;
1398         }
1399 }
1400
1401 /**
1402  * amdgpu_device_check_vm_size - validate the vm size
1403  *
1404  * @adev: amdgpu_device pointer
1405  *
1406  * Validates the vm size in GB specified via module parameter.
1407  * The VM size is the size of the GPU virtual memory space in GB.
1408  */
1409 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1410 {
1411         /* no need to check the default value */
1412         if (amdgpu_vm_size == -1)
1413                 return;
1414
1415         if (amdgpu_vm_size < 1) {
1416                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1417                          amdgpu_vm_size);
1418                 amdgpu_vm_size = -1;
1419         }
1420 }
1421
1422 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1423 {
1424         struct sysinfo si;
1425         bool is_os_64 = (sizeof(void *) == 8);
1426         uint64_t total_memory;
1427         uint64_t dram_size_seven_GB = 0x1B8000000;
1428         uint64_t dram_size_three_GB = 0xB8000000;
1429
1430         if (amdgpu_smu_memory_pool_size == 0)
1431                 return;
1432
1433         if (!is_os_64) {
1434                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1435                 goto def_value;
1436         }
1437         si_meminfo(&si);
1438         total_memory = (uint64_t)si.totalram * si.mem_unit;
1439
1440         if ((amdgpu_smu_memory_pool_size == 1) ||
1441                 (amdgpu_smu_memory_pool_size == 2)) {
1442                 if (total_memory < dram_size_three_GB)
1443                         goto def_value1;
1444         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1445                 (amdgpu_smu_memory_pool_size == 8)) {
1446                 if (total_memory < dram_size_seven_GB)
1447                         goto def_value1;
1448         } else {
1449                 DRM_WARN("Smu memory pool size not supported\n");
1450                 goto def_value;
1451         }
1452         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1453
1454         return;
1455
1456 def_value1:
1457         DRM_WARN("No enough system memory\n");
1458 def_value:
1459         adev->pm.smu_prv_buffer_size = 0;
1460 }
1461
1462 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1463 {
1464         if (!(adev->flags & AMD_IS_APU) ||
1465             adev->asic_type < CHIP_RAVEN)
1466                 return 0;
1467
1468         switch (adev->asic_type) {
1469         case CHIP_RAVEN:
1470                 if (adev->pdev->device == 0x15dd)
1471                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1472                 if (adev->pdev->device == 0x15d8)
1473                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1474                 break;
1475         case CHIP_RENOIR:
1476                 if ((adev->pdev->device == 0x1636) ||
1477                     (adev->pdev->device == 0x164c))
1478                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1479                 else
1480                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1481                 break;
1482         case CHIP_VANGOGH:
1483                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1484                 break;
1485         case CHIP_YELLOW_CARP:
1486                 break;
1487         case CHIP_CYAN_SKILLFISH:
1488                 if ((adev->pdev->device == 0x13FE) ||
1489                     (adev->pdev->device == 0x143F))
1490                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1491                 break;
1492         default:
1493                 break;
1494         }
1495
1496         return 0;
1497 }
1498
1499 /**
1500  * amdgpu_device_check_arguments - validate module params
1501  *
1502  * @adev: amdgpu_device pointer
1503  *
1504  * Validates certain module parameters and updates
1505  * the associated values used by the driver (all asics).
1506  */
1507 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1508 {
1509         if (amdgpu_sched_jobs < 4) {
1510                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1511                          amdgpu_sched_jobs);
1512                 amdgpu_sched_jobs = 4;
1513         } else if (!is_power_of_2(amdgpu_sched_jobs)){
1514                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1515                          amdgpu_sched_jobs);
1516                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1517         }
1518
1519         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1520                 /* gart size must be greater or equal to 32M */
1521                 dev_warn(adev->dev, "gart size (%d) too small\n",
1522                          amdgpu_gart_size);
1523                 amdgpu_gart_size = -1;
1524         }
1525
1526         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1527                 /* gtt size must be greater or equal to 32M */
1528                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1529                                  amdgpu_gtt_size);
1530                 amdgpu_gtt_size = -1;
1531         }
1532
1533         /* valid range is between 4 and 9 inclusive */
1534         if (amdgpu_vm_fragment_size != -1 &&
1535             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1536                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1537                 amdgpu_vm_fragment_size = -1;
1538         }
1539
1540         if (amdgpu_sched_hw_submission < 2) {
1541                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1542                          amdgpu_sched_hw_submission);
1543                 amdgpu_sched_hw_submission = 2;
1544         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1545                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1546                          amdgpu_sched_hw_submission);
1547                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1548         }
1549
1550         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1551                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1552                 amdgpu_reset_method = -1;
1553         }
1554
1555         amdgpu_device_check_smu_prv_buffer_size(adev);
1556
1557         amdgpu_device_check_vm_size(adev);
1558
1559         amdgpu_device_check_block_size(adev);
1560
1561         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1562
1563         return 0;
1564 }
1565
1566 /**
1567  * amdgpu_switcheroo_set_state - set switcheroo state
1568  *
1569  * @pdev: pci dev pointer
1570  * @state: vga_switcheroo state
1571  *
1572  * Callback for the switcheroo driver.  Suspends or resumes
1573  * the asics before or after it is powered up using ACPI methods.
1574  */
1575 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1576                                         enum vga_switcheroo_state state)
1577 {
1578         struct drm_device *dev = pci_get_drvdata(pdev);
1579         int r;
1580
1581         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1582                 return;
1583
1584         if (state == VGA_SWITCHEROO_ON) {
1585                 pr_info("switched on\n");
1586                 /* don't suspend or resume card normally */
1587                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1588
1589                 pci_set_power_state(pdev, PCI_D0);
1590                 amdgpu_device_load_pci_state(pdev);
1591                 r = pci_enable_device(pdev);
1592                 if (r)
1593                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1594                 amdgpu_device_resume(dev, true);
1595
1596                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1597         } else {
1598                 pr_info("switched off\n");
1599                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1600                 amdgpu_device_suspend(dev, true);
1601                 amdgpu_device_cache_pci_state(pdev);
1602                 /* Shut down the device */
1603                 pci_disable_device(pdev);
1604                 pci_set_power_state(pdev, PCI_D3cold);
1605                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1606         }
1607 }
1608
1609 /**
1610  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1611  *
1612  * @pdev: pci dev pointer
1613  *
1614  * Callback for the switcheroo driver.  Check of the switcheroo
1615  * state can be changed.
1616  * Returns true if the state can be changed, false if not.
1617  */
1618 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1619 {
1620         struct drm_device *dev = pci_get_drvdata(pdev);
1621
1622         /*
1623         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1624         * locking inversion with the driver load path. And the access here is
1625         * completely racy anyway. So don't bother with locking for now.
1626         */
1627         return atomic_read(&dev->open_count) == 0;
1628 }
1629
1630 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1631         .set_gpu_state = amdgpu_switcheroo_set_state,
1632         .reprobe = NULL,
1633         .can_switch = amdgpu_switcheroo_can_switch,
1634 };
1635
1636 /**
1637  * amdgpu_device_ip_set_clockgating_state - set the CG state
1638  *
1639  * @dev: amdgpu_device pointer
1640  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1641  * @state: clockgating state (gate or ungate)
1642  *
1643  * Sets the requested clockgating state for all instances of
1644  * the hardware IP specified.
1645  * Returns the error code from the last instance.
1646  */
1647 int amdgpu_device_ip_set_clockgating_state(void *dev,
1648                                            enum amd_ip_block_type block_type,
1649                                            enum amd_clockgating_state state)
1650 {
1651         struct amdgpu_device *adev = dev;
1652         int i, r = 0;
1653
1654         for (i = 0; i < adev->num_ip_blocks; i++) {
1655                 if (!adev->ip_blocks[i].status.valid)
1656                         continue;
1657                 if (adev->ip_blocks[i].version->type != block_type)
1658                         continue;
1659                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1660                         continue;
1661                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1662                         (void *)adev, state);
1663                 if (r)
1664                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1665                                   adev->ip_blocks[i].version->funcs->name, r);
1666         }
1667         return r;
1668 }
1669
1670 /**
1671  * amdgpu_device_ip_set_powergating_state - set the PG state
1672  *
1673  * @dev: amdgpu_device pointer
1674  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1675  * @state: powergating state (gate or ungate)
1676  *
1677  * Sets the requested powergating state for all instances of
1678  * the hardware IP specified.
1679  * Returns the error code from the last instance.
1680  */
1681 int amdgpu_device_ip_set_powergating_state(void *dev,
1682                                            enum amd_ip_block_type block_type,
1683                                            enum amd_powergating_state state)
1684 {
1685         struct amdgpu_device *adev = dev;
1686         int i, r = 0;
1687
1688         for (i = 0; i < adev->num_ip_blocks; i++) {
1689                 if (!adev->ip_blocks[i].status.valid)
1690                         continue;
1691                 if (adev->ip_blocks[i].version->type != block_type)
1692                         continue;
1693                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1694                         continue;
1695                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1696                         (void *)adev, state);
1697                 if (r)
1698                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1699                                   adev->ip_blocks[i].version->funcs->name, r);
1700         }
1701         return r;
1702 }
1703
1704 /**
1705  * amdgpu_device_ip_get_clockgating_state - get the CG state
1706  *
1707  * @adev: amdgpu_device pointer
1708  * @flags: clockgating feature flags
1709  *
1710  * Walks the list of IPs on the device and updates the clockgating
1711  * flags for each IP.
1712  * Updates @flags with the feature flags for each hardware IP where
1713  * clockgating is enabled.
1714  */
1715 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1716                                             u64 *flags)
1717 {
1718         int i;
1719
1720         for (i = 0; i < adev->num_ip_blocks; i++) {
1721                 if (!adev->ip_blocks[i].status.valid)
1722                         continue;
1723                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1724                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1725         }
1726 }
1727
1728 /**
1729  * amdgpu_device_ip_wait_for_idle - wait for idle
1730  *
1731  * @adev: amdgpu_device pointer
1732  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1733  *
1734  * Waits for the request hardware IP to be idle.
1735  * Returns 0 for success or a negative error code on failure.
1736  */
1737 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1738                                    enum amd_ip_block_type block_type)
1739 {
1740         int i, r;
1741
1742         for (i = 0; i < adev->num_ip_blocks; i++) {
1743                 if (!adev->ip_blocks[i].status.valid)
1744                         continue;
1745                 if (adev->ip_blocks[i].version->type == block_type) {
1746                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1747                         if (r)
1748                                 return r;
1749                         break;
1750                 }
1751         }
1752         return 0;
1753
1754 }
1755
1756 /**
1757  * amdgpu_device_ip_is_idle - is the hardware IP idle
1758  *
1759  * @adev: amdgpu_device pointer
1760  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1761  *
1762  * Check if the hardware IP is idle or not.
1763  * Returns true if it the IP is idle, false if not.
1764  */
1765 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
1766                               enum amd_ip_block_type block_type)
1767 {
1768         int i;
1769
1770         for (i = 0; i < adev->num_ip_blocks; i++) {
1771                 if (!adev->ip_blocks[i].status.valid)
1772                         continue;
1773                 if (adev->ip_blocks[i].version->type == block_type)
1774                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
1775         }
1776         return true;
1777
1778 }
1779
1780 /**
1781  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
1782  *
1783  * @adev: amdgpu_device pointer
1784  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
1785  *
1786  * Returns a pointer to the hardware IP block structure
1787  * if it exists for the asic, otherwise NULL.
1788  */
1789 struct amdgpu_ip_block *
1790 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
1791                               enum amd_ip_block_type type)
1792 {
1793         int i;
1794
1795         for (i = 0; i < adev->num_ip_blocks; i++)
1796                 if (adev->ip_blocks[i].version->type == type)
1797                         return &adev->ip_blocks[i];
1798
1799         return NULL;
1800 }
1801
1802 /**
1803  * amdgpu_device_ip_block_version_cmp
1804  *
1805  * @adev: amdgpu_device pointer
1806  * @type: enum amd_ip_block_type
1807  * @major: major version
1808  * @minor: minor version
1809  *
1810  * return 0 if equal or greater
1811  * return 1 if smaller or the ip_block doesn't exist
1812  */
1813 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
1814                                        enum amd_ip_block_type type,
1815                                        u32 major, u32 minor)
1816 {
1817         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
1818
1819         if (ip_block && ((ip_block->version->major > major) ||
1820                         ((ip_block->version->major == major) &&
1821                         (ip_block->version->minor >= minor))))
1822                 return 0;
1823
1824         return 1;
1825 }
1826
1827 /**
1828  * amdgpu_device_ip_block_add
1829  *
1830  * @adev: amdgpu_device pointer
1831  * @ip_block_version: pointer to the IP to add
1832  *
1833  * Adds the IP block driver information to the collection of IPs
1834  * on the asic.
1835  */
1836 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
1837                                const struct amdgpu_ip_block_version *ip_block_version)
1838 {
1839         if (!ip_block_version)
1840                 return -EINVAL;
1841
1842         switch (ip_block_version->type) {
1843         case AMD_IP_BLOCK_TYPE_VCN:
1844                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
1845                         return 0;
1846                 break;
1847         case AMD_IP_BLOCK_TYPE_JPEG:
1848                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
1849                         return 0;
1850                 break;
1851         default:
1852                 break;
1853         }
1854
1855         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
1856                   ip_block_version->funcs->name);
1857
1858         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
1859
1860         return 0;
1861 }
1862
1863 /**
1864  * amdgpu_device_enable_virtual_display - enable virtual display feature
1865  *
1866  * @adev: amdgpu_device pointer
1867  *
1868  * Enabled the virtual display feature if the user has enabled it via
1869  * the module parameter virtual_display.  This feature provides a virtual
1870  * display hardware on headless boards or in virtualized environments.
1871  * This function parses and validates the configuration string specified by
1872  * the user and configues the virtual display configuration (number of
1873  * virtual connectors, crtcs, etc.) specified.
1874  */
1875 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
1876 {
1877         adev->enable_virtual_display = false;
1878
1879         if (amdgpu_virtual_display) {
1880                 const char *pci_address_name = pci_name(adev->pdev);
1881                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
1882
1883                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
1884                 pciaddstr_tmp = pciaddstr;
1885                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
1886                         pciaddname = strsep(&pciaddname_tmp, ",");
1887                         if (!strcmp("all", pciaddname)
1888                             || !strcmp(pci_address_name, pciaddname)) {
1889                                 long num_crtc;
1890                                 int res = -1;
1891
1892                                 adev->enable_virtual_display = true;
1893
1894                                 if (pciaddname_tmp)
1895                                         res = kstrtol(pciaddname_tmp, 10,
1896                                                       &num_crtc);
1897
1898                                 if (!res) {
1899                                         if (num_crtc < 1)
1900                                                 num_crtc = 1;
1901                                         if (num_crtc > 6)
1902                                                 num_crtc = 6;
1903                                         adev->mode_info.num_crtc = num_crtc;
1904                                 } else {
1905                                         adev->mode_info.num_crtc = 1;
1906                                 }
1907                                 break;
1908                         }
1909                 }
1910
1911                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
1912                          amdgpu_virtual_display, pci_address_name,
1913                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1914
1915                 kfree(pciaddstr);
1916         }
1917 }
1918
1919 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
1920 {
1921         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
1922                 adev->mode_info.num_crtc = 1;
1923                 adev->enable_virtual_display = true;
1924                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
1925                          adev->enable_virtual_display, adev->mode_info.num_crtc);
1926         }
1927 }
1928
1929 /**
1930  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
1931  *
1932  * @adev: amdgpu_device pointer
1933  *
1934  * Parses the asic configuration parameters specified in the gpu info
1935  * firmware and makes them availale to the driver for use in configuring
1936  * the asic.
1937  * Returns 0 on success, -EINVAL on failure.
1938  */
1939 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
1940 {
1941         const char *chip_name;
1942         char fw_name[40];
1943         int err;
1944         const struct gpu_info_firmware_header_v1_0 *hdr;
1945
1946         adev->firmware.gpu_info_fw = NULL;
1947
1948         if (adev->mman.discovery_bin) {
1949                 /*
1950                  * FIXME: The bounding box is still needed by Navi12, so
1951                  * temporarily read it from gpu_info firmware. Should be dropped
1952                  * when DAL no longer needs it.
1953                  */
1954                 if (adev->asic_type != CHIP_NAVI12)
1955                         return 0;
1956         }
1957
1958         switch (adev->asic_type) {
1959         default:
1960                 return 0;
1961         case CHIP_VEGA10:
1962                 chip_name = "vega10";
1963                 break;
1964         case CHIP_VEGA12:
1965                 chip_name = "vega12";
1966                 break;
1967         case CHIP_RAVEN:
1968                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
1969                         chip_name = "raven2";
1970                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
1971                         chip_name = "picasso";
1972                 else
1973                         chip_name = "raven";
1974                 break;
1975         case CHIP_ARCTURUS:
1976                 chip_name = "arcturus";
1977                 break;
1978         case CHIP_NAVI12:
1979                 chip_name = "navi12";
1980                 break;
1981         }
1982
1983         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
1984         err = request_firmware(&adev->firmware.gpu_info_fw, fw_name, adev->dev);
1985         if (err) {
1986                 dev_err(adev->dev,
1987                         "Failed to load gpu_info firmware \"%s\"\n",
1988                         fw_name);
1989                 goto out;
1990         }
1991         err = amdgpu_ucode_validate(adev->firmware.gpu_info_fw);
1992         if (err) {
1993                 dev_err(adev->dev,
1994                         "Failed to validate gpu_info firmware \"%s\"\n",
1995                         fw_name);
1996                 goto out;
1997         }
1998
1999         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2000         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2001
2002         switch (hdr->version_major) {
2003         case 1:
2004         {
2005                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2006                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2007                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2008
2009                 /*
2010                  * Should be droped when DAL no longer needs it.
2011                  */
2012                 if (adev->asic_type == CHIP_NAVI12)
2013                         goto parse_soc_bounding_box;
2014
2015                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2016                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2017                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2018                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2019                 adev->gfx.config.max_texture_channel_caches =
2020                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2021                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2022                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2023                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2024                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2025                 adev->gfx.config.double_offchip_lds_buf =
2026                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2027                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2028                 adev->gfx.cu_info.max_waves_per_simd =
2029                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2030                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2031                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2032                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2033                 if (hdr->version_minor >= 1) {
2034                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2035                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2036                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2037                         adev->gfx.config.num_sc_per_sh =
2038                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2039                         adev->gfx.config.num_packer_per_sc =
2040                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2041                 }
2042
2043 parse_soc_bounding_box:
2044                 /*
2045                  * soc bounding box info is not integrated in disocovery table,
2046                  * we always need to parse it from gpu info firmware if needed.
2047                  */
2048                 if (hdr->version_minor == 2) {
2049                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2050                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2051                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2052                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2053                 }
2054                 break;
2055         }
2056         default:
2057                 dev_err(adev->dev,
2058                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2059                 err = -EINVAL;
2060                 goto out;
2061         }
2062 out:
2063         return err;
2064 }
2065
2066 /**
2067  * amdgpu_device_ip_early_init - run early init for hardware IPs
2068  *
2069  * @adev: amdgpu_device pointer
2070  *
2071  * Early initialization pass for hardware IPs.  The hardware IPs that make
2072  * up each asic are discovered each IP's early_init callback is run.  This
2073  * is the first stage in initializing the asic.
2074  * Returns 0 on success, negative error code on failure.
2075  */
2076 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2077 {
2078         struct drm_device *dev = adev_to_drm(adev);
2079         struct pci_dev *parent;
2080         int i, r;
2081
2082         amdgpu_device_enable_virtual_display(adev);
2083
2084         if (amdgpu_sriov_vf(adev)) {
2085                 r = amdgpu_virt_request_full_gpu(adev, true);
2086                 if (r)
2087                         return r;
2088         }
2089
2090         switch (adev->asic_type) {
2091 #ifdef CONFIG_DRM_AMDGPU_SI
2092         case CHIP_VERDE:
2093         case CHIP_TAHITI:
2094         case CHIP_PITCAIRN:
2095         case CHIP_OLAND:
2096         case CHIP_HAINAN:
2097                 adev->family = AMDGPU_FAMILY_SI;
2098                 r = si_set_ip_blocks(adev);
2099                 if (r)
2100                         return r;
2101                 break;
2102 #endif
2103 #ifdef CONFIG_DRM_AMDGPU_CIK
2104         case CHIP_BONAIRE:
2105         case CHIP_HAWAII:
2106         case CHIP_KAVERI:
2107         case CHIP_KABINI:
2108         case CHIP_MULLINS:
2109                 if (adev->flags & AMD_IS_APU)
2110                         adev->family = AMDGPU_FAMILY_KV;
2111                 else
2112                         adev->family = AMDGPU_FAMILY_CI;
2113
2114                 r = cik_set_ip_blocks(adev);
2115                 if (r)
2116                         return r;
2117                 break;
2118 #endif
2119         case CHIP_TOPAZ:
2120         case CHIP_TONGA:
2121         case CHIP_FIJI:
2122         case CHIP_POLARIS10:
2123         case CHIP_POLARIS11:
2124         case CHIP_POLARIS12:
2125         case CHIP_VEGAM:
2126         case CHIP_CARRIZO:
2127         case CHIP_STONEY:
2128                 if (adev->flags & AMD_IS_APU)
2129                         adev->family = AMDGPU_FAMILY_CZ;
2130                 else
2131                         adev->family = AMDGPU_FAMILY_VI;
2132
2133                 r = vi_set_ip_blocks(adev);
2134                 if (r)
2135                         return r;
2136                 break;
2137         default:
2138                 r = amdgpu_discovery_set_ip_blocks(adev);
2139                 if (r)
2140                         return r;
2141                 break;
2142         }
2143
2144         if (amdgpu_has_atpx() &&
2145             (amdgpu_is_atpx_hybrid() ||
2146              amdgpu_has_atpx_dgpu_power_cntl()) &&
2147             ((adev->flags & AMD_IS_APU) == 0) &&
2148             !pci_is_thunderbolt_attached(to_pci_dev(dev->dev)))
2149                 adev->flags |= AMD_IS_PX;
2150
2151         if (!(adev->flags & AMD_IS_APU)) {
2152                 parent = pci_upstream_bridge(adev->pdev);
2153                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2154         }
2155
2156         amdgpu_amdkfd_device_probe(adev);
2157
2158         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2159         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2160                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2161         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2162                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2163
2164         for (i = 0; i < adev->num_ip_blocks; i++) {
2165                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2166                         DRM_ERROR("disabled ip block: %d <%s>\n",
2167                                   i, adev->ip_blocks[i].version->funcs->name);
2168                         adev->ip_blocks[i].status.valid = false;
2169                 } else {
2170                         if (adev->ip_blocks[i].version->funcs->early_init) {
2171                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2172                                 if (r == -ENOENT) {
2173                                         adev->ip_blocks[i].status.valid = false;
2174                                 } else if (r) {
2175                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2176                                                   adev->ip_blocks[i].version->funcs->name, r);
2177                                         return r;
2178                                 } else {
2179                                         adev->ip_blocks[i].status.valid = true;
2180                                 }
2181                         } else {
2182                                 adev->ip_blocks[i].status.valid = true;
2183                         }
2184                 }
2185                 /* get the vbios after the asic_funcs are set up */
2186                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2187                         r = amdgpu_device_parse_gpu_info_fw(adev);
2188                         if (r)
2189                                 return r;
2190
2191                         /* Read BIOS */
2192                         if (!amdgpu_get_bios(adev))
2193                                 return -EINVAL;
2194
2195                         r = amdgpu_atombios_init(adev);
2196                         if (r) {
2197                                 dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2198                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2199                                 return r;
2200                         }
2201
2202                         /*get pf2vf msg info at it's earliest time*/
2203                         if (amdgpu_sriov_vf(adev))
2204                                 amdgpu_virt_init_data_exchange(adev);
2205
2206                 }
2207         }
2208
2209         adev->cg_flags &= amdgpu_cg_mask;
2210         adev->pg_flags &= amdgpu_pg_mask;
2211
2212         return 0;
2213 }
2214
2215 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2216 {
2217         int i, r;
2218
2219         for (i = 0; i < adev->num_ip_blocks; i++) {
2220                 if (!adev->ip_blocks[i].status.sw)
2221                         continue;
2222                 if (adev->ip_blocks[i].status.hw)
2223                         continue;
2224                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2225                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2226                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2227                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2228                         if (r) {
2229                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2230                                           adev->ip_blocks[i].version->funcs->name, r);
2231                                 return r;
2232                         }
2233                         adev->ip_blocks[i].status.hw = true;
2234                 }
2235         }
2236
2237         return 0;
2238 }
2239
2240 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2241 {
2242         int i, r;
2243
2244         for (i = 0; i < adev->num_ip_blocks; i++) {
2245                 if (!adev->ip_blocks[i].status.sw)
2246                         continue;
2247                 if (adev->ip_blocks[i].status.hw)
2248                         continue;
2249                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2250                 if (r) {
2251                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2252                                   adev->ip_blocks[i].version->funcs->name, r);
2253                         return r;
2254                 }
2255                 adev->ip_blocks[i].status.hw = true;
2256         }
2257
2258         return 0;
2259 }
2260
2261 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2262 {
2263         int r = 0;
2264         int i;
2265         uint32_t smu_version;
2266
2267         if (adev->asic_type >= CHIP_VEGA10) {
2268                 for (i = 0; i < adev->num_ip_blocks; i++) {
2269                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2270                                 continue;
2271
2272                         if (!adev->ip_blocks[i].status.sw)
2273                                 continue;
2274
2275                         /* no need to do the fw loading again if already done*/
2276                         if (adev->ip_blocks[i].status.hw == true)
2277                                 break;
2278
2279                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2280                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2281                                 if (r) {
2282                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2283                                                           adev->ip_blocks[i].version->funcs->name, r);
2284                                         return r;
2285                                 }
2286                         } else {
2287                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2288                                 if (r) {
2289                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2290                                                           adev->ip_blocks[i].version->funcs->name, r);
2291                                         return r;
2292                                 }
2293                         }
2294
2295                         adev->ip_blocks[i].status.hw = true;
2296                         break;
2297                 }
2298         }
2299
2300         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2301                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2302
2303         return r;
2304 }
2305
2306 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2307 {
2308         long timeout;
2309         int r, i;
2310
2311         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2312                 struct amdgpu_ring *ring = adev->rings[i];
2313
2314                 /* No need to setup the GPU scheduler for rings that don't need it */
2315                 if (!ring || ring->no_scheduler)
2316                         continue;
2317
2318                 switch (ring->funcs->type) {
2319                 case AMDGPU_RING_TYPE_GFX:
2320                         timeout = adev->gfx_timeout;
2321                         break;
2322                 case AMDGPU_RING_TYPE_COMPUTE:
2323                         timeout = adev->compute_timeout;
2324                         break;
2325                 case AMDGPU_RING_TYPE_SDMA:
2326                         timeout = adev->sdma_timeout;
2327                         break;
2328                 default:
2329                         timeout = adev->video_timeout;
2330                         break;
2331                 }
2332
2333                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2334                                    ring->num_hw_submission, amdgpu_job_hang_limit,
2335                                    timeout, adev->reset_domain->wq,
2336                                    ring->sched_score, ring->name,
2337                                    adev->dev);
2338                 if (r) {
2339                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2340                                   ring->name);
2341                         return r;
2342                 }
2343         }
2344
2345         return 0;
2346 }
2347
2348
2349 /**
2350  * amdgpu_device_ip_init - run init for hardware IPs
2351  *
2352  * @adev: amdgpu_device pointer
2353  *
2354  * Main initialization pass for hardware IPs.  The list of all the hardware
2355  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2356  * are run.  sw_init initializes the software state associated with each IP
2357  * and hw_init initializes the hardware associated with each IP.
2358  * Returns 0 on success, negative error code on failure.
2359  */
2360 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2361 {
2362         int i, r;
2363
2364         r = amdgpu_ras_init(adev);
2365         if (r)
2366                 return r;
2367
2368         for (i = 0; i < adev->num_ip_blocks; i++) {
2369                 if (!adev->ip_blocks[i].status.valid)
2370                         continue;
2371                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2372                 if (r) {
2373                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2374                                   adev->ip_blocks[i].version->funcs->name, r);
2375                         goto init_failed;
2376                 }
2377                 adev->ip_blocks[i].status.sw = true;
2378
2379                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2380                         /* need to do common hw init early so everything is set up for gmc */
2381                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2382                         if (r) {
2383                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2384                                 goto init_failed;
2385                         }
2386                         adev->ip_blocks[i].status.hw = true;
2387                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2388                         /* need to do gmc hw init early so we can allocate gpu mem */
2389                         /* Try to reserve bad pages early */
2390                         if (amdgpu_sriov_vf(adev))
2391                                 amdgpu_virt_exchange_data(adev);
2392
2393                         r = amdgpu_device_vram_scratch_init(adev);
2394                         if (r) {
2395                                 DRM_ERROR("amdgpu_vram_scratch_init failed %d\n", r);
2396                                 goto init_failed;
2397                         }
2398                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2399                         if (r) {
2400                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2401                                 goto init_failed;
2402                         }
2403                         r = amdgpu_device_wb_init(adev);
2404                         if (r) {
2405                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2406                                 goto init_failed;
2407                         }
2408                         adev->ip_blocks[i].status.hw = true;
2409
2410                         /* right after GMC hw init, we create CSA */
2411                         if (amdgpu_mcbp) {
2412                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2413                                                                 AMDGPU_GEM_DOMAIN_VRAM,
2414                                                                 AMDGPU_CSA_SIZE);
2415                                 if (r) {
2416                                         DRM_ERROR("allocate CSA failed %d\n", r);
2417                                         goto init_failed;
2418                                 }
2419                         }
2420                 }
2421         }
2422
2423         if (amdgpu_sriov_vf(adev))
2424                 amdgpu_virt_init_data_exchange(adev);
2425
2426         r = amdgpu_ib_pool_init(adev);
2427         if (r) {
2428                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2429                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2430                 goto init_failed;
2431         }
2432
2433         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2434         if (r)
2435                 goto init_failed;
2436
2437         r = amdgpu_device_ip_hw_init_phase1(adev);
2438         if (r)
2439                 goto init_failed;
2440
2441         r = amdgpu_device_fw_loading(adev);
2442         if (r)
2443                 goto init_failed;
2444
2445         r = amdgpu_device_ip_hw_init_phase2(adev);
2446         if (r)
2447                 goto init_failed;
2448
2449         /*
2450          * retired pages will be loaded from eeprom and reserved here,
2451          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2452          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2453          * for I2C communication which only true at this point.
2454          *
2455          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2456          * failure from bad gpu situation and stop amdgpu init process
2457          * accordingly. For other failed cases, it will still release all
2458          * the resource and print error message, rather than returning one
2459          * negative value to upper level.
2460          *
2461          * Note: theoretically, this should be called before all vram allocations
2462          * to protect retired page from abusing
2463          */
2464         r = amdgpu_ras_recovery_init(adev);
2465         if (r)
2466                 goto init_failed;
2467
2468         /**
2469          * In case of XGMI grab extra reference for reset domain for this device
2470          */
2471         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2472                 if (amdgpu_xgmi_add_device(adev) == 0) {
2473                         if (!amdgpu_sriov_vf(adev)) {
2474                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2475
2476                                 if (WARN_ON(!hive)) {
2477                                         r = -ENOENT;
2478                                         goto init_failed;
2479                                 }
2480
2481                                 if (!hive->reset_domain ||
2482                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2483                                         r = -ENOENT;
2484                                         amdgpu_put_xgmi_hive(hive);
2485                                         goto init_failed;
2486                                 }
2487
2488                                 /* Drop the early temporary reset domain we created for device */
2489                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2490                                 adev->reset_domain = hive->reset_domain;
2491                                 amdgpu_put_xgmi_hive(hive);
2492                         }
2493                 }
2494         }
2495
2496         r = amdgpu_device_init_schedulers(adev);
2497         if (r)
2498                 goto init_failed;
2499
2500         /* Don't init kfd if whole hive need to be reset during init */
2501         if (!adev->gmc.xgmi.pending_reset)
2502                 amdgpu_amdkfd_device_init(adev);
2503
2504         amdgpu_fru_get_product_info(adev);
2505
2506 init_failed:
2507         if (amdgpu_sriov_vf(adev))
2508                 amdgpu_virt_release_full_gpu(adev, true);
2509
2510         return r;
2511 }
2512
2513 /**
2514  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2515  *
2516  * @adev: amdgpu_device pointer
2517  *
2518  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2519  * this function before a GPU reset.  If the value is retained after a
2520  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2521  */
2522 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2523 {
2524         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2525 }
2526
2527 /**
2528  * amdgpu_device_check_vram_lost - check if vram is valid
2529  *
2530  * @adev: amdgpu_device pointer
2531  *
2532  * Checks the reset magic value written to the gart pointer in VRAM.
2533  * The driver calls this after a GPU reset to see if the contents of
2534  * VRAM is lost or now.
2535  * returns true if vram is lost, false if not.
2536  */
2537 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2538 {
2539         if (memcmp(adev->gart.ptr, adev->reset_magic,
2540                         AMDGPU_RESET_MAGIC_NUM))
2541                 return true;
2542
2543         if (!amdgpu_in_reset(adev))
2544                 return false;
2545
2546         /*
2547          * For all ASICs with baco/mode1 reset, the VRAM is
2548          * always assumed to be lost.
2549          */
2550         switch (amdgpu_asic_reset_method(adev)) {
2551         case AMD_RESET_METHOD_BACO:
2552         case AMD_RESET_METHOD_MODE1:
2553                 return true;
2554         default:
2555                 return false;
2556         }
2557 }
2558
2559 /**
2560  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2561  *
2562  * @adev: amdgpu_device pointer
2563  * @state: clockgating state (gate or ungate)
2564  *
2565  * The list of all the hardware IPs that make up the asic is walked and the
2566  * set_clockgating_state callbacks are run.
2567  * Late initialization pass enabling clockgating for hardware IPs.
2568  * Fini or suspend, pass disabling clockgating for hardware IPs.
2569  * Returns 0 on success, negative error code on failure.
2570  */
2571
2572 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2573                                enum amd_clockgating_state state)
2574 {
2575         int i, j, r;
2576
2577         if (amdgpu_emu_mode == 1)
2578                 return 0;
2579
2580         for (j = 0; j < adev->num_ip_blocks; j++) {
2581                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2582                 if (!adev->ip_blocks[i].status.late_initialized)
2583                         continue;
2584                 /* skip CG for GFX on S0ix */
2585                 if (adev->in_s0ix &&
2586                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2587                         continue;
2588                 /* skip CG for VCE/UVD, it's handled specially */
2589                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2590                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2591                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2592                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2593                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2594                         /* enable clockgating to save power */
2595                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2596                                                                                      state);
2597                         if (r) {
2598                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2599                                           adev->ip_blocks[i].version->funcs->name, r);
2600                                 return r;
2601                         }
2602                 }
2603         }
2604
2605         return 0;
2606 }
2607
2608 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2609                                enum amd_powergating_state state)
2610 {
2611         int i, j, r;
2612
2613         if (amdgpu_emu_mode == 1)
2614                 return 0;
2615
2616         for (j = 0; j < adev->num_ip_blocks; j++) {
2617                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2618                 if (!adev->ip_blocks[i].status.late_initialized)
2619                         continue;
2620                 /* skip PG for GFX on S0ix */
2621                 if (adev->in_s0ix &&
2622                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX)
2623                         continue;
2624                 /* skip CG for VCE/UVD, it's handled specially */
2625                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2626                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2627                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2628                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2629                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2630                         /* enable powergating to save power */
2631                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2632                                                                                         state);
2633                         if (r) {
2634                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2635                                           adev->ip_blocks[i].version->funcs->name, r);
2636                                 return r;
2637                         }
2638                 }
2639         }
2640         return 0;
2641 }
2642
2643 static int amdgpu_device_enable_mgpu_fan_boost(void)
2644 {
2645         struct amdgpu_gpu_instance *gpu_ins;
2646         struct amdgpu_device *adev;
2647         int i, ret = 0;
2648
2649         mutex_lock(&mgpu_info.mutex);
2650
2651         /*
2652          * MGPU fan boost feature should be enabled
2653          * only when there are two or more dGPUs in
2654          * the system
2655          */
2656         if (mgpu_info.num_dgpu < 2)
2657                 goto out;
2658
2659         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2660                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2661                 adev = gpu_ins->adev;
2662                 if (!(adev->flags & AMD_IS_APU) &&
2663                     !gpu_ins->mgpu_fan_enabled) {
2664                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2665                         if (ret)
2666                                 break;
2667
2668                         gpu_ins->mgpu_fan_enabled = 1;
2669                 }
2670         }
2671
2672 out:
2673         mutex_unlock(&mgpu_info.mutex);
2674
2675         return ret;
2676 }
2677
2678 /**
2679  * amdgpu_device_ip_late_init - run late init for hardware IPs
2680  *
2681  * @adev: amdgpu_device pointer
2682  *
2683  * Late initialization pass for hardware IPs.  The list of all the hardware
2684  * IPs that make up the asic is walked and the late_init callbacks are run.
2685  * late_init covers any special initialization that an IP requires
2686  * after all of the have been initialized or something that needs to happen
2687  * late in the init process.
2688  * Returns 0 on success, negative error code on failure.
2689  */
2690 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2691 {
2692         struct amdgpu_gpu_instance *gpu_instance;
2693         int i = 0, r;
2694
2695         for (i = 0; i < adev->num_ip_blocks; i++) {
2696                 if (!adev->ip_blocks[i].status.hw)
2697                         continue;
2698                 if (adev->ip_blocks[i].version->funcs->late_init) {
2699                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2700                         if (r) {
2701                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2702                                           adev->ip_blocks[i].version->funcs->name, r);
2703                                 return r;
2704                         }
2705                 }
2706                 adev->ip_blocks[i].status.late_initialized = true;
2707         }
2708
2709         r = amdgpu_ras_late_init(adev);
2710         if (r) {
2711                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2712                 return r;
2713         }
2714
2715         amdgpu_ras_set_error_query_ready(adev, true);
2716
2717         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2718         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2719
2720         amdgpu_device_fill_reset_magic(adev);
2721
2722         r = amdgpu_device_enable_mgpu_fan_boost();
2723         if (r)
2724                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2725
2726         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2727         if (amdgpu_passthrough(adev) && ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1)||
2728                                adev->asic_type == CHIP_ALDEBARAN ))
2729                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2730
2731         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2732                 mutex_lock(&mgpu_info.mutex);
2733
2734                 /*
2735                  * Reset device p-state to low as this was booted with high.
2736                  *
2737                  * This should be performed only after all devices from the same
2738                  * hive get initialized.
2739                  *
2740                  * However, it's unknown how many device in the hive in advance.
2741                  * As this is counted one by one during devices initializations.
2742                  *
2743                  * So, we wait for all XGMI interlinked devices initialized.
2744                  * This may bring some delays as those devices may come from
2745                  * different hives. But that should be OK.
2746                  */
2747                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
2748                         for (i = 0; i < mgpu_info.num_gpu; i++) {
2749                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
2750                                 if (gpu_instance->adev->flags & AMD_IS_APU)
2751                                         continue;
2752
2753                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
2754                                                 AMDGPU_XGMI_PSTATE_MIN);
2755                                 if (r) {
2756                                         DRM_ERROR("pstate setting failed (%d).\n", r);
2757                                         break;
2758                                 }
2759                         }
2760                 }
2761
2762                 mutex_unlock(&mgpu_info.mutex);
2763         }
2764
2765         return 0;
2766 }
2767
2768 /**
2769  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
2770  *
2771  * @adev: amdgpu_device pointer
2772  *
2773  * For ASICs need to disable SMC first
2774  */
2775 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
2776 {
2777         int i, r;
2778
2779         if (adev->ip_versions[GC_HWIP][0] > IP_VERSION(9, 0, 0))
2780                 return;
2781
2782         for (i = 0; i < adev->num_ip_blocks; i++) {
2783                 if (!adev->ip_blocks[i].status.hw)
2784                         continue;
2785                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
2786                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2787                         /* XXX handle errors */
2788                         if (r) {
2789                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2790                                           adev->ip_blocks[i].version->funcs->name, r);
2791                         }
2792                         adev->ip_blocks[i].status.hw = false;
2793                         break;
2794                 }
2795         }
2796 }
2797
2798 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
2799 {
2800         int i, r;
2801
2802         for (i = 0; i < adev->num_ip_blocks; i++) {
2803                 if (!adev->ip_blocks[i].version->funcs->early_fini)
2804                         continue;
2805
2806                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
2807                 if (r) {
2808                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
2809                                   adev->ip_blocks[i].version->funcs->name, r);
2810                 }
2811         }
2812
2813         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2814         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2815
2816         amdgpu_amdkfd_suspend(adev, false);
2817
2818         /* Workaroud for ASICs need to disable SMC first */
2819         amdgpu_device_smu_fini_early(adev);
2820
2821         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2822                 if (!adev->ip_blocks[i].status.hw)
2823                         continue;
2824
2825                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
2826                 /* XXX handle errors */
2827                 if (r) {
2828                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
2829                                   adev->ip_blocks[i].version->funcs->name, r);
2830                 }
2831
2832                 adev->ip_blocks[i].status.hw = false;
2833         }
2834
2835         if (amdgpu_sriov_vf(adev)) {
2836                 if (amdgpu_virt_release_full_gpu(adev, false))
2837                         DRM_ERROR("failed to release exclusive mode on fini\n");
2838         }
2839
2840         return 0;
2841 }
2842
2843 /**
2844  * amdgpu_device_ip_fini - run fini for hardware IPs
2845  *
2846  * @adev: amdgpu_device pointer
2847  *
2848  * Main teardown pass for hardware IPs.  The list of all the hardware
2849  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
2850  * are run.  hw_fini tears down the hardware associated with each IP
2851  * and sw_fini tears down any software state associated with each IP.
2852  * Returns 0 on success, negative error code on failure.
2853  */
2854 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
2855 {
2856         int i, r;
2857
2858         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
2859                 amdgpu_virt_release_ras_err_handler_data(adev);
2860
2861         if (adev->gmc.xgmi.num_physical_nodes > 1)
2862                 amdgpu_xgmi_remove_device(adev);
2863
2864         amdgpu_amdkfd_device_fini_sw(adev);
2865
2866         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2867                 if (!adev->ip_blocks[i].status.sw)
2868                         continue;
2869
2870                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2871                         amdgpu_ucode_free_bo(adev);
2872                         amdgpu_free_static_csa(&adev->virt.csa_obj);
2873                         amdgpu_device_wb_fini(adev);
2874                         amdgpu_device_vram_scratch_fini(adev);
2875                         amdgpu_ib_pool_fini(adev);
2876                 }
2877
2878                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
2879                 /* XXX handle errors */
2880                 if (r) {
2881                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
2882                                   adev->ip_blocks[i].version->funcs->name, r);
2883                 }
2884                 adev->ip_blocks[i].status.sw = false;
2885                 adev->ip_blocks[i].status.valid = false;
2886         }
2887
2888         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2889                 if (!adev->ip_blocks[i].status.late_initialized)
2890                         continue;
2891                 if (adev->ip_blocks[i].version->funcs->late_fini)
2892                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
2893                 adev->ip_blocks[i].status.late_initialized = false;
2894         }
2895
2896         amdgpu_ras_fini(adev);
2897
2898         return 0;
2899 }
2900
2901 /**
2902  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
2903  *
2904  * @work: work_struct.
2905  */
2906 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
2907 {
2908         struct amdgpu_device *adev =
2909                 container_of(work, struct amdgpu_device, delayed_init_work.work);
2910         int r;
2911
2912         r = amdgpu_ib_ring_tests(adev);
2913         if (r)
2914                 DRM_ERROR("ib ring test failed (%d).\n", r);
2915 }
2916
2917 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
2918 {
2919         struct amdgpu_device *adev =
2920                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
2921
2922         WARN_ON_ONCE(adev->gfx.gfx_off_state);
2923         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
2924
2925         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
2926                 adev->gfx.gfx_off_state = true;
2927 }
2928
2929 /**
2930  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
2931  *
2932  * @adev: amdgpu_device pointer
2933  *
2934  * Main suspend function for hardware IPs.  The list of all the hardware
2935  * IPs that make up the asic is walked, clockgating is disabled and the
2936  * suspend callbacks are run.  suspend puts the hardware and software state
2937  * in each IP into a state suitable for suspend.
2938  * Returns 0 on success, negative error code on failure.
2939  */
2940 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
2941 {
2942         int i, r;
2943
2944         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
2945         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
2946
2947         /*
2948          * Per PMFW team's suggestion, driver needs to handle gfxoff
2949          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
2950          * scenario. Add the missing df cstate disablement here.
2951          */
2952         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
2953                 dev_warn(adev->dev, "Failed to disallow df cstate");
2954
2955         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2956                 if (!adev->ip_blocks[i].status.valid)
2957                         continue;
2958
2959                 /* displays are handled separately */
2960                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
2961                         continue;
2962
2963                 /* XXX handle errors */
2964                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
2965                 /* XXX handle errors */
2966                 if (r) {
2967                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
2968                                   adev->ip_blocks[i].version->funcs->name, r);
2969                         return r;
2970                 }
2971
2972                 adev->ip_blocks[i].status.hw = false;
2973         }
2974
2975         return 0;
2976 }
2977
2978 /**
2979  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
2980  *
2981  * @adev: amdgpu_device pointer
2982  *
2983  * Main suspend function for hardware IPs.  The list of all the hardware
2984  * IPs that make up the asic is walked, clockgating is disabled and the
2985  * suspend callbacks are run.  suspend puts the hardware and software state
2986  * in each IP into a state suitable for suspend.
2987  * Returns 0 on success, negative error code on failure.
2988  */
2989 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
2990 {
2991         int i, r;
2992
2993         if (adev->in_s0ix)
2994                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
2995
2996         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
2997                 if (!adev->ip_blocks[i].status.valid)
2998                         continue;
2999                 /* displays are handled in phase1 */
3000                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3001                         continue;
3002                 /* PSP lost connection when err_event_athub occurs */
3003                 if (amdgpu_ras_intr_triggered() &&
3004                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3005                         adev->ip_blocks[i].status.hw = false;
3006                         continue;
3007                 }
3008
3009                 /* skip unnecessary suspend if we do not initialize them yet */
3010                 if (adev->gmc.xgmi.pending_reset &&
3011                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3012                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3013                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3014                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3015                         adev->ip_blocks[i].status.hw = false;
3016                         continue;
3017                 }
3018
3019                 /* skip suspend of gfx and psp for S0ix
3020                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3021                  * like at runtime. PSP is also part of the always on hardware
3022                  * so no need to suspend it.
3023                  */
3024                 if (adev->in_s0ix &&
3025                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3026                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX))
3027                         continue;
3028
3029                 /* XXX handle errors */
3030                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3031                 /* XXX handle errors */
3032                 if (r) {
3033                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3034                                   adev->ip_blocks[i].version->funcs->name, r);
3035                 }
3036                 adev->ip_blocks[i].status.hw = false;
3037                 /* handle putting the SMC in the appropriate state */
3038                 if(!amdgpu_sriov_vf(adev)){
3039                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3040                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3041                                 if (r) {
3042                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3043                                                         adev->mp1_state, r);
3044                                         return r;
3045                                 }
3046                         }
3047                 }
3048         }
3049
3050         return 0;
3051 }
3052
3053 /**
3054  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3055  *
3056  * @adev: amdgpu_device pointer
3057  *
3058  * Main suspend function for hardware IPs.  The list of all the hardware
3059  * IPs that make up the asic is walked, clockgating is disabled and the
3060  * suspend callbacks are run.  suspend puts the hardware and software state
3061  * in each IP into a state suitable for suspend.
3062  * Returns 0 on success, negative error code on failure.
3063  */
3064 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3065 {
3066         int r;
3067
3068         if (amdgpu_sriov_vf(adev)) {
3069                 amdgpu_virt_fini_data_exchange(adev);
3070                 amdgpu_virt_request_full_gpu(adev, false);
3071         }
3072
3073         r = amdgpu_device_ip_suspend_phase1(adev);
3074         if (r)
3075                 return r;
3076         r = amdgpu_device_ip_suspend_phase2(adev);
3077
3078         if (amdgpu_sriov_vf(adev))
3079                 amdgpu_virt_release_full_gpu(adev, false);
3080
3081         return r;
3082 }
3083
3084 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3085 {
3086         int i, r;
3087
3088         static enum amd_ip_block_type ip_order[] = {
3089                 AMD_IP_BLOCK_TYPE_COMMON,
3090                 AMD_IP_BLOCK_TYPE_GMC,
3091                 AMD_IP_BLOCK_TYPE_PSP,
3092                 AMD_IP_BLOCK_TYPE_IH,
3093         };
3094
3095         for (i = 0; i < adev->num_ip_blocks; i++) {
3096                 int j;
3097                 struct amdgpu_ip_block *block;
3098
3099                 block = &adev->ip_blocks[i];
3100                 block->status.hw = false;
3101
3102                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3103
3104                         if (block->version->type != ip_order[j] ||
3105                                 !block->status.valid)
3106                                 continue;
3107
3108                         r = block->version->funcs->hw_init(adev);
3109                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3110                         if (r)
3111                                 return r;
3112                         block->status.hw = true;
3113                 }
3114         }
3115
3116         return 0;
3117 }
3118
3119 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3120 {
3121         int i, r;
3122
3123         static enum amd_ip_block_type ip_order[] = {
3124                 AMD_IP_BLOCK_TYPE_SMC,
3125                 AMD_IP_BLOCK_TYPE_DCE,
3126                 AMD_IP_BLOCK_TYPE_GFX,
3127                 AMD_IP_BLOCK_TYPE_SDMA,
3128                 AMD_IP_BLOCK_TYPE_UVD,
3129                 AMD_IP_BLOCK_TYPE_VCE,
3130                 AMD_IP_BLOCK_TYPE_VCN
3131         };
3132
3133         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3134                 int j;
3135                 struct amdgpu_ip_block *block;
3136
3137                 for (j = 0; j < adev->num_ip_blocks; j++) {
3138                         block = &adev->ip_blocks[j];
3139
3140                         if (block->version->type != ip_order[i] ||
3141                                 !block->status.valid ||
3142                                 block->status.hw)
3143                                 continue;
3144
3145                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3146                                 r = block->version->funcs->resume(adev);
3147                         else
3148                                 r = block->version->funcs->hw_init(adev);
3149
3150                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3151                         if (r)
3152                                 return r;
3153                         block->status.hw = true;
3154                 }
3155         }
3156
3157         return 0;
3158 }
3159
3160 /**
3161  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3162  *
3163  * @adev: amdgpu_device pointer
3164  *
3165  * First resume function for hardware IPs.  The list of all the hardware
3166  * IPs that make up the asic is walked and the resume callbacks are run for
3167  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3168  * after a suspend and updates the software state as necessary.  This
3169  * function is also used for restoring the GPU after a GPU reset.
3170  * Returns 0 on success, negative error code on failure.
3171  */
3172 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3173 {
3174         int i, r;
3175
3176         for (i = 0; i < adev->num_ip_blocks; i++) {
3177                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3178                         continue;
3179                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3180                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3181                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3182                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3183
3184                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3185                         if (r) {
3186                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3187                                           adev->ip_blocks[i].version->funcs->name, r);
3188                                 return r;
3189                         }
3190                         adev->ip_blocks[i].status.hw = true;
3191                 }
3192         }
3193
3194         return 0;
3195 }
3196
3197 /**
3198  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3199  *
3200  * @adev: amdgpu_device pointer
3201  *
3202  * First resume function for hardware IPs.  The list of all the hardware
3203  * IPs that make up the asic is walked and the resume callbacks are run for
3204  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3205  * functional state after a suspend and updates the software state as
3206  * necessary.  This function is also used for restoring the GPU after a GPU
3207  * reset.
3208  * Returns 0 on success, negative error code on failure.
3209  */
3210 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3211 {
3212         int i, r;
3213
3214         for (i = 0; i < adev->num_ip_blocks; i++) {
3215                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3216                         continue;
3217                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3218                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3219                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3220                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3221                         continue;
3222                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3223                 if (r) {
3224                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3225                                   adev->ip_blocks[i].version->funcs->name, r);
3226                         return r;
3227                 }
3228                 adev->ip_blocks[i].status.hw = true;
3229
3230                 if (adev->in_s0ix && adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3231                         /* disable gfxoff for IP resume. The gfxoff will be re-enabled in
3232                          * amdgpu_device_resume() after IP resume.
3233                          */
3234                         amdgpu_gfx_off_ctrl(adev, false);
3235                         DRM_DEBUG("will disable gfxoff for re-initializing other blocks\n");
3236                 }
3237
3238         }
3239
3240         return 0;
3241 }
3242
3243 /**
3244  * amdgpu_device_ip_resume - run resume for hardware IPs
3245  *
3246  * @adev: amdgpu_device pointer
3247  *
3248  * Main resume function for hardware IPs.  The hardware IPs
3249  * are split into two resume functions because they are
3250  * are also used in in recovering from a GPU reset and some additional
3251  * steps need to be take between them.  In this case (S3/S4) they are
3252  * run sequentially.
3253  * Returns 0 on success, negative error code on failure.
3254  */
3255 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3256 {
3257         int r;
3258
3259         r = amdgpu_amdkfd_resume_iommu(adev);
3260         if (r)
3261                 return r;
3262
3263         r = amdgpu_device_ip_resume_phase1(adev);
3264         if (r)
3265                 return r;
3266
3267         r = amdgpu_device_fw_loading(adev);
3268         if (r)
3269                 return r;
3270
3271         r = amdgpu_device_ip_resume_phase2(adev);
3272
3273         return r;
3274 }
3275
3276 /**
3277  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3278  *
3279  * @adev: amdgpu_device pointer
3280  *
3281  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3282  */
3283 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3284 {
3285         if (amdgpu_sriov_vf(adev)) {
3286                 if (adev->is_atom_fw) {
3287                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3288                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3289                 } else {
3290                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3291                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3292                 }
3293
3294                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3295                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3296         }
3297 }
3298
3299 /**
3300  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3301  *
3302  * @asic_type: AMD asic type
3303  *
3304  * Check if there is DC (new modesetting infrastructre) support for an asic.
3305  * returns true if DC has support, false if not.
3306  */
3307 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3308 {
3309         switch (asic_type) {
3310 #ifdef CONFIG_DRM_AMDGPU_SI
3311         case CHIP_HAINAN:
3312 #endif
3313         case CHIP_TOPAZ:
3314                 /* chips with no display hardware */
3315                 return false;
3316 #if defined(CONFIG_DRM_AMD_DC)
3317         case CHIP_TAHITI:
3318         case CHIP_PITCAIRN:
3319         case CHIP_VERDE:
3320         case CHIP_OLAND:
3321                 /*
3322                  * We have systems in the wild with these ASICs that require
3323                  * LVDS and VGA support which is not supported with DC.
3324                  *
3325                  * Fallback to the non-DC driver here by default so as not to
3326                  * cause regressions.
3327                  */
3328 #if defined(CONFIG_DRM_AMD_DC_SI)
3329                 return amdgpu_dc > 0;
3330 #else
3331                 return false;
3332 #endif
3333         case CHIP_BONAIRE:
3334         case CHIP_KAVERI:
3335         case CHIP_KABINI:
3336         case CHIP_MULLINS:
3337                 /*
3338                  * We have systems in the wild with these ASICs that require
3339                  * VGA support which is not supported with DC.
3340                  *
3341                  * Fallback to the non-DC driver here by default so as not to
3342                  * cause regressions.
3343                  */
3344                 return amdgpu_dc > 0;
3345         default:
3346                 return amdgpu_dc != 0;
3347 #else
3348         default:
3349                 if (amdgpu_dc > 0)
3350                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter "
3351                                          "but isn't supported by ASIC, ignoring\n");
3352                 return false;
3353 #endif
3354         }
3355 }
3356
3357 /**
3358  * amdgpu_device_has_dc_support - check if dc is supported
3359  *
3360  * @adev: amdgpu_device pointer
3361  *
3362  * Returns true for supported, false for not supported
3363  */
3364 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3365 {
3366         if (adev->enable_virtual_display ||
3367             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3368                 return false;
3369
3370         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3371 }
3372
3373 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3374 {
3375         struct amdgpu_device *adev =
3376                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3377         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3378
3379         /* It's a bug to not have a hive within this function */
3380         if (WARN_ON(!hive))
3381                 return;
3382
3383         /*
3384          * Use task barrier to synchronize all xgmi reset works across the
3385          * hive. task_barrier_enter and task_barrier_exit will block
3386          * until all the threads running the xgmi reset works reach
3387          * those points. task_barrier_full will do both blocks.
3388          */
3389         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3390
3391                 task_barrier_enter(&hive->tb);
3392                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3393
3394                 if (adev->asic_reset_res)
3395                         goto fail;
3396
3397                 task_barrier_exit(&hive->tb);
3398                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3399
3400                 if (adev->asic_reset_res)
3401                         goto fail;
3402
3403                 if (adev->mmhub.ras && adev->mmhub.ras->ras_block.hw_ops &&
3404                     adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
3405                         adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(adev);
3406         } else {
3407
3408                 task_barrier_full(&hive->tb);
3409                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3410         }
3411
3412 fail:
3413         if (adev->asic_reset_res)
3414                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3415                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3416         amdgpu_put_xgmi_hive(hive);
3417 }
3418
3419 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3420 {
3421         char *input = amdgpu_lockup_timeout;
3422         char *timeout_setting = NULL;
3423         int index = 0;
3424         long timeout;
3425         int ret = 0;
3426
3427         /*
3428          * By default timeout for non compute jobs is 10000
3429          * and 60000 for compute jobs.
3430          * In SR-IOV or passthrough mode, timeout for compute
3431          * jobs are 60000 by default.
3432          */
3433         adev->gfx_timeout = msecs_to_jiffies(10000);
3434         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3435         if (amdgpu_sriov_vf(adev))
3436                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3437                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3438         else
3439                 adev->compute_timeout =  msecs_to_jiffies(60000);
3440
3441         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3442                 while ((timeout_setting = strsep(&input, ",")) &&
3443                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3444                         ret = kstrtol(timeout_setting, 0, &timeout);
3445                         if (ret)
3446                                 return ret;
3447
3448                         if (timeout == 0) {
3449                                 index++;
3450                                 continue;
3451                         } else if (timeout < 0) {
3452                                 timeout = MAX_SCHEDULE_TIMEOUT;
3453                                 dev_warn(adev->dev, "lockup timeout disabled");
3454                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3455                         } else {
3456                                 timeout = msecs_to_jiffies(timeout);
3457                         }
3458
3459                         switch (index++) {
3460                         case 0:
3461                                 adev->gfx_timeout = timeout;
3462                                 break;
3463                         case 1:
3464                                 adev->compute_timeout = timeout;
3465                                 break;
3466                         case 2:
3467                                 adev->sdma_timeout = timeout;
3468                                 break;
3469                         case 3:
3470                                 adev->video_timeout = timeout;
3471                                 break;
3472                         default:
3473                                 break;
3474                         }
3475                 }
3476                 /*
3477                  * There is only one value specified and
3478                  * it should apply to all non-compute jobs.
3479                  */
3480                 if (index == 1) {
3481                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3482                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3483                                 adev->compute_timeout = adev->gfx_timeout;
3484                 }
3485         }
3486
3487         return ret;
3488 }
3489
3490 /**
3491  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3492  *
3493  * @adev: amdgpu_device pointer
3494  *
3495  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3496  */
3497 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3498 {
3499         struct iommu_domain *domain;
3500
3501         domain = iommu_get_domain_for_dev(adev->dev);
3502         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3503                 adev->ram_is_direct_mapped = true;
3504 }
3505
3506 static const struct attribute *amdgpu_dev_attributes[] = {
3507         &dev_attr_product_name.attr,
3508         &dev_attr_product_number.attr,
3509         &dev_attr_serial_number.attr,
3510         &dev_attr_pcie_replay_count.attr,
3511         NULL
3512 };
3513
3514 /**
3515  * amdgpu_device_init - initialize the driver
3516  *
3517  * @adev: amdgpu_device pointer
3518  * @flags: driver flags
3519  *
3520  * Initializes the driver info and hw (all asics).
3521  * Returns 0 for success or an error on failure.
3522  * Called at driver startup.
3523  */
3524 int amdgpu_device_init(struct amdgpu_device *adev,
3525                        uint32_t flags)
3526 {
3527         struct drm_device *ddev = adev_to_drm(adev);
3528         struct pci_dev *pdev = adev->pdev;
3529         int r, i;
3530         bool px = false;
3531         u32 max_MBps;
3532
3533         adev->shutdown = false;
3534         adev->flags = flags;
3535
3536         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3537                 adev->asic_type = amdgpu_force_asic_type;
3538         else
3539                 adev->asic_type = flags & AMD_ASIC_MASK;
3540
3541         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3542         if (amdgpu_emu_mode == 1)
3543                 adev->usec_timeout *= 10;
3544         adev->gmc.gart_size = 512 * 1024 * 1024;
3545         adev->accel_working = false;
3546         adev->num_rings = 0;
3547         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3548         adev->mman.buffer_funcs = NULL;
3549         adev->mman.buffer_funcs_ring = NULL;
3550         adev->vm_manager.vm_pte_funcs = NULL;
3551         adev->vm_manager.vm_pte_num_scheds = 0;
3552         adev->gmc.gmc_funcs = NULL;
3553         adev->harvest_ip_mask = 0x0;
3554         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3555         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3556
3557         adev->smc_rreg = &amdgpu_invalid_rreg;
3558         adev->smc_wreg = &amdgpu_invalid_wreg;
3559         adev->pcie_rreg = &amdgpu_invalid_rreg;
3560         adev->pcie_wreg = &amdgpu_invalid_wreg;
3561         adev->pciep_rreg = &amdgpu_invalid_rreg;
3562         adev->pciep_wreg = &amdgpu_invalid_wreg;
3563         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3564         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3565         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3566         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3567         adev->didt_rreg = &amdgpu_invalid_rreg;
3568         adev->didt_wreg = &amdgpu_invalid_wreg;
3569         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3570         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3571         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3572         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3573
3574         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3575                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3576                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3577
3578         /* mutex initialization are all done here so we
3579          * can recall function without having locking issues */
3580         mutex_init(&adev->firmware.mutex);
3581         mutex_init(&adev->pm.mutex);
3582         mutex_init(&adev->gfx.gpu_clock_mutex);
3583         mutex_init(&adev->srbm_mutex);
3584         mutex_init(&adev->gfx.pipe_reserve_mutex);
3585         mutex_init(&adev->gfx.gfx_off_mutex);
3586         mutex_init(&adev->grbm_idx_mutex);
3587         mutex_init(&adev->mn_lock);
3588         mutex_init(&adev->virt.vf_errors.lock);
3589         hash_init(adev->mn_hash);
3590         mutex_init(&adev->psp.mutex);
3591         mutex_init(&adev->notifier_lock);
3592         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3593         mutex_init(&adev->benchmark_mutex);
3594
3595         amdgpu_device_init_apu_flags(adev);
3596
3597         r = amdgpu_device_check_arguments(adev);
3598         if (r)
3599                 return r;
3600
3601         spin_lock_init(&adev->mmio_idx_lock);
3602         spin_lock_init(&adev->smc_idx_lock);
3603         spin_lock_init(&adev->pcie_idx_lock);
3604         spin_lock_init(&adev->uvd_ctx_idx_lock);
3605         spin_lock_init(&adev->didt_idx_lock);
3606         spin_lock_init(&adev->gc_cac_idx_lock);
3607         spin_lock_init(&adev->se_cac_idx_lock);
3608         spin_lock_init(&adev->audio_endpt_idx_lock);
3609         spin_lock_init(&adev->mm_stats.lock);
3610
3611         INIT_LIST_HEAD(&adev->shadow_list);
3612         mutex_init(&adev->shadow_list_lock);
3613
3614         INIT_LIST_HEAD(&adev->reset_list);
3615
3616         INIT_LIST_HEAD(&adev->ras_list);
3617
3618         INIT_DELAYED_WORK(&adev->delayed_init_work,
3619                           amdgpu_device_delayed_init_work_handler);
3620         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3621                           amdgpu_device_delay_enable_gfx_off);
3622
3623         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3624
3625         adev->gfx.gfx_off_req_count = 1;
3626         adev->gfx.gfx_off_residency = 0;
3627         adev->gfx.gfx_off_entrycount = 0;
3628         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3629
3630         atomic_set(&adev->throttling_logging_enabled, 1);
3631         /*
3632          * If throttling continues, logging will be performed every minute
3633          * to avoid log flooding. "-1" is subtracted since the thermal
3634          * throttling interrupt comes every second. Thus, the total logging
3635          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3636          * for throttling interrupt) = 60 seconds.
3637          */
3638         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3639         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3640
3641         /* Registers mapping */
3642         /* TODO: block userspace mapping of io register */
3643         if (adev->asic_type >= CHIP_BONAIRE) {
3644                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3645                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3646         } else {
3647                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3648                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3649         }
3650
3651         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3652                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3653
3654         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3655         if (adev->rmmio == NULL) {
3656                 return -ENOMEM;
3657         }
3658         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3659         DRM_INFO("register mmio size: %u\n", (unsigned)adev->rmmio_size);
3660
3661         amdgpu_device_get_pcie_info(adev);
3662
3663         if (amdgpu_mcbp)
3664                 DRM_INFO("MCBP is enabled\n");
3665
3666         /*
3667          * Reset domain needs to be present early, before XGMI hive discovered
3668          * (if any) and intitialized to use reset sem and in_gpu reset flag
3669          * early on during init and before calling to RREG32.
3670          */
3671         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3672         if (!adev->reset_domain)
3673                 return -ENOMEM;
3674
3675         /* detect hw virtualization here */
3676         amdgpu_detect_virtualization(adev);
3677
3678         r = amdgpu_device_get_job_timeout_settings(adev);
3679         if (r) {
3680                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3681                 return r;
3682         }
3683
3684         /* early init functions */
3685         r = amdgpu_device_ip_early_init(adev);
3686         if (r)
3687                 return r;
3688
3689         /* Enable TMZ based on IP_VERSION */
3690         amdgpu_gmc_tmz_set(adev);
3691
3692         amdgpu_gmc_noretry_set(adev);
3693         /* Need to get xgmi info early to decide the reset behavior*/
3694         if (adev->gmc.xgmi.supported) {
3695                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3696                 if (r)
3697                         return r;
3698         }
3699
3700         /* enable PCIE atomic ops */
3701         if (amdgpu_sriov_vf(adev))
3702                 adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3703                         adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3704                         (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3705         else
3706                 adev->have_atomics_support =
3707                         !pci_enable_atomic_ops_to_root(adev->pdev,
3708                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
3709                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
3710         if (!adev->have_atomics_support)
3711                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
3712
3713         /* doorbell bar mapping and doorbell index init*/
3714         amdgpu_device_doorbell_init(adev);
3715
3716         if (amdgpu_emu_mode == 1) {
3717                 /* post the asic on emulation mode */
3718                 emu_soc_asic_init(adev);
3719                 goto fence_driver_init;
3720         }
3721
3722         amdgpu_reset_init(adev);
3723
3724         /* detect if we are with an SRIOV vbios */
3725         amdgpu_device_detect_sriov_bios(adev);
3726
3727         /* check if we need to reset the asic
3728          *  E.g., driver was not cleanly unloaded previously, etc.
3729          */
3730         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
3731                 if (adev->gmc.xgmi.num_physical_nodes) {
3732                         dev_info(adev->dev, "Pending hive reset.\n");
3733                         adev->gmc.xgmi.pending_reset = true;
3734                         /* Only need to init necessary block for SMU to handle the reset */
3735                         for (i = 0; i < adev->num_ip_blocks; i++) {
3736                                 if (!adev->ip_blocks[i].status.valid)
3737                                         continue;
3738                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3739                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3740                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3741                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
3742                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
3743                                                 adev->ip_blocks[i].version->funcs->name);
3744                                         adev->ip_blocks[i].status.hw = true;
3745                                 }
3746                         }
3747                 } else {
3748                         r = amdgpu_asic_reset(adev);
3749                         if (r) {
3750                                 dev_err(adev->dev, "asic reset on init failed\n");
3751                                 goto failed;
3752                         }
3753                 }
3754         }
3755
3756         pci_enable_pcie_error_reporting(adev->pdev);
3757
3758         /* Post card if necessary */
3759         if (amdgpu_device_need_post(adev)) {
3760                 if (!adev->bios) {
3761                         dev_err(adev->dev, "no vBIOS found\n");
3762                         r = -EINVAL;
3763                         goto failed;
3764                 }
3765                 DRM_INFO("GPU posting now...\n");
3766                 r = amdgpu_device_asic_init(adev);
3767                 if (r) {
3768                         dev_err(adev->dev, "gpu post error!\n");
3769                         goto failed;
3770                 }
3771         }
3772
3773         if (adev->is_atom_fw) {
3774                 /* Initialize clocks */
3775                 r = amdgpu_atomfirmware_get_clock_info(adev);
3776                 if (r) {
3777                         dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
3778                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3779                         goto failed;
3780                 }
3781         } else {
3782                 /* Initialize clocks */
3783                 r = amdgpu_atombios_get_clock_info(adev);
3784                 if (r) {
3785                         dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
3786                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
3787                         goto failed;
3788                 }
3789                 /* init i2c buses */
3790                 if (!amdgpu_device_has_dc_support(adev))
3791                         amdgpu_atombios_i2c_init(adev);
3792         }
3793
3794 fence_driver_init:
3795         /* Fence driver */
3796         r = amdgpu_fence_driver_sw_init(adev);
3797         if (r) {
3798                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
3799                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
3800                 goto failed;
3801         }
3802
3803         /* init the mode config */
3804         drm_mode_config_init(adev_to_drm(adev));
3805
3806         r = amdgpu_device_ip_init(adev);
3807         if (r) {
3808                 /* failed in exclusive mode due to timeout */
3809                 if (amdgpu_sriov_vf(adev) &&
3810                     !amdgpu_sriov_runtime(adev) &&
3811                     amdgpu_virt_mmio_blocked(adev) &&
3812                     !amdgpu_virt_wait_reset(adev)) {
3813                         dev_err(adev->dev, "VF exclusive mode timeout\n");
3814                         /* Don't send request since VF is inactive. */
3815                         adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
3816                         adev->virt.ops = NULL;
3817                         r = -EAGAIN;
3818                         goto release_ras_con;
3819                 }
3820                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
3821                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
3822                 goto release_ras_con;
3823         }
3824
3825         amdgpu_fence_driver_hw_init(adev);
3826
3827         dev_info(adev->dev,
3828                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
3829                         adev->gfx.config.max_shader_engines,
3830                         adev->gfx.config.max_sh_per_se,
3831                         adev->gfx.config.max_cu_per_sh,
3832                         adev->gfx.cu_info.number);
3833
3834         adev->accel_working = true;
3835
3836         amdgpu_vm_check_compute_bug(adev);
3837
3838         /* Initialize the buffer migration limit. */
3839         if (amdgpu_moverate >= 0)
3840                 max_MBps = amdgpu_moverate;
3841         else
3842                 max_MBps = 8; /* Allow 8 MB/s. */
3843         /* Get a log2 for easy divisions. */
3844         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
3845
3846         r = amdgpu_pm_sysfs_init(adev);
3847         if (r) {
3848                 adev->pm_sysfs_en = false;
3849                 DRM_ERROR("registering pm debugfs failed (%d).\n", r);
3850         } else
3851                 adev->pm_sysfs_en = true;
3852
3853         r = amdgpu_ucode_sysfs_init(adev);
3854         if (r) {
3855                 adev->ucode_sysfs_en = false;
3856                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
3857         } else
3858                 adev->ucode_sysfs_en = true;
3859
3860         r = amdgpu_psp_sysfs_init(adev);
3861         if (r) {
3862                 adev->psp_sysfs_en = false;
3863                 if (!amdgpu_sriov_vf(adev))
3864                         DRM_ERROR("Creating psp sysfs failed\n");
3865         } else
3866                 adev->psp_sysfs_en = true;
3867
3868         /*
3869          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
3870          * Otherwise the mgpu fan boost feature will be skipped due to the
3871          * gpu instance is counted less.
3872          */
3873         amdgpu_register_gpu_instance(adev);
3874
3875         /* enable clockgating, etc. after ib tests, etc. since some blocks require
3876          * explicit gating rather than handling it automatically.
3877          */
3878         if (!adev->gmc.xgmi.pending_reset) {
3879                 r = amdgpu_device_ip_late_init(adev);
3880                 if (r) {
3881                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
3882                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
3883                         goto release_ras_con;
3884                 }
3885                 /* must succeed. */
3886                 amdgpu_ras_resume(adev);
3887                 queue_delayed_work(system_wq, &adev->delayed_init_work,
3888                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3889         }
3890
3891         if (amdgpu_sriov_vf(adev))
3892                 flush_delayed_work(&adev->delayed_init_work);
3893
3894         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
3895         if (r)
3896                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
3897
3898         if (IS_ENABLED(CONFIG_PERF_EVENTS))
3899                 r = amdgpu_pmu_init(adev);
3900         if (r)
3901                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
3902
3903         /* Have stored pci confspace at hand for restore in sudden PCI error */
3904         if (amdgpu_device_cache_pci_state(adev->pdev))
3905                 pci_restore_state(pdev);
3906
3907         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
3908         /* this will fail for cards that aren't VGA class devices, just
3909          * ignore it */
3910         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
3911                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
3912
3913         if (amdgpu_device_supports_px(ddev)) {
3914                 px = true;
3915                 vga_switcheroo_register_client(adev->pdev,
3916                                                &amdgpu_switcheroo_ops, px);
3917                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
3918         }
3919
3920         if (adev->gmc.xgmi.pending_reset)
3921                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
3922                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
3923
3924         amdgpu_device_check_iommu_direct_map(adev);
3925
3926         return 0;
3927
3928 release_ras_con:
3929         amdgpu_release_ras_context(adev);
3930
3931 failed:
3932         amdgpu_vf_error_trans_all(adev);
3933
3934         return r;
3935 }
3936
3937 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
3938 {
3939
3940         /* Clear all CPU mappings pointing to this device */
3941         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
3942
3943         /* Unmap all mapped bars - Doorbell, registers and VRAM */
3944         amdgpu_device_doorbell_fini(adev);
3945
3946         iounmap(adev->rmmio);
3947         adev->rmmio = NULL;
3948         if (adev->mman.aper_base_kaddr)
3949                 iounmap(adev->mman.aper_base_kaddr);
3950         adev->mman.aper_base_kaddr = NULL;
3951
3952         /* Memory manager related */
3953         if (!adev->gmc.xgmi.connected_to_cpu) {
3954                 arch_phys_wc_del(adev->gmc.vram_mtrr);
3955                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
3956         }
3957 }
3958
3959 /**
3960  * amdgpu_device_fini_hw - tear down the driver
3961  *
3962  * @adev: amdgpu_device pointer
3963  *
3964  * Tear down the driver info (all asics).
3965  * Called at driver shutdown.
3966  */
3967 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
3968 {
3969         dev_info(adev->dev, "amdgpu: finishing device.\n");
3970         flush_delayed_work(&adev->delayed_init_work);
3971         adev->shutdown = true;
3972
3973         /* make sure IB test finished before entering exclusive mode
3974          * to avoid preemption on IB test
3975          * */
3976         if (amdgpu_sriov_vf(adev)) {
3977                 amdgpu_virt_request_full_gpu(adev, false);
3978                 amdgpu_virt_fini_data_exchange(adev);
3979         }
3980
3981         /* disable all interrupts */
3982         amdgpu_irq_disable_all(adev);
3983         if (adev->mode_info.mode_config_initialized){
3984                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
3985                         drm_helper_force_disable_all(adev_to_drm(adev));
3986                 else
3987                         drm_atomic_helper_shutdown(adev_to_drm(adev));
3988         }
3989         amdgpu_fence_driver_hw_fini(adev);
3990
3991         if (adev->mman.initialized) {
3992                 flush_delayed_work(&adev->mman.bdev.wq);
3993                 ttm_bo_lock_delayed_workqueue(&adev->mman.bdev);
3994         }
3995
3996         if (adev->pm_sysfs_en)
3997                 amdgpu_pm_sysfs_fini(adev);
3998         if (adev->ucode_sysfs_en)
3999                 amdgpu_ucode_sysfs_fini(adev);
4000         if (adev->psp_sysfs_en)
4001                 amdgpu_psp_sysfs_fini(adev);
4002         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4003
4004         /* disable ras feature must before hw fini */
4005         amdgpu_ras_pre_fini(adev);
4006
4007         amdgpu_device_ip_fini_early(adev);
4008
4009         amdgpu_irq_fini_hw(adev);
4010
4011         if (adev->mman.initialized)
4012                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4013
4014         amdgpu_gart_dummy_page_fini(adev);
4015
4016         amdgpu_device_unmap_mmio(adev);
4017
4018 }
4019
4020 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4021 {
4022         int idx;
4023
4024         amdgpu_fence_driver_sw_fini(adev);
4025         amdgpu_device_ip_fini(adev);
4026         release_firmware(adev->firmware.gpu_info_fw);
4027         adev->firmware.gpu_info_fw = NULL;
4028         adev->accel_working = false;
4029         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4030
4031         amdgpu_reset_fini(adev);
4032
4033         /* free i2c buses */
4034         if (!amdgpu_device_has_dc_support(adev))
4035                 amdgpu_i2c_fini(adev);
4036
4037         if (amdgpu_emu_mode != 1)
4038                 amdgpu_atombios_fini(adev);
4039
4040         kfree(adev->bios);
4041         adev->bios = NULL;
4042         if (amdgpu_device_supports_px(adev_to_drm(adev))) {
4043                 vga_switcheroo_unregister_client(adev->pdev);
4044                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4045         }
4046         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4047                 vga_client_unregister(adev->pdev);
4048
4049         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4050
4051                 iounmap(adev->rmmio);
4052                 adev->rmmio = NULL;
4053                 amdgpu_device_doorbell_fini(adev);
4054                 drm_dev_exit(idx);
4055         }
4056
4057         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4058                 amdgpu_pmu_fini(adev);
4059         if (adev->mman.discovery_bin)
4060                 amdgpu_discovery_fini(adev);
4061
4062         amdgpu_reset_put_reset_domain(adev->reset_domain);
4063         adev->reset_domain = NULL;
4064
4065         kfree(adev->pci_state);
4066
4067 }
4068
4069 /**
4070  * amdgpu_device_evict_resources - evict device resources
4071  * @adev: amdgpu device object
4072  *
4073  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4074  * of the vram memory type. Mainly used for evicting device resources
4075  * at suspend time.
4076  *
4077  */
4078 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4079 {
4080         int ret;
4081
4082         /* No need to evict vram on APUs for suspend to ram or s2idle */
4083         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4084                 return 0;
4085
4086         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4087         if (ret)
4088                 DRM_WARN("evicting device resources failed\n");
4089         return ret;
4090 }
4091
4092 /*
4093  * Suspend & resume.
4094  */
4095 /**
4096  * amdgpu_device_suspend - initiate device suspend
4097  *
4098  * @dev: drm dev pointer
4099  * @fbcon : notify the fbdev of suspend
4100  *
4101  * Puts the hw in the suspend state (all asics).
4102  * Returns 0 for success or an error on failure.
4103  * Called at driver suspend.
4104  */
4105 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4106 {
4107         struct amdgpu_device *adev = drm_to_adev(dev);
4108         int r = 0;
4109
4110         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4111                 return 0;
4112
4113         adev->in_suspend = true;
4114
4115         if (amdgpu_sriov_vf(adev)) {
4116                 amdgpu_virt_fini_data_exchange(adev);
4117                 r = amdgpu_virt_request_full_gpu(adev, false);
4118                 if (r)
4119                         return r;
4120         }
4121
4122         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4123                 DRM_WARN("smart shift update failed\n");
4124
4125         drm_kms_helper_poll_disable(dev);
4126
4127         if (fbcon)
4128                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4129
4130         cancel_delayed_work_sync(&adev->delayed_init_work);
4131
4132         amdgpu_ras_suspend(adev);
4133
4134         amdgpu_device_ip_suspend_phase1(adev);
4135
4136         if (!adev->in_s0ix)
4137                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4138
4139         r = amdgpu_device_evict_resources(adev);
4140         if (r)
4141                 return r;
4142
4143         amdgpu_fence_driver_hw_fini(adev);
4144
4145         amdgpu_device_ip_suspend_phase2(adev);
4146
4147         if (amdgpu_sriov_vf(adev))
4148                 amdgpu_virt_release_full_gpu(adev, false);
4149
4150         return 0;
4151 }
4152
4153 /**
4154  * amdgpu_device_resume - initiate device resume
4155  *
4156  * @dev: drm dev pointer
4157  * @fbcon : notify the fbdev of resume
4158  *
4159  * Bring the hw back to operating state (all asics).
4160  * Returns 0 for success or an error on failure.
4161  * Called at driver resume.
4162  */
4163 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4164 {
4165         struct amdgpu_device *adev = drm_to_adev(dev);
4166         int r = 0;
4167
4168         if (amdgpu_sriov_vf(adev)) {
4169                 r = amdgpu_virt_request_full_gpu(adev, true);
4170                 if (r)
4171                         return r;
4172         }
4173
4174         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4175                 return 0;
4176
4177         if (adev->in_s0ix)
4178                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4179
4180         /* post card */
4181         if (amdgpu_device_need_post(adev)) {
4182                 r = amdgpu_device_asic_init(adev);
4183                 if (r)
4184                         dev_err(adev->dev, "amdgpu asic init failed\n");
4185         }
4186
4187         r = amdgpu_device_ip_resume(adev);
4188
4189         if (r) {
4190                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4191                 goto exit;
4192         }
4193         amdgpu_fence_driver_hw_init(adev);
4194
4195         r = amdgpu_device_ip_late_init(adev);
4196         if (r)
4197                 goto exit;
4198
4199         queue_delayed_work(system_wq, &adev->delayed_init_work,
4200                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4201
4202         if (!adev->in_s0ix) {
4203                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4204                 if (r)
4205                         goto exit;
4206         }
4207
4208 exit:
4209         if (amdgpu_sriov_vf(adev)) {
4210                 amdgpu_virt_init_data_exchange(adev);
4211                 amdgpu_virt_release_full_gpu(adev, true);
4212         }
4213
4214         if (r)
4215                 return r;
4216
4217         /* Make sure IB tests flushed */
4218         flush_delayed_work(&adev->delayed_init_work);
4219
4220         if (adev->in_s0ix) {
4221                 /* re-enable gfxoff after IP resume. This re-enables gfxoff after
4222                  * it was disabled for IP resume in amdgpu_device_ip_resume_phase2().
4223                  */
4224                 amdgpu_gfx_off_ctrl(adev, true);
4225                 DRM_DEBUG("will enable gfxoff for the mission mode\n");
4226         }
4227         if (fbcon)
4228                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4229
4230         drm_kms_helper_poll_enable(dev);
4231
4232         amdgpu_ras_resume(adev);
4233
4234         if (adev->mode_info.num_crtc) {
4235                 /*
4236                  * Most of the connector probing functions try to acquire runtime pm
4237                  * refs to ensure that the GPU is powered on when connector polling is
4238                  * performed. Since we're calling this from a runtime PM callback,
4239                  * trying to acquire rpm refs will cause us to deadlock.
4240                  *
4241                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4242                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4243                  */
4244 #ifdef CONFIG_PM
4245                 dev->dev->power.disable_depth++;
4246 #endif
4247                 if (!adev->dc_enabled)
4248                         drm_helper_hpd_irq_event(dev);
4249                 else
4250                         drm_kms_helper_hotplug_event(dev);
4251 #ifdef CONFIG_PM
4252                 dev->dev->power.disable_depth--;
4253 #endif
4254         }
4255         adev->in_suspend = false;
4256
4257         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4258                 DRM_WARN("smart shift update failed\n");
4259
4260         return 0;
4261 }
4262
4263 /**
4264  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4265  *
4266  * @adev: amdgpu_device pointer
4267  *
4268  * The list of all the hardware IPs that make up the asic is walked and
4269  * the check_soft_reset callbacks are run.  check_soft_reset determines
4270  * if the asic is still hung or not.
4271  * Returns true if any of the IPs are still in a hung state, false if not.
4272  */
4273 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4274 {
4275         int i;
4276         bool asic_hang = false;
4277
4278         if (amdgpu_sriov_vf(adev))
4279                 return true;
4280
4281         if (amdgpu_asic_need_full_reset(adev))
4282                 return true;
4283
4284         for (i = 0; i < adev->num_ip_blocks; i++) {
4285                 if (!adev->ip_blocks[i].status.valid)
4286                         continue;
4287                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4288                         adev->ip_blocks[i].status.hang =
4289                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4290                 if (adev->ip_blocks[i].status.hang) {
4291                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4292                         asic_hang = true;
4293                 }
4294         }
4295         return asic_hang;
4296 }
4297
4298 /**
4299  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4300  *
4301  * @adev: amdgpu_device pointer
4302  *
4303  * The list of all the hardware IPs that make up the asic is walked and the
4304  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4305  * handles any IP specific hardware or software state changes that are
4306  * necessary for a soft reset to succeed.
4307  * Returns 0 on success, negative error code on failure.
4308  */
4309 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4310 {
4311         int i, r = 0;
4312
4313         for (i = 0; i < adev->num_ip_blocks; i++) {
4314                 if (!adev->ip_blocks[i].status.valid)
4315                         continue;
4316                 if (adev->ip_blocks[i].status.hang &&
4317                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4318                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4319                         if (r)
4320                                 return r;
4321                 }
4322         }
4323
4324         return 0;
4325 }
4326
4327 /**
4328  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4329  *
4330  * @adev: amdgpu_device pointer
4331  *
4332  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4333  * reset is necessary to recover.
4334  * Returns true if a full asic reset is required, false if not.
4335  */
4336 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4337 {
4338         int i;
4339
4340         if (amdgpu_asic_need_full_reset(adev))
4341                 return true;
4342
4343         for (i = 0; i < adev->num_ip_blocks; i++) {
4344                 if (!adev->ip_blocks[i].status.valid)
4345                         continue;
4346                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4347                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4348                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4349                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4350                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4351                         if (adev->ip_blocks[i].status.hang) {
4352                                 dev_info(adev->dev, "Some block need full reset!\n");
4353                                 return true;
4354                         }
4355                 }
4356         }
4357         return false;
4358 }
4359
4360 /**
4361  * amdgpu_device_ip_soft_reset - do a soft reset
4362  *
4363  * @adev: amdgpu_device pointer
4364  *
4365  * The list of all the hardware IPs that make up the asic is walked and the
4366  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4367  * IP specific hardware or software state changes that are necessary to soft
4368  * reset the IP.
4369  * Returns 0 on success, negative error code on failure.
4370  */
4371 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4372 {
4373         int i, r = 0;
4374
4375         for (i = 0; i < adev->num_ip_blocks; i++) {
4376                 if (!adev->ip_blocks[i].status.valid)
4377                         continue;
4378                 if (adev->ip_blocks[i].status.hang &&
4379                     adev->ip_blocks[i].version->funcs->soft_reset) {
4380                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4381                         if (r)
4382                                 return r;
4383                 }
4384         }
4385
4386         return 0;
4387 }
4388
4389 /**
4390  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4391  *
4392  * @adev: amdgpu_device pointer
4393  *
4394  * The list of all the hardware IPs that make up the asic is walked and the
4395  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4396  * handles any IP specific hardware or software state changes that are
4397  * necessary after the IP has been soft reset.
4398  * Returns 0 on success, negative error code on failure.
4399  */
4400 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4401 {
4402         int i, r = 0;
4403
4404         for (i = 0; i < adev->num_ip_blocks; i++) {
4405                 if (!adev->ip_blocks[i].status.valid)
4406                         continue;
4407                 if (adev->ip_blocks[i].status.hang &&
4408                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4409                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4410                 if (r)
4411                         return r;
4412         }
4413
4414         return 0;
4415 }
4416
4417 /**
4418  * amdgpu_device_recover_vram - Recover some VRAM contents
4419  *
4420  * @adev: amdgpu_device pointer
4421  *
4422  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4423  * restore things like GPUVM page tables after a GPU reset where
4424  * the contents of VRAM might be lost.
4425  *
4426  * Returns:
4427  * 0 on success, negative error code on failure.
4428  */
4429 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4430 {
4431         struct dma_fence *fence = NULL, *next = NULL;
4432         struct amdgpu_bo *shadow;
4433         struct amdgpu_bo_vm *vmbo;
4434         long r = 1, tmo;
4435
4436         if (amdgpu_sriov_runtime(adev))
4437                 tmo = msecs_to_jiffies(8000);
4438         else
4439                 tmo = msecs_to_jiffies(100);
4440
4441         dev_info(adev->dev, "recover vram bo from shadow start\n");
4442         mutex_lock(&adev->shadow_list_lock);
4443         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4444                 shadow = &vmbo->bo;
4445                 /* No need to recover an evicted BO */
4446                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4447                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4448                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4449                         continue;
4450
4451                 r = amdgpu_bo_restore_shadow(shadow, &next);
4452                 if (r)
4453                         break;
4454
4455                 if (fence) {
4456                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4457                         dma_fence_put(fence);
4458                         fence = next;
4459                         if (tmo == 0) {
4460                                 r = -ETIMEDOUT;
4461                                 break;
4462                         } else if (tmo < 0) {
4463                                 r = tmo;
4464                                 break;
4465                         }
4466                 } else {
4467                         fence = next;
4468                 }
4469         }
4470         mutex_unlock(&adev->shadow_list_lock);
4471
4472         if (fence)
4473                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4474         dma_fence_put(fence);
4475
4476         if (r < 0 || tmo <= 0) {
4477                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4478                 return -EIO;
4479         }
4480
4481         dev_info(adev->dev, "recover vram bo from shadow done\n");
4482         return 0;
4483 }
4484
4485
4486 /**
4487  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4488  *
4489  * @adev: amdgpu_device pointer
4490  * @from_hypervisor: request from hypervisor
4491  *
4492  * do VF FLR and reinitialize Asic
4493  * return 0 means succeeded otherwise failed
4494  */
4495 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4496                                      bool from_hypervisor)
4497 {
4498         int r;
4499         struct amdgpu_hive_info *hive = NULL;
4500         int retry_limit = 0;
4501
4502 retry:
4503         amdgpu_amdkfd_pre_reset(adev);
4504
4505         if (from_hypervisor)
4506                 r = amdgpu_virt_request_full_gpu(adev, true);
4507         else
4508                 r = amdgpu_virt_reset_gpu(adev);
4509         if (r)
4510                 return r;
4511
4512         /* Resume IP prior to SMC */
4513         r = amdgpu_device_ip_reinit_early_sriov(adev);
4514         if (r)
4515                 goto error;
4516
4517         amdgpu_virt_init_data_exchange(adev);
4518
4519         r = amdgpu_device_fw_loading(adev);
4520         if (r)
4521                 return r;
4522
4523         /* now we are okay to resume SMC/CP/SDMA */
4524         r = amdgpu_device_ip_reinit_late_sriov(adev);
4525         if (r)
4526                 goto error;
4527
4528         hive = amdgpu_get_xgmi_hive(adev);
4529         /* Update PSP FW topology after reset */
4530         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4531                 r = amdgpu_xgmi_update_topology(hive, adev);
4532
4533         if (hive)
4534                 amdgpu_put_xgmi_hive(hive);
4535
4536         if (!r) {
4537                 amdgpu_irq_gpu_reset_resume_helper(adev);
4538                 r = amdgpu_ib_ring_tests(adev);
4539
4540                 amdgpu_amdkfd_post_reset(adev);
4541         }
4542
4543 error:
4544         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4545                 amdgpu_inc_vram_lost(adev);
4546                 r = amdgpu_device_recover_vram(adev);
4547         }
4548         amdgpu_virt_release_full_gpu(adev, true);
4549
4550         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4551                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4552                         retry_limit++;
4553                         goto retry;
4554                 } else
4555                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4556         }
4557
4558         return r;
4559 }
4560
4561 /**
4562  * amdgpu_device_has_job_running - check if there is any job in mirror list
4563  *
4564  * @adev: amdgpu_device pointer
4565  *
4566  * check if there is any job in mirror list
4567  */
4568 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4569 {
4570         int i;
4571         struct drm_sched_job *job;
4572
4573         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4574                 struct amdgpu_ring *ring = adev->rings[i];
4575
4576                 if (!ring || !ring->sched.thread)
4577                         continue;
4578
4579                 spin_lock(&ring->sched.job_list_lock);
4580                 job = list_first_entry_or_null(&ring->sched.pending_list,
4581                                                struct drm_sched_job, list);
4582                 spin_unlock(&ring->sched.job_list_lock);
4583                 if (job)
4584                         return true;
4585         }
4586         return false;
4587 }
4588
4589 /**
4590  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4591  *
4592  * @adev: amdgpu_device pointer
4593  *
4594  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4595  * a hung GPU.
4596  */
4597 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4598 {
4599
4600         if (amdgpu_gpu_recovery == 0)
4601                 goto disabled;
4602
4603         /* Skip soft reset check in fatal error mode */
4604         if (!amdgpu_ras_is_poison_mode_supported(adev))
4605                 return true;
4606
4607         if (!amdgpu_device_ip_check_soft_reset(adev)) {
4608                 dev_info(adev->dev,"Timeout, but no hardware hang detected.\n");
4609                 return false;
4610         }
4611
4612         if (amdgpu_sriov_vf(adev))
4613                 return true;
4614
4615         if (amdgpu_gpu_recovery == -1) {
4616                 switch (adev->asic_type) {
4617 #ifdef CONFIG_DRM_AMDGPU_SI
4618                 case CHIP_VERDE:
4619                 case CHIP_TAHITI:
4620                 case CHIP_PITCAIRN:
4621                 case CHIP_OLAND:
4622                 case CHIP_HAINAN:
4623 #endif
4624 #ifdef CONFIG_DRM_AMDGPU_CIK
4625                 case CHIP_KAVERI:
4626                 case CHIP_KABINI:
4627                 case CHIP_MULLINS:
4628 #endif
4629                 case CHIP_CARRIZO:
4630                 case CHIP_STONEY:
4631                 case CHIP_CYAN_SKILLFISH:
4632                         goto disabled;
4633                 default:
4634                         break;
4635                 }
4636         }
4637
4638         return true;
4639
4640 disabled:
4641                 dev_info(adev->dev, "GPU recovery disabled.\n");
4642                 return false;
4643 }
4644
4645 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
4646 {
4647         u32 i;
4648         int ret = 0;
4649
4650         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
4651
4652         dev_info(adev->dev, "GPU mode1 reset\n");
4653
4654         /* disable BM */
4655         pci_clear_master(adev->pdev);
4656
4657         amdgpu_device_cache_pci_state(adev->pdev);
4658
4659         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
4660                 dev_info(adev->dev, "GPU smu mode1 reset\n");
4661                 ret = amdgpu_dpm_mode1_reset(adev);
4662         } else {
4663                 dev_info(adev->dev, "GPU psp mode1 reset\n");
4664                 ret = psp_gpu_reset(adev);
4665         }
4666
4667         if (ret)
4668                 dev_err(adev->dev, "GPU mode1 reset failed\n");
4669
4670         amdgpu_device_load_pci_state(adev->pdev);
4671
4672         /* wait for asic to come out of reset */
4673         for (i = 0; i < adev->usec_timeout; i++) {
4674                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
4675
4676                 if (memsize != 0xffffffff)
4677                         break;
4678                 udelay(1);
4679         }
4680
4681         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
4682         return ret;
4683 }
4684
4685 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
4686                                  struct amdgpu_reset_context *reset_context)
4687 {
4688         int i, r = 0;
4689         struct amdgpu_job *job = NULL;
4690         bool need_full_reset =
4691                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4692
4693         if (reset_context->reset_req_dev == adev)
4694                 job = reset_context->job;
4695
4696         if (amdgpu_sriov_vf(adev)) {
4697                 /* stop the data exchange thread */
4698                 amdgpu_virt_fini_data_exchange(adev);
4699         }
4700
4701         amdgpu_fence_driver_isr_toggle(adev, true);
4702
4703         /* block all schedulers and reset given job's ring */
4704         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4705                 struct amdgpu_ring *ring = adev->rings[i];
4706
4707                 if (!ring || !ring->sched.thread)
4708                         continue;
4709
4710                 /*clear job fence from fence drv to avoid force_completion
4711                  *leave NULL and vm flush fence in fence drv */
4712                 amdgpu_fence_driver_clear_job_fences(ring);
4713
4714                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
4715                 amdgpu_fence_driver_force_completion(ring);
4716         }
4717
4718         amdgpu_fence_driver_isr_toggle(adev, false);
4719
4720         if (job && job->vm)
4721                 drm_sched_increase_karma(&job->base);
4722
4723         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
4724         /* If reset handler not implemented, continue; otherwise return */
4725         if (r == -ENOSYS)
4726                 r = 0;
4727         else
4728                 return r;
4729
4730         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
4731         if (!amdgpu_sriov_vf(adev)) {
4732
4733                 if (!need_full_reset)
4734                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
4735
4736                 if (!need_full_reset && amdgpu_gpu_recovery) {
4737                         amdgpu_device_ip_pre_soft_reset(adev);
4738                         r = amdgpu_device_ip_soft_reset(adev);
4739                         amdgpu_device_ip_post_soft_reset(adev);
4740                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
4741                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
4742                                 need_full_reset = true;
4743                         }
4744                 }
4745
4746                 if (need_full_reset)
4747                         r = amdgpu_device_ip_suspend(adev);
4748                 if (need_full_reset)
4749                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4750                 else
4751                         clear_bit(AMDGPU_NEED_FULL_RESET,
4752                                   &reset_context->flags);
4753         }
4754
4755         return r;
4756 }
4757
4758 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
4759 {
4760         int i;
4761
4762         lockdep_assert_held(&adev->reset_domain->sem);
4763
4764         for (i = 0; i < adev->num_regs; i++) {
4765                 adev->reset_dump_reg_value[i] = RREG32(adev->reset_dump_reg_list[i]);
4766                 trace_amdgpu_reset_reg_dumps(adev->reset_dump_reg_list[i],
4767                                              adev->reset_dump_reg_value[i]);
4768         }
4769
4770         return 0;
4771 }
4772
4773 #ifdef CONFIG_DEV_COREDUMP
4774 static ssize_t amdgpu_devcoredump_read(char *buffer, loff_t offset,
4775                 size_t count, void *data, size_t datalen)
4776 {
4777         struct drm_printer p;
4778         struct amdgpu_device *adev = data;
4779         struct drm_print_iterator iter;
4780         int i;
4781
4782         iter.data = buffer;
4783         iter.offset = 0;
4784         iter.start = offset;
4785         iter.remain = count;
4786
4787         p = drm_coredump_printer(&iter);
4788
4789         drm_printf(&p, "**** AMDGPU Device Coredump ****\n");
4790         drm_printf(&p, "kernel: " UTS_RELEASE "\n");
4791         drm_printf(&p, "module: " KBUILD_MODNAME "\n");
4792         drm_printf(&p, "time: %lld.%09ld\n", adev->reset_time.tv_sec, adev->reset_time.tv_nsec);
4793         if (adev->reset_task_info.pid)
4794                 drm_printf(&p, "process_name: %s PID: %d\n",
4795                            adev->reset_task_info.process_name,
4796                            adev->reset_task_info.pid);
4797
4798         if (adev->reset_vram_lost)
4799                 drm_printf(&p, "VRAM is lost due to GPU reset!\n");
4800         if (adev->num_regs) {
4801                 drm_printf(&p, "AMDGPU register dumps:\nOffset:     Value:\n");
4802
4803                 for (i = 0; i < adev->num_regs; i++)
4804                         drm_printf(&p, "0x%08x: 0x%08x\n",
4805                                    adev->reset_dump_reg_list[i],
4806                                    adev->reset_dump_reg_value[i]);
4807         }
4808
4809         return count - iter.remain;
4810 }
4811
4812 static void amdgpu_devcoredump_free(void *data)
4813 {
4814 }
4815
4816 static void amdgpu_reset_capture_coredumpm(struct amdgpu_device *adev)
4817 {
4818         struct drm_device *dev = adev_to_drm(adev);
4819
4820         ktime_get_ts64(&adev->reset_time);
4821         dev_coredumpm(dev->dev, THIS_MODULE, adev, 0, GFP_KERNEL,
4822                       amdgpu_devcoredump_read, amdgpu_devcoredump_free);
4823 }
4824 #endif
4825
4826 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
4827                          struct amdgpu_reset_context *reset_context)
4828 {
4829         struct amdgpu_device *tmp_adev = NULL;
4830         bool need_full_reset, skip_hw_reset, vram_lost = false;
4831         int r = 0;
4832         bool gpu_reset_for_dev_remove = 0;
4833
4834         /* Try reset handler method first */
4835         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
4836                                     reset_list);
4837         amdgpu_reset_reg_dumps(tmp_adev);
4838
4839         reset_context->reset_device_list = device_list_handle;
4840         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
4841         /* If reset handler not implemented, continue; otherwise return */
4842         if (r == -ENOSYS)
4843                 r = 0;
4844         else
4845                 return r;
4846
4847         /* Reset handler not implemented, use the default method */
4848         need_full_reset =
4849                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4850         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
4851
4852         gpu_reset_for_dev_remove =
4853                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
4854                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
4855
4856         /*
4857          * ASIC reset has to be done on all XGMI hive nodes ASAP
4858          * to allow proper links negotiation in FW (within 1 sec)
4859          */
4860         if (!skip_hw_reset && need_full_reset) {
4861                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4862                         /* For XGMI run all resets in parallel to speed up the process */
4863                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4864                                 tmp_adev->gmc.xgmi.pending_reset = false;
4865                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
4866                                         r = -EALREADY;
4867                         } else
4868                                 r = amdgpu_asic_reset(tmp_adev);
4869
4870                         if (r) {
4871                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
4872                                          r, adev_to_drm(tmp_adev)->unique);
4873                                 break;
4874                         }
4875                 }
4876
4877                 /* For XGMI wait for all resets to complete before proceed */
4878                 if (!r) {
4879                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4880                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
4881                                         flush_work(&tmp_adev->xgmi_reset_work);
4882                                         r = tmp_adev->asic_reset_res;
4883                                         if (r)
4884                                                 break;
4885                                 }
4886                         }
4887                 }
4888         }
4889
4890         if (!r && amdgpu_ras_intr_triggered()) {
4891                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4892                         if (tmp_adev->mmhub.ras && tmp_adev->mmhub.ras->ras_block.hw_ops &&
4893                             tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count)
4894                                 tmp_adev->mmhub.ras->ras_block.hw_ops->reset_ras_error_count(tmp_adev);
4895                 }
4896
4897                 amdgpu_ras_intr_cleared();
4898         }
4899
4900         /* Since the mode1 reset affects base ip blocks, the
4901          * phase1 ip blocks need to be resumed. Otherwise there
4902          * will be a BIOS signature error and the psp bootloader
4903          * can't load kdb on the next amdgpu install.
4904          */
4905         if (gpu_reset_for_dev_remove) {
4906                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
4907                         amdgpu_device_ip_resume_phase1(tmp_adev);
4908
4909                 goto end;
4910         }
4911
4912         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
4913                 if (need_full_reset) {
4914                         /* post card */
4915                         r = amdgpu_device_asic_init(tmp_adev);
4916                         if (r) {
4917                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
4918                         } else {
4919                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
4920                                 r = amdgpu_amdkfd_resume_iommu(tmp_adev);
4921                                 if (r)
4922                                         goto out;
4923
4924                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
4925                                 if (r)
4926                                         goto out;
4927
4928                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
4929 #ifdef CONFIG_DEV_COREDUMP
4930                                 tmp_adev->reset_vram_lost = vram_lost;
4931                                 memset(&tmp_adev->reset_task_info, 0,
4932                                                 sizeof(tmp_adev->reset_task_info));
4933                                 if (reset_context->job && reset_context->job->vm)
4934                                         tmp_adev->reset_task_info =
4935                                                 reset_context->job->vm->task_info;
4936                                 amdgpu_reset_capture_coredumpm(tmp_adev);
4937 #endif
4938                                 if (vram_lost) {
4939                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
4940                                         amdgpu_inc_vram_lost(tmp_adev);
4941                                 }
4942
4943                                 r = amdgpu_device_fw_loading(tmp_adev);
4944                                 if (r)
4945                                         return r;
4946
4947                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
4948                                 if (r)
4949                                         goto out;
4950
4951                                 if (vram_lost)
4952                                         amdgpu_device_fill_reset_magic(tmp_adev);
4953
4954                                 /*
4955                                  * Add this ASIC as tracked as reset was already
4956                                  * complete successfully.
4957                                  */
4958                                 amdgpu_register_gpu_instance(tmp_adev);
4959
4960                                 if (!reset_context->hive &&
4961                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4962                                         amdgpu_xgmi_add_device(tmp_adev);
4963
4964                                 r = amdgpu_device_ip_late_init(tmp_adev);
4965                                 if (r)
4966                                         goto out;
4967
4968                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
4969
4970                                 /*
4971                                  * The GPU enters bad state once faulty pages
4972                                  * by ECC has reached the threshold, and ras
4973                                  * recovery is scheduled next. So add one check
4974                                  * here to break recovery if it indeed exceeds
4975                                  * bad page threshold, and remind user to
4976                                  * retire this GPU or setting one bigger
4977                                  * bad_page_threshold value to fix this once
4978                                  * probing driver again.
4979                                  */
4980                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
4981                                         /* must succeed. */
4982                                         amdgpu_ras_resume(tmp_adev);
4983                                 } else {
4984                                         r = -EINVAL;
4985                                         goto out;
4986                                 }
4987
4988                                 /* Update PSP FW topology after reset */
4989                                 if (reset_context->hive &&
4990                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
4991                                         r = amdgpu_xgmi_update_topology(
4992                                                 reset_context->hive, tmp_adev);
4993                         }
4994                 }
4995
4996 out:
4997                 if (!r) {
4998                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
4999                         r = amdgpu_ib_ring_tests(tmp_adev);
5000                         if (r) {
5001                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5002                                 need_full_reset = true;
5003                                 r = -EAGAIN;
5004                                 goto end;
5005                         }
5006                 }
5007
5008                 if (!r)
5009                         r = amdgpu_device_recover_vram(tmp_adev);
5010                 else
5011                         tmp_adev->asic_reset_res = r;
5012         }
5013
5014 end:
5015         if (need_full_reset)
5016                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5017         else
5018                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5019         return r;
5020 }
5021
5022 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5023 {
5024
5025         switch (amdgpu_asic_reset_method(adev)) {
5026         case AMD_RESET_METHOD_MODE1:
5027                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5028                 break;
5029         case AMD_RESET_METHOD_MODE2:
5030                 adev->mp1_state = PP_MP1_STATE_RESET;
5031                 break;
5032         default:
5033                 adev->mp1_state = PP_MP1_STATE_NONE;
5034                 break;
5035         }
5036 }
5037
5038 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5039 {
5040         amdgpu_vf_error_trans_all(adev);
5041         adev->mp1_state = PP_MP1_STATE_NONE;
5042 }
5043
5044 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5045 {
5046         struct pci_dev *p = NULL;
5047
5048         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5049                         adev->pdev->bus->number, 1);
5050         if (p) {
5051                 pm_runtime_enable(&(p->dev));
5052                 pm_runtime_resume(&(p->dev));
5053         }
5054
5055         pci_dev_put(p);
5056 }
5057
5058 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5059 {
5060         enum amd_reset_method reset_method;
5061         struct pci_dev *p = NULL;
5062         u64 expires;
5063
5064         /*
5065          * For now, only BACO and mode1 reset are confirmed
5066          * to suffer the audio issue without proper suspended.
5067          */
5068         reset_method = amdgpu_asic_reset_method(adev);
5069         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5070              (reset_method != AMD_RESET_METHOD_MODE1))
5071                 return -EINVAL;
5072
5073         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5074                         adev->pdev->bus->number, 1);
5075         if (!p)
5076                 return -ENODEV;
5077
5078         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5079         if (!expires)
5080                 /*
5081                  * If we cannot get the audio device autosuspend delay,
5082                  * a fixed 4S interval will be used. Considering 3S is
5083                  * the audio controller default autosuspend delay setting.
5084                  * 4S used here is guaranteed to cover that.
5085                  */
5086                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5087
5088         while (!pm_runtime_status_suspended(&(p->dev))) {
5089                 if (!pm_runtime_suspend(&(p->dev)))
5090                         break;
5091
5092                 if (expires < ktime_get_mono_fast_ns()) {
5093                         dev_warn(adev->dev, "failed to suspend display audio\n");
5094                         pci_dev_put(p);
5095                         /* TODO: abort the succeeding gpu reset? */
5096                         return -ETIMEDOUT;
5097                 }
5098         }
5099
5100         pm_runtime_disable(&(p->dev));
5101
5102         pci_dev_put(p);
5103         return 0;
5104 }
5105
5106 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5107 {
5108         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5109
5110 #if defined(CONFIG_DEBUG_FS)
5111         if (!amdgpu_sriov_vf(adev))
5112                 cancel_work(&adev->reset_work);
5113 #endif
5114
5115         if (adev->kfd.dev)
5116                 cancel_work(&adev->kfd.reset_work);
5117
5118         if (amdgpu_sriov_vf(adev))
5119                 cancel_work(&adev->virt.flr_work);
5120
5121         if (con && adev->ras_enabled)
5122                 cancel_work(&con->recovery_work);
5123
5124 }
5125
5126 /**
5127  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5128  *
5129  * @adev: amdgpu_device pointer
5130  * @job: which job trigger hang
5131  *
5132  * Attempt to reset the GPU if it has hung (all asics).
5133  * Attempt to do soft-reset or full-reset and reinitialize Asic
5134  * Returns 0 for success or an error on failure.
5135  */
5136
5137 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5138                               struct amdgpu_job *job,
5139                               struct amdgpu_reset_context *reset_context)
5140 {
5141         struct list_head device_list, *device_list_handle =  NULL;
5142         bool job_signaled = false;
5143         struct amdgpu_hive_info *hive = NULL;
5144         struct amdgpu_device *tmp_adev = NULL;
5145         int i, r = 0;
5146         bool need_emergency_restart = false;
5147         bool audio_suspended = false;
5148         bool gpu_reset_for_dev_remove = false;
5149
5150         gpu_reset_for_dev_remove =
5151                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5152                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5153
5154         /*
5155          * Special case: RAS triggered and full reset isn't supported
5156          */
5157         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5158
5159         /*
5160          * Flush RAM to disk so that after reboot
5161          * the user can read log and see why the system rebooted.
5162          */
5163         if (need_emergency_restart && amdgpu_ras_get_context(adev)->reboot) {
5164                 DRM_WARN("Emergency reboot.");
5165
5166                 ksys_sync_helper();
5167                 emergency_restart();
5168         }
5169
5170         dev_info(adev->dev, "GPU %s begin!\n",
5171                 need_emergency_restart ? "jobs stop":"reset");
5172
5173         if (!amdgpu_sriov_vf(adev))
5174                 hive = amdgpu_get_xgmi_hive(adev);
5175         if (hive)
5176                 mutex_lock(&hive->hive_lock);
5177
5178         reset_context->job = job;
5179         reset_context->hive = hive;
5180         /*
5181          * Build list of devices to reset.
5182          * In case we are in XGMI hive mode, resort the device list
5183          * to put adev in the 1st position.
5184          */
5185         INIT_LIST_HEAD(&device_list);
5186         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5187                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5188                         list_add_tail(&tmp_adev->reset_list, &device_list);
5189                         if (gpu_reset_for_dev_remove && adev->shutdown)
5190                                 tmp_adev->shutdown = true;
5191                 }
5192                 if (!list_is_first(&adev->reset_list, &device_list))
5193                         list_rotate_to_front(&adev->reset_list, &device_list);
5194                 device_list_handle = &device_list;
5195         } else {
5196                 list_add_tail(&adev->reset_list, &device_list);
5197                 device_list_handle = &device_list;
5198         }
5199
5200         /* We need to lock reset domain only once both for XGMI and single device */
5201         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5202                                     reset_list);
5203         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5204
5205         /* block all schedulers and reset given job's ring */
5206         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5207
5208                 amdgpu_device_set_mp1_state(tmp_adev);
5209
5210                 /*
5211                  * Try to put the audio codec into suspend state
5212                  * before gpu reset started.
5213                  *
5214                  * Due to the power domain of the graphics device
5215                  * is shared with AZ power domain. Without this,
5216                  * we may change the audio hardware from behind
5217                  * the audio driver's back. That will trigger
5218                  * some audio codec errors.
5219                  */
5220                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5221                         audio_suspended = true;
5222
5223                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5224
5225                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5226
5227                 if (!amdgpu_sriov_vf(tmp_adev))
5228                         amdgpu_amdkfd_pre_reset(tmp_adev);
5229
5230                 /*
5231                  * Mark these ASICs to be reseted as untracked first
5232                  * And add them back after reset completed
5233                  */
5234                 amdgpu_unregister_gpu_instance(tmp_adev);
5235
5236                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5237
5238                 /* disable ras on ALL IPs */
5239                 if (!need_emergency_restart &&
5240                       amdgpu_device_ip_need_full_reset(tmp_adev))
5241                         amdgpu_ras_suspend(tmp_adev);
5242
5243                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5244                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5245
5246                         if (!ring || !ring->sched.thread)
5247                                 continue;
5248
5249                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5250
5251                         if (need_emergency_restart)
5252                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5253                 }
5254                 atomic_inc(&tmp_adev->gpu_reset_counter);
5255         }
5256
5257         if (need_emergency_restart)
5258                 goto skip_sched_resume;
5259
5260         /*
5261          * Must check guilty signal here since after this point all old
5262          * HW fences are force signaled.
5263          *
5264          * job->base holds a reference to parent fence
5265          */
5266         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5267                 job_signaled = true;
5268                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5269                 goto skip_hw_reset;
5270         }
5271
5272 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5273         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5274                 if (gpu_reset_for_dev_remove) {
5275                         /* Workaroud for ASICs need to disable SMC first */
5276                         amdgpu_device_smu_fini_early(tmp_adev);
5277                 }
5278                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5279                 /*TODO Should we stop ?*/
5280                 if (r) {
5281                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5282                                   r, adev_to_drm(tmp_adev)->unique);
5283                         tmp_adev->asic_reset_res = r;
5284                 }
5285
5286                 /*
5287                  * Drop all pending non scheduler resets. Scheduler resets
5288                  * were already dropped during drm_sched_stop
5289                  */
5290                 amdgpu_device_stop_pending_resets(tmp_adev);
5291         }
5292
5293         /* Actual ASIC resets if needed.*/
5294         /* Host driver will handle XGMI hive reset for SRIOV */
5295         if (amdgpu_sriov_vf(adev)) {
5296                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5297                 if (r)
5298                         adev->asic_reset_res = r;
5299
5300                 /* Aldebaran supports ras in SRIOV, so need resume ras during reset */
5301                 if (adev->ip_versions[GC_HWIP][0] == IP_VERSION(9, 4, 2))
5302                         amdgpu_ras_resume(adev);
5303         } else {
5304                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5305                 if (r && r == -EAGAIN)
5306                         goto retry;
5307
5308                 if (!r && gpu_reset_for_dev_remove)
5309                         goto recover_end;
5310         }
5311
5312 skip_hw_reset:
5313
5314         /* Post ASIC reset for all devs .*/
5315         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5316
5317                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5318                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5319
5320                         if (!ring || !ring->sched.thread)
5321                                 continue;
5322
5323                         drm_sched_start(&ring->sched, true);
5324                 }
5325
5326                 if (adev->enable_mes && adev->ip_versions[GC_HWIP][0] != IP_VERSION(11, 0, 3))
5327                         amdgpu_mes_self_test(tmp_adev);
5328
5329                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled) {
5330                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5331                 }
5332
5333                 if (tmp_adev->asic_reset_res)
5334                         r = tmp_adev->asic_reset_res;
5335
5336                 tmp_adev->asic_reset_res = 0;
5337
5338                 if (r) {
5339                         /* bad news, how to tell it to userspace ? */
5340                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5341                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5342                 } else {
5343                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5344                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5345                                 DRM_WARN("smart shift update failed\n");
5346                 }
5347         }
5348
5349 skip_sched_resume:
5350         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5351                 /* unlock kfd: SRIOV would do it separately */
5352                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5353                         amdgpu_amdkfd_post_reset(tmp_adev);
5354
5355                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5356                  * need to bring up kfd here if it's not be initialized before
5357                  */
5358                 if (!adev->kfd.init_complete)
5359                         amdgpu_amdkfd_device_init(adev);
5360
5361                 if (audio_suspended)
5362                         amdgpu_device_resume_display_audio(tmp_adev);
5363
5364                 amdgpu_device_unset_mp1_state(tmp_adev);
5365
5366                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5367         }
5368
5369 recover_end:
5370         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5371                                             reset_list);
5372         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5373
5374         if (hive) {
5375                 mutex_unlock(&hive->hive_lock);
5376                 amdgpu_put_xgmi_hive(hive);
5377         }
5378
5379         if (r)
5380                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5381
5382         atomic_set(&adev->reset_domain->reset_res, r);
5383         return r;
5384 }
5385
5386 /**
5387  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5388  *
5389  * @adev: amdgpu_device pointer
5390  *
5391  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5392  * and lanes) of the slot the device is in. Handles APUs and
5393  * virtualized environments where PCIE config space may not be available.
5394  */
5395 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5396 {
5397         struct pci_dev *pdev;
5398         enum pci_bus_speed speed_cap, platform_speed_cap;
5399         enum pcie_link_width platform_link_width;
5400
5401         if (amdgpu_pcie_gen_cap)
5402                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5403
5404         if (amdgpu_pcie_lane_cap)
5405                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5406
5407         /* covers APUs as well */
5408         if (pci_is_root_bus(adev->pdev->bus)) {
5409                 if (adev->pm.pcie_gen_mask == 0)
5410                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5411                 if (adev->pm.pcie_mlw_mask == 0)
5412                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5413                 return;
5414         }
5415
5416         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5417                 return;
5418
5419         pcie_bandwidth_available(adev->pdev, NULL,
5420                                  &platform_speed_cap, &platform_link_width);
5421
5422         if (adev->pm.pcie_gen_mask == 0) {
5423                 /* asic caps */
5424                 pdev = adev->pdev;
5425                 speed_cap = pcie_get_speed_cap(pdev);
5426                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5427                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5428                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5429                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5430                 } else {
5431                         if (speed_cap == PCIE_SPEED_32_0GT)
5432                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5433                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5434                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5435                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5436                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5437                         else if (speed_cap == PCIE_SPEED_16_0GT)
5438                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5439                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5440                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5441                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5442                         else if (speed_cap == PCIE_SPEED_8_0GT)
5443                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5444                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5445                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5446                         else if (speed_cap == PCIE_SPEED_5_0GT)
5447                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5448                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5449                         else
5450                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5451                 }
5452                 /* platform caps */
5453                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5454                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5455                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5456                 } else {
5457                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5458                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5459                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5460                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5461                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5462                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5463                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5464                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5465                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5466                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5467                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5468                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5469                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5470                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5471                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5472                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5473                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5474                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5475                         else
5476                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5477
5478                 }
5479         }
5480         if (adev->pm.pcie_mlw_mask == 0) {
5481                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5482                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5483                 } else {
5484                         switch (platform_link_width) {
5485                         case PCIE_LNK_X32:
5486                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5487                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5488                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5489                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5490                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5491                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5492                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5493                                 break;
5494                         case PCIE_LNK_X16:
5495                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5496                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5497                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5498                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5499                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5500                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5501                                 break;
5502                         case PCIE_LNK_X12:
5503                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5504                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5505                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5506                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5507                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5508                                 break;
5509                         case PCIE_LNK_X8:
5510                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5511                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5512                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5513                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5514                                 break;
5515                         case PCIE_LNK_X4:
5516                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5517                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5518                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5519                                 break;
5520                         case PCIE_LNK_X2:
5521                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5522                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5523                                 break;
5524                         case PCIE_LNK_X1:
5525                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5526                                 break;
5527                         default:
5528                                 break;
5529                         }
5530                 }
5531         }
5532 }
5533
5534 /**
5535  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5536  *
5537  * @adev: amdgpu_device pointer
5538  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5539  *
5540  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5541  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5542  * @peer_adev.
5543  */
5544 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5545                                       struct amdgpu_device *peer_adev)
5546 {
5547 #ifdef CONFIG_HSA_AMD_P2P
5548         uint64_t address_mask = peer_adev->dev->dma_mask ?
5549                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5550         resource_size_t aper_limit =
5551                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5552         bool p2p_access =
5553                 !adev->gmc.xgmi.connected_to_cpu &&
5554                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5555
5556         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5557                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5558                 !(adev->gmc.aper_base & address_mask ||
5559                   aper_limit & address_mask));
5560 #else
5561         return false;
5562 #endif
5563 }
5564
5565 int amdgpu_device_baco_enter(struct drm_device *dev)
5566 {
5567         struct amdgpu_device *adev = drm_to_adev(dev);
5568         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5569
5570         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5571                 return -ENOTSUPP;
5572
5573         if (ras && adev->ras_enabled &&
5574             adev->nbio.funcs->enable_doorbell_interrupt)
5575                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5576
5577         return amdgpu_dpm_baco_enter(adev);
5578 }
5579
5580 int amdgpu_device_baco_exit(struct drm_device *dev)
5581 {
5582         struct amdgpu_device *adev = drm_to_adev(dev);
5583         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5584         int ret = 0;
5585
5586         if (!amdgpu_device_supports_baco(adev_to_drm(adev)))
5587                 return -ENOTSUPP;
5588
5589         ret = amdgpu_dpm_baco_exit(adev);
5590         if (ret)
5591                 return ret;
5592
5593         if (ras && adev->ras_enabled &&
5594             adev->nbio.funcs->enable_doorbell_interrupt)
5595                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5596
5597         if (amdgpu_passthrough(adev) &&
5598             adev->nbio.funcs->clear_doorbell_interrupt)
5599                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5600
5601         return 0;
5602 }
5603
5604 /**
5605  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5606  * @pdev: PCI device struct
5607  * @state: PCI channel state
5608  *
5609  * Description: Called when a PCI error is detected.
5610  *
5611  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5612  */
5613 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5614 {
5615         struct drm_device *dev = pci_get_drvdata(pdev);
5616         struct amdgpu_device *adev = drm_to_adev(dev);
5617         int i;
5618
5619         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5620
5621         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5622                 DRM_WARN("No support for XGMI hive yet...");
5623                 return PCI_ERS_RESULT_DISCONNECT;
5624         }
5625
5626         adev->pci_channel_state = state;
5627
5628         switch (state) {
5629         case pci_channel_io_normal:
5630                 return PCI_ERS_RESULT_CAN_RECOVER;
5631         /* Fatal error, prepare for slot reset */
5632         case pci_channel_io_frozen:
5633                 /*
5634                  * Locking adev->reset_domain->sem will prevent any external access
5635                  * to GPU during PCI error recovery
5636                  */
5637                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5638                 amdgpu_device_set_mp1_state(adev);
5639
5640                 /*
5641                  * Block any work scheduling as we do for regular GPU reset
5642                  * for the duration of the recovery
5643                  */
5644                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5645                         struct amdgpu_ring *ring = adev->rings[i];
5646
5647                         if (!ring || !ring->sched.thread)
5648                                 continue;
5649
5650                         drm_sched_stop(&ring->sched, NULL);
5651                 }
5652                 atomic_inc(&adev->gpu_reset_counter);
5653                 return PCI_ERS_RESULT_NEED_RESET;
5654         case pci_channel_io_perm_failure:
5655                 /* Permanent error, prepare for device removal */
5656                 return PCI_ERS_RESULT_DISCONNECT;
5657         }
5658
5659         return PCI_ERS_RESULT_NEED_RESET;
5660 }
5661
5662 /**
5663  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
5664  * @pdev: pointer to PCI device
5665  */
5666 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
5667 {
5668
5669         DRM_INFO("PCI error: mmio enabled callback!!\n");
5670
5671         /* TODO - dump whatever for debugging purposes */
5672
5673         /* This called only if amdgpu_pci_error_detected returns
5674          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
5675          * works, no need to reset slot.
5676          */
5677
5678         return PCI_ERS_RESULT_RECOVERED;
5679 }
5680
5681 /**
5682  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
5683  * @pdev: PCI device struct
5684  *
5685  * Description: This routine is called by the pci error recovery
5686  * code after the PCI slot has been reset, just before we
5687  * should resume normal operations.
5688  */
5689 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
5690 {
5691         struct drm_device *dev = pci_get_drvdata(pdev);
5692         struct amdgpu_device *adev = drm_to_adev(dev);
5693         int r, i;
5694         struct amdgpu_reset_context reset_context;
5695         u32 memsize;
5696         struct list_head device_list;
5697
5698         DRM_INFO("PCI error: slot reset callback!!\n");
5699
5700         memset(&reset_context, 0, sizeof(reset_context));
5701
5702         INIT_LIST_HEAD(&device_list);
5703         list_add_tail(&adev->reset_list, &device_list);
5704
5705         /* wait for asic to come out of reset */
5706         msleep(500);
5707
5708         /* Restore PCI confspace */
5709         amdgpu_device_load_pci_state(pdev);
5710
5711         /* confirm  ASIC came out of reset */
5712         for (i = 0; i < adev->usec_timeout; i++) {
5713                 memsize = amdgpu_asic_get_config_memsize(adev);
5714
5715                 if (memsize != 0xffffffff)
5716                         break;
5717                 udelay(1);
5718         }
5719         if (memsize == 0xffffffff) {
5720                 r = -ETIME;
5721                 goto out;
5722         }
5723
5724         reset_context.method = AMD_RESET_METHOD_NONE;
5725         reset_context.reset_req_dev = adev;
5726         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
5727         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
5728
5729         adev->no_hw_access = true;
5730         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
5731         adev->no_hw_access = false;
5732         if (r)
5733                 goto out;
5734
5735         r = amdgpu_do_asic_reset(&device_list, &reset_context);
5736
5737 out:
5738         if (!r) {
5739                 if (amdgpu_device_cache_pci_state(adev->pdev))
5740                         pci_restore_state(adev->pdev);
5741
5742                 DRM_INFO("PCIe error recovery succeeded\n");
5743         } else {
5744                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
5745                 amdgpu_device_unset_mp1_state(adev);
5746                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
5747         }
5748
5749         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
5750 }
5751
5752 /**
5753  * amdgpu_pci_resume() - resume normal ops after PCI reset
5754  * @pdev: pointer to PCI device
5755  *
5756  * Called when the error recovery driver tells us that its
5757  * OK to resume normal operation.
5758  */
5759 void amdgpu_pci_resume(struct pci_dev *pdev)
5760 {
5761         struct drm_device *dev = pci_get_drvdata(pdev);
5762         struct amdgpu_device *adev = drm_to_adev(dev);
5763         int i;
5764
5765
5766         DRM_INFO("PCI error: resume callback!!\n");
5767
5768         /* Only continue execution for the case of pci_channel_io_frozen */
5769         if (adev->pci_channel_state != pci_channel_io_frozen)
5770                 return;
5771
5772         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5773                 struct amdgpu_ring *ring = adev->rings[i];
5774
5775                 if (!ring || !ring->sched.thread)
5776                         continue;
5777
5778                 drm_sched_start(&ring->sched, true);
5779         }
5780
5781         amdgpu_device_unset_mp1_state(adev);
5782         amdgpu_device_unlock_reset_domain(adev->reset_domain);
5783 }
5784
5785 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
5786 {
5787         struct drm_device *dev = pci_get_drvdata(pdev);
5788         struct amdgpu_device *adev = drm_to_adev(dev);
5789         int r;
5790
5791         r = pci_save_state(pdev);
5792         if (!r) {
5793                 kfree(adev->pci_state);
5794
5795                 adev->pci_state = pci_store_saved_state(pdev);
5796
5797                 if (!adev->pci_state) {
5798                         DRM_ERROR("Failed to store PCI saved state");
5799                         return false;
5800                 }
5801         } else {
5802                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
5803                 return false;
5804         }
5805
5806         return true;
5807 }
5808
5809 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
5810 {
5811         struct drm_device *dev = pci_get_drvdata(pdev);
5812         struct amdgpu_device *adev = drm_to_adev(dev);
5813         int r;
5814
5815         if (!adev->pci_state)
5816                 return false;
5817
5818         r = pci_load_saved_state(pdev, adev->pci_state);
5819
5820         if (!r) {
5821                 pci_restore_state(pdev);
5822         } else {
5823                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
5824                 return false;
5825         }
5826
5827         return true;
5828 }
5829
5830 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
5831                 struct amdgpu_ring *ring)
5832 {
5833 #ifdef CONFIG_X86_64
5834         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5835                 return;
5836 #endif
5837         if (adev->gmc.xgmi.connected_to_cpu)
5838                 return;
5839
5840         if (ring && ring->funcs->emit_hdp_flush)
5841                 amdgpu_ring_emit_hdp_flush(ring);
5842         else
5843                 amdgpu_asic_flush_hdp(adev, ring);
5844 }
5845
5846 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
5847                 struct amdgpu_ring *ring)
5848 {
5849 #ifdef CONFIG_X86_64
5850         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
5851                 return;
5852 #endif
5853         if (adev->gmc.xgmi.connected_to_cpu)
5854                 return;
5855
5856         amdgpu_asic_invalidate_hdp(adev, ring);
5857 }
5858
5859 int amdgpu_in_reset(struct amdgpu_device *adev)
5860 {
5861         return atomic_read(&adev->reset_domain->in_gpu_reset);
5862         }
5863
5864 /**
5865  * amdgpu_device_halt() - bring hardware to some kind of halt state
5866  *
5867  * @adev: amdgpu_device pointer
5868  *
5869  * Bring hardware to some kind of halt state so that no one can touch it
5870  * any more. It will help to maintain error context when error occurred.
5871  * Compare to a simple hang, the system will keep stable at least for SSH
5872  * access. Then it should be trivial to inspect the hardware state and
5873  * see what's going on. Implemented as following:
5874  *
5875  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
5876  *    clears all CPU mappings to device, disallows remappings through page faults
5877  * 2. amdgpu_irq_disable_all() disables all interrupts
5878  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
5879  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
5880  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
5881  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
5882  *    flush any in flight DMA operations
5883  */
5884 void amdgpu_device_halt(struct amdgpu_device *adev)
5885 {
5886         struct pci_dev *pdev = adev->pdev;
5887         struct drm_device *ddev = adev_to_drm(adev);
5888
5889         drm_dev_unplug(ddev);
5890
5891         amdgpu_irq_disable_all(adev);
5892
5893         amdgpu_fence_driver_hw_fini(adev);
5894
5895         adev->no_hw_access = true;
5896
5897         amdgpu_device_unmap_mmio(adev);
5898
5899         pci_disable_device(pdev);
5900         pci_wait_for_pending_transaction(pdev);
5901 }
5902
5903 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
5904                                 u32 reg)
5905 {
5906         unsigned long flags, address, data;
5907         u32 r;
5908
5909         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5910         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5911
5912         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5913         WREG32(address, reg * 4);
5914         (void)RREG32(address);
5915         r = RREG32(data);
5916         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5917         return r;
5918 }
5919
5920 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
5921                                 u32 reg, u32 v)
5922 {
5923         unsigned long flags, address, data;
5924
5925         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
5926         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
5927
5928         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
5929         WREG32(address, reg * 4);
5930         (void)RREG32(address);
5931         WREG32(data, v);
5932         (void)RREG32(data);
5933         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
5934 }
5935
5936 /**
5937  * amdgpu_device_switch_gang - switch to a new gang
5938  * @adev: amdgpu_device pointer
5939  * @gang: the gang to switch to
5940  *
5941  * Try to switch to a new gang.
5942  * Returns: NULL if we switched to the new gang or a reference to the current
5943  * gang leader.
5944  */
5945 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
5946                                             struct dma_fence *gang)
5947 {
5948         struct dma_fence *old = NULL;
5949
5950         do {
5951                 dma_fence_put(old);
5952                 rcu_read_lock();
5953                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
5954                 rcu_read_unlock();
5955
5956                 if (old == gang)
5957                         break;
5958
5959                 if (!dma_fence_is_signaled(old))
5960                         return old;
5961
5962         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
5963                          old, gang) != old);
5964
5965         dma_fence_put(old);
5966         return NULL;
5967 }
5968
5969 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
5970 {
5971         switch (adev->asic_type) {
5972 #ifdef CONFIG_DRM_AMDGPU_SI
5973         case CHIP_HAINAN:
5974 #endif
5975         case CHIP_TOPAZ:
5976                 /* chips with no display hardware */
5977                 return false;
5978 #ifdef CONFIG_DRM_AMDGPU_SI
5979         case CHIP_TAHITI:
5980         case CHIP_PITCAIRN:
5981         case CHIP_VERDE:
5982         case CHIP_OLAND:
5983 #endif
5984 #ifdef CONFIG_DRM_AMDGPU_CIK
5985         case CHIP_BONAIRE:
5986         case CHIP_HAWAII:
5987         case CHIP_KAVERI:
5988         case CHIP_KABINI:
5989         case CHIP_MULLINS:
5990 #endif
5991         case CHIP_TONGA:
5992         case CHIP_FIJI:
5993         case CHIP_POLARIS10:
5994         case CHIP_POLARIS11:
5995         case CHIP_POLARIS12:
5996         case CHIP_VEGAM:
5997         case CHIP_CARRIZO:
5998         case CHIP_STONEY:
5999                 /* chips with display hardware */
6000                 return true;
6001         default:
6002                 /* IP discovery */
6003                 if (!adev->ip_versions[DCE_HWIP][0] ||
6004                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6005                         return false;
6006                 return true;
6007         }
6008 }