drivers/gpu/drm/amd/amdgpu/amdgpu_device.c

   1 /*
   2  * Copyright 2008 Advanced Micro Devices, Inc.
   3  * Copyright 2008 Red Hat Inc.
   4  * Copyright 2009 Jerome Glisse.
   5  *
   6  * Permission is hereby granted, free of charge, to any person obtaining a
   7  * copy of this software and associated documentation files (the "Software"),
   8  * to deal in the Software without restriction, including without limitation
   9  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  10  * and/or sell copies of the Software, and to permit persons to whom the
  11  * Software is furnished to do so, subject to the following conditions:
  12  *
  13  * The above copyright notice and this permission notice shall be included in
  14  * all copies or substantial portions of the Software.
  15  *
  16  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  19  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22  * OTHER DEALINGS IN THE SOFTWARE.
  23  *
  24  * Authors: Dave Airlie
  25  *          Alex Deucher
  26  *          Jerome Glisse
  27  */
  28 #include <linux/power_supply.h>
  29 #include <linux/kthread.h>
  30 #include <linux/module.h>
  31 #include <linux/console.h>
  32 #include <linux/slab.h>
  33 #include <linux/iommu.h>
  34 #include <linux/pci.h>
  35 #include <linux/pci-p2pdma.h>
  36 #include <linux/apple-gmux.h>
  37
  38 #include <drm/drm_aperture.h>
  39 #include <drm/drm_atomic_helper.h>
  40 #include <drm/drm_crtc_helper.h>
  41 #include <drm/drm_fb_helper.h>
  42 #include <drm/drm_probe_helper.h>
  43 #include <drm/amdgpu_drm.h>
  44 #include <linux/device.h>
  45 #include <linux/vgaarb.h>
  46 #include <linux/vga_switcheroo.h>
  47 #include <linux/efi.h>
  48 #include "amdgpu.h"
  49 #include "amdgpu_trace.h"
  50 #include "amdgpu_i2c.h"
  51 #include "atom.h"
  52 #include "amdgpu_atombios.h"
  53 #include "amdgpu_atomfirmware.h"
  54 #include "amd_pcie.h"
  55 #ifdef CONFIG_DRM_AMDGPU_SI
  56 #include "si.h"
  57 #endif
  58 #ifdef CONFIG_DRM_AMDGPU_CIK
  59 #include "cik.h"
  60 #endif
  61 #include "vi.h"
  62 #include "soc15.h"
  63 #include "nv.h"
  64 #include "bif/bif_4_1_d.h"
  65 #include <linux/firmware.h>
  66 #include "amdgpu_vf_error.h"
  67
  68 #include "amdgpu_amdkfd.h"
  69 #include "amdgpu_pm.h"
  70
  71 #include "amdgpu_xgmi.h"
  72 #include "amdgpu_ras.h"
  73 #include "amdgpu_pmu.h"
  74 #include "amdgpu_fru_eeprom.h"
  75 #include "amdgpu_reset.h"
  76 #include "amdgpu_virt.h"
  77
  78 #include <linux/suspend.h>
  79 #include <drm/task_barrier.h>
  80 #include <linux/pm_runtime.h>
  81
  82 #include <drm/drm_drv.h>
  83
  84 #if IS_ENABLED(CONFIG_X86)
  85 #include <asm/intel-family.h>
  86 #endif
  87
  88 MODULE_FIRMWARE("amdgpu/vega10_gpu_info.bin");
  89 MODULE_FIRMWARE("amdgpu/vega12_gpu_info.bin");
  90 MODULE_FIRMWARE("amdgpu/raven_gpu_info.bin");
  91 MODULE_FIRMWARE("amdgpu/picasso_gpu_info.bin");
  92 MODULE_FIRMWARE("amdgpu/raven2_gpu_info.bin");
  93 MODULE_FIRMWARE("amdgpu/arcturus_gpu_info.bin");
  94 MODULE_FIRMWARE("amdgpu/navi12_gpu_info.bin");
  95
  96 #define AMDGPU_RESUME_MS                2000
  97 #define AMDGPU_MAX_RETRY_LIMIT          2
  98 #define AMDGPU_RETRY_SRIOV_RESET(r) ((r) == -EBUSY || (r) == -ETIMEDOUT || (r) == -EINVAL)
  99
 100 static const struct drm_driver amdgpu_kms_driver;
 101
 102 const char *amdgpu_asic_name[] = {
 103         "TAHITI",
 104         "PITCAIRN",
 105         "VERDE",
 106         "OLAND",
 107         "HAINAN",
 108         "BONAIRE",
 109         "KAVERI",
 110         "KABINI",
 111         "HAWAII",
 112         "MULLINS",
 113         "TOPAZ",
 114         "TONGA",
 115         "FIJI",
 116         "CARRIZO",
 117         "STONEY",
 118         "POLARIS10",
 119         "POLARIS11",
 120         "POLARIS12",
 121         "VEGAM",
 122         "VEGA10",
 123         "VEGA12",
 124         "VEGA20",
 125         "RAVEN",
 126         "ARCTURUS",
 127         "RENOIR",
 128         "ALDEBARAN",
 129         "NAVI10",
 130         "CYAN_SKILLFISH",
 131         "NAVI14",
 132         "NAVI12",
 133         "SIENNA_CICHLID",
 134         "NAVY_FLOUNDER",
 135         "VANGOGH",
 136         "DIMGREY_CAVEFISH",
 137         "BEIGE_GOBY",
 138         "YELLOW_CARP",
 139         "IP DISCOVERY",
 140         "LAST",
 141 };
 142
 143 /**
 144  * DOC: pcie_replay_count
 145  *
 146  * The amdgpu driver provides a sysfs API for reporting the total number
 147  * of PCIe replays (NAKs)
 148  * The file pcie_replay_count is used for this and returns the total
 149  * number of replays as a sum of the NAKs generated and NAKs received
 150  */
 151
 152 static ssize_t amdgpu_device_get_pcie_replay_count(struct device *dev,
 153                 struct device_attribute *attr, char *buf)
 154 {
 155         struct drm_device *ddev = dev_get_drvdata(dev);
 156         struct amdgpu_device *adev = drm_to_adev(ddev);
 157         uint64_t cnt = amdgpu_asic_get_pcie_replay_count(adev);
 158
 159         return sysfs_emit(buf, "%llu\n", cnt);
 160 }
 161
 162 static DEVICE_ATTR(pcie_replay_count, 0444,
 163                 amdgpu_device_get_pcie_replay_count, NULL);
 164
 165 /**
 166  * DOC: board_info
 167  *
 168  * The amdgpu driver provides a sysfs API for giving board related information.
 169  * It provides the form factor information in the format
 170  *
 171  *   type : form factor
 172  *
 173  * Possible form factor values
 174  *
 175  * - "cem"              - PCIE CEM card
 176  * - "oam"              - Open Compute Accelerator Module
 177  * - "unknown"  - Not known
 178  *
 179  */
 180
 181 static ssize_t amdgpu_device_get_board_info(struct device *dev,
 182                                             struct device_attribute *attr,
 183                                             char *buf)
 184 {
 185         struct drm_device *ddev = dev_get_drvdata(dev);
 186         struct amdgpu_device *adev = drm_to_adev(ddev);
 187         enum amdgpu_pkg_type pkg_type = AMDGPU_PKG_TYPE_CEM;
 188         const char *pkg;
 189
 190         if (adev->smuio.funcs && adev->smuio.funcs->get_pkg_type)
 191                 pkg_type = adev->smuio.funcs->get_pkg_type(adev);
 192
 193         switch (pkg_type) {
 194         case AMDGPU_PKG_TYPE_CEM:
 195                 pkg = "cem";
 196                 break;
 197         case AMDGPU_PKG_TYPE_OAM:
 198                 pkg = "oam";
 199                 break;
 200         default:
 201                 pkg = "unknown";
 202                 break;
 203         }
 204
 205         return sysfs_emit(buf, "%s : %s\n", "type", pkg);
 206 }
 207
 208 static DEVICE_ATTR(board_info, 0444, amdgpu_device_get_board_info, NULL);
 209
 210 static struct attribute *amdgpu_board_attrs[] = {
 211         &dev_attr_board_info.attr,
 212         NULL,
 213 };
 214
 215 static umode_t amdgpu_board_attrs_is_visible(struct kobject *kobj,
 216                                              struct attribute *attr, int n)
 217 {
 218         struct device *dev = kobj_to_dev(kobj);
 219         struct drm_device *ddev = dev_get_drvdata(dev);
 220         struct amdgpu_device *adev = drm_to_adev(ddev);
 221
 222         if (adev->flags & AMD_IS_APU)
 223                 return 0;
 224
 225         return attr->mode;
 226 }
 227
 228 static const struct attribute_group amdgpu_board_attrs_group = {
 229         .attrs = amdgpu_board_attrs,
 230         .is_visible = amdgpu_board_attrs_is_visible
 231 };
 232
 233 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev);
 234
 235
 236 /**
 237  * amdgpu_device_supports_px - Is the device a dGPU with ATPX power control
 238  *
 239  * @dev: drm_device pointer
 240  *
 241  * Returns true if the device is a dGPU with ATPX power control,
 242  * otherwise return false.
 243  */
 244 bool amdgpu_device_supports_px(struct drm_device *dev)
 245 {
 246         struct amdgpu_device *adev = drm_to_adev(dev);
 247
 248         if ((adev->flags & AMD_IS_PX) && !amdgpu_is_atpx_hybrid())
 249                 return true;
 250         return false;
 251 }
 252
 253 /**
 254  * amdgpu_device_supports_boco - Is the device a dGPU with ACPI power resources
 255  *
 256  * @dev: drm_device pointer
 257  *
 258  * Returns true if the device is a dGPU with ACPI power control,
 259  * otherwise return false.
 260  */
 261 bool amdgpu_device_supports_boco(struct drm_device *dev)
 262 {
 263         struct amdgpu_device *adev = drm_to_adev(dev);
 264
 265         if (adev->has_pr3 ||
 266             ((adev->flags & AMD_IS_PX) && amdgpu_is_atpx_hybrid()))
 267                 return true;
 268         return false;
 269 }
 270
 271 /**
 272  * amdgpu_device_supports_baco - Does the device support BACO
 273  *
 274  * @dev: drm_device pointer
 275  *
 276  * Returns true if the device supporte BACO,
 277  * otherwise return false.
 278  */
 279 bool amdgpu_device_supports_baco(struct drm_device *dev)
 280 {
 281         struct amdgpu_device *adev = drm_to_adev(dev);
 282
 283         return amdgpu_asic_supports_baco(adev);
 284 }
 285
 286 /**
 287  * amdgpu_device_supports_smart_shift - Is the device dGPU with
 288  * smart shift support
 289  *
 290  * @dev: drm_device pointer
 291  *
 292  * Returns true if the device is a dGPU with Smart Shift support,
 293  * otherwise returns false.
 294  */
 295 bool amdgpu_device_supports_smart_shift(struct drm_device *dev)
 296 {
 297         return (amdgpu_device_supports_boco(dev) &&
 298                 amdgpu_acpi_is_power_shift_control_supported());
 299 }
 300
 301 /*
 302  * VRAM access helper functions
 303  */
 304
 305 /**
 306  * amdgpu_device_mm_access - access vram by MM_INDEX/MM_DATA
 307  *
 308  * @adev: amdgpu_device pointer
 309  * @pos: offset of the buffer in vram
 310  * @buf: virtual address of the buffer in system memory
 311  * @size: read/write size, sizeof(@buf) must > @size
 312  * @write: true - write to vram, otherwise - read from vram
 313  */
 314 void amdgpu_device_mm_access(struct amdgpu_device *adev, loff_t pos,
 315                              void *buf, size_t size, bool write)
 316 {
 317         unsigned long flags;
 318         uint32_t hi = ~0, tmp = 0;
 319         uint32_t *data = buf;
 320         uint64_t last;
 321         int idx;
 322
 323         if (!drm_dev_enter(adev_to_drm(adev), &idx))
 324                 return;
 325
 326         BUG_ON(!IS_ALIGNED(pos, 4) || !IS_ALIGNED(size, 4));
 327
 328         spin_lock_irqsave(&adev->mmio_idx_lock, flags);
 329         for (last = pos + size; pos < last; pos += 4) {
 330                 tmp = pos >> 31;
 331
 332                 WREG32_NO_KIQ(mmMM_INDEX, ((uint32_t)pos) | 0x80000000);
 333                 if (tmp != hi) {
 334                         WREG32_NO_KIQ(mmMM_INDEX_HI, tmp);
 335                         hi = tmp;
 336                 }
 337                 if (write)
 338                         WREG32_NO_KIQ(mmMM_DATA, *data++);
 339                 else
 340                         *data++ = RREG32_NO_KIQ(mmMM_DATA);
 341         }
 342
 343         spin_unlock_irqrestore(&adev->mmio_idx_lock, flags);
 344         drm_dev_exit(idx);
 345 }
 346
 347 /**
 348  * amdgpu_device_aper_access - access vram by vram aperature
 349  *
 350  * @adev: amdgpu_device pointer
 351  * @pos: offset of the buffer in vram
 352  * @buf: virtual address of the buffer in system memory
 353  * @size: read/write size, sizeof(@buf) must > @size
 354  * @write: true - write to vram, otherwise - read from vram
 355  *
 356  * The return value means how many bytes have been transferred.
 357  */
 358 size_t amdgpu_device_aper_access(struct amdgpu_device *adev, loff_t pos,
 359                                  void *buf, size_t size, bool write)
 360 {
 361 #ifdef CONFIG_64BIT
 362         void __iomem *addr;
 363         size_t count = 0;
 364         uint64_t last;
 365
 366         if (!adev->mman.aper_base_kaddr)
 367                 return 0;
 368
 369         last = min(pos + size, adev->gmc.visible_vram_size);
 370         if (last > pos) {
 371                 addr = adev->mman.aper_base_kaddr + pos;
 372                 count = last - pos;
 373
 374                 if (write) {
 375                         memcpy_toio(addr, buf, count);
 376                         /* Make sure HDP write cache flush happens without any reordering
 377                          * after the system memory contents are sent over PCIe device
 378                          */
 379                         mb();
 380                         amdgpu_device_flush_hdp(adev, NULL);
 381                 } else {
 382                         amdgpu_device_invalidate_hdp(adev, NULL);
 383                         /* Make sure HDP read cache is invalidated before issuing a read
 384                          * to the PCIe device
 385                          */
 386                         mb();
 387                         memcpy_fromio(buf, addr, count);
 388                 }
 389
 390         }
 391
 392         return count;
 393 #else
 394         return 0;
 395 #endif
 396 }
 397
 398 /**
 399  * amdgpu_device_vram_access - read/write a buffer in vram
 400  *
 401  * @adev: amdgpu_device pointer
 402  * @pos: offset of the buffer in vram
 403  * @buf: virtual address of the buffer in system memory
 404  * @size: read/write size, sizeof(@buf) must > @size
 405  * @write: true - write to vram, otherwise - read from vram
 406  */
 407 void amdgpu_device_vram_access(struct amdgpu_device *adev, loff_t pos,
 408                                void *buf, size_t size, bool write)
 409 {
 410         size_t count;
 411
 412         /* try to using vram apreature to access vram first */
 413         count = amdgpu_device_aper_access(adev, pos, buf, size, write);
 414         size -= count;
 415         if (size) {
 416                 /* using MM to access rest vram */
 417                 pos += count;
 418                 buf += count;
 419                 amdgpu_device_mm_access(adev, pos, buf, size, write);
 420         }
 421 }
 422
 423 /*
 424  * register access helper functions.
 425  */
 426
 427 /* Check if hw access should be skipped because of hotplug or device error */
 428 bool amdgpu_device_skip_hw_access(struct amdgpu_device *adev)
 429 {
 430         if (adev->no_hw_access)
 431                 return true;
 432
 433 #ifdef CONFIG_LOCKDEP
 434         /*
 435          * This is a bit complicated to understand, so worth a comment. What we assert
 436          * here is that the GPU reset is not running on another thread in parallel.
 437          *
 438          * For this we trylock the read side of the reset semaphore, if that succeeds
 439          * we know that the reset is not running in paralell.
 440          *
 441          * If the trylock fails we assert that we are either already holding the read
 442          * side of the lock or are the reset thread itself and hold the write side of
 443          * the lock.
 444          */
 445         if (in_task()) {
 446                 if (down_read_trylock(&adev->reset_domain->sem))
 447                         up_read(&adev->reset_domain->sem);
 448                 else
 449                         lockdep_assert_held(&adev->reset_domain->sem);
 450         }
 451 #endif
 452         return false;
 453 }
 454
 455 /**
 456  * amdgpu_device_rreg - read a memory mapped IO or indirect register
 457  *
 458  * @adev: amdgpu_device pointer
 459  * @reg: dword aligned register offset
 460  * @acc_flags: access flags which require special behavior
 461  *
 462  * Returns the 32 bit value from the offset specified.
 463  */
 464 uint32_t amdgpu_device_rreg(struct amdgpu_device *adev,
 465                             uint32_t reg, uint32_t acc_flags)
 466 {
 467         uint32_t ret;
 468
 469         if (amdgpu_device_skip_hw_access(adev))
 470                 return 0;
 471
 472         if ((reg * 4) < adev->rmmio_size) {
 473                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 474                     amdgpu_sriov_runtime(adev) &&
 475                     down_read_trylock(&adev->reset_domain->sem)) {
 476                         ret = amdgpu_kiq_rreg(adev, reg, 0);
 477                         up_read(&adev->reset_domain->sem);
 478                 } else {
 479                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 480                 }
 481         } else {
 482                 ret = adev->pcie_rreg(adev, reg * 4);
 483         }
 484
 485         trace_amdgpu_device_rreg(adev->pdev->device, reg, ret);
 486
 487         return ret;
 488 }
 489
 490 /*
 491  * MMIO register read with bytes helper functions
 492  * @offset:bytes offset from MMIO start
 493  */
 494
 495 /**
 496  * amdgpu_mm_rreg8 - read a memory mapped IO register
 497  *
 498  * @adev: amdgpu_device pointer
 499  * @offset: byte aligned register offset
 500  *
 501  * Returns the 8 bit value from the offset specified.
 502  */
 503 uint8_t amdgpu_mm_rreg8(struct amdgpu_device *adev, uint32_t offset)
 504 {
 505         if (amdgpu_device_skip_hw_access(adev))
 506                 return 0;
 507
 508         if (offset < adev->rmmio_size)
 509                 return (readb(adev->rmmio + offset));
 510         BUG();
 511 }
 512
 513
 514 /**
 515  * amdgpu_device_xcc_rreg - read a memory mapped IO or indirect register with specific XCC
 516  *
 517  * @adev: amdgpu_device pointer
 518  * @reg: dword aligned register offset
 519  * @acc_flags: access flags which require special behavior
 520  * @xcc_id: xcc accelerated compute core id
 521  *
 522  * Returns the 32 bit value from the offset specified.
 523  */
 524 uint32_t amdgpu_device_xcc_rreg(struct amdgpu_device *adev,
 525                                 uint32_t reg, uint32_t acc_flags,
 526                                 uint32_t xcc_id)
 527 {
 528         uint32_t ret, rlcg_flag;
 529
 530         if (amdgpu_device_skip_hw_access(adev))
 531                 return 0;
 532
 533         if ((reg * 4) < adev->rmmio_size) {
 534                 if (amdgpu_sriov_vf(adev) &&
 535                     !amdgpu_sriov_runtime(adev) &&
 536                     adev->gfx.rlc.rlcg_reg_access_supported &&
 537                     amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
 538                                                          GC_HWIP, false,
 539                                                          &rlcg_flag)) {
 540                         ret = amdgpu_virt_rlcg_reg_rw(adev, reg, 0, rlcg_flag, xcc_id);
 541                 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 542                     amdgpu_sriov_runtime(adev) &&
 543                     down_read_trylock(&adev->reset_domain->sem)) {
 544                         ret = amdgpu_kiq_rreg(adev, reg, xcc_id);
 545                         up_read(&adev->reset_domain->sem);
 546                 } else {
 547                         ret = readl(((void __iomem *)adev->rmmio) + (reg * 4));
 548                 }
 549         } else {
 550                 ret = adev->pcie_rreg(adev, reg * 4);
 551         }
 552
 553         return ret;
 554 }
 555
 556 /*
 557  * MMIO register write with bytes helper functions
 558  * @offset:bytes offset from MMIO start
 559  * @value: the value want to be written to the register
 560  */
 561
 562 /**
 563  * amdgpu_mm_wreg8 - read a memory mapped IO register
 564  *
 565  * @adev: amdgpu_device pointer
 566  * @offset: byte aligned register offset
 567  * @value: 8 bit value to write
 568  *
 569  * Writes the value specified to the offset specified.
 570  */
 571 void amdgpu_mm_wreg8(struct amdgpu_device *adev, uint32_t offset, uint8_t value)
 572 {
 573         if (amdgpu_device_skip_hw_access(adev))
 574                 return;
 575
 576         if (offset < adev->rmmio_size)
 577                 writeb(value, adev->rmmio + offset);
 578         else
 579                 BUG();
 580 }
 581
 582 /**
 583  * amdgpu_device_wreg - write to a memory mapped IO or indirect register
 584  *
 585  * @adev: amdgpu_device pointer
 586  * @reg: dword aligned register offset
 587  * @v: 32 bit value to write to the register
 588  * @acc_flags: access flags which require special behavior
 589  *
 590  * Writes the value specified to the offset specified.
 591  */
 592 void amdgpu_device_wreg(struct amdgpu_device *adev,
 593                         uint32_t reg, uint32_t v,
 594                         uint32_t acc_flags)
 595 {
 596         if (amdgpu_device_skip_hw_access(adev))
 597                 return;
 598
 599         if ((reg * 4) < adev->rmmio_size) {
 600                 if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 601                     amdgpu_sriov_runtime(adev) &&
 602                     down_read_trylock(&adev->reset_domain->sem)) {
 603                         amdgpu_kiq_wreg(adev, reg, v, 0);
 604                         up_read(&adev->reset_domain->sem);
 605                 } else {
 606                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 607                 }
 608         } else {
 609                 adev->pcie_wreg(adev, reg * 4, v);
 610         }
 611
 612         trace_amdgpu_device_wreg(adev->pdev->device, reg, v);
 613 }
 614
 615 /**
 616  * amdgpu_mm_wreg_mmio_rlc -  write register either with direct/indirect mmio or with RLC path if in range
 617  *
 618  * @adev: amdgpu_device pointer
 619  * @reg: mmio/rlc register
 620  * @v: value to write
 621  * @xcc_id: xcc accelerated compute core id
 622  *
 623  * this function is invoked only for the debugfs register access
 624  */
 625 void amdgpu_mm_wreg_mmio_rlc(struct amdgpu_device *adev,
 626                              uint32_t reg, uint32_t v,
 627                              uint32_t xcc_id)
 628 {
 629         if (amdgpu_device_skip_hw_access(adev))
 630                 return;
 631
 632         if (amdgpu_sriov_fullaccess(adev) &&
 633             adev->gfx.rlc.funcs &&
 634             adev->gfx.rlc.funcs->is_rlcg_access_range) {
 635                 if (adev->gfx.rlc.funcs->is_rlcg_access_range(adev, reg))
 636                         return amdgpu_sriov_wreg(adev, reg, v, 0, 0, xcc_id);
 637         } else if ((reg * 4) >= adev->rmmio_size) {
 638                 adev->pcie_wreg(adev, reg * 4, v);
 639         } else {
 640                 writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 641         }
 642 }
 643
 644 /**
 645  * amdgpu_device_xcc_wreg - write to a memory mapped IO or indirect register with specific XCC
 646  *
 647  * @adev: amdgpu_device pointer
 648  * @reg: dword aligned register offset
 649  * @v: 32 bit value to write to the register
 650  * @acc_flags: access flags which require special behavior
 651  * @xcc_id: xcc accelerated compute core id
 652  *
 653  * Writes the value specified to the offset specified.
 654  */
 655 void amdgpu_device_xcc_wreg(struct amdgpu_device *adev,
 656                         uint32_t reg, uint32_t v,
 657                         uint32_t acc_flags, uint32_t xcc_id)
 658 {
 659         uint32_t rlcg_flag;
 660
 661         if (amdgpu_device_skip_hw_access(adev))
 662                 return;
 663
 664         if ((reg * 4) < adev->rmmio_size) {
 665                 if (amdgpu_sriov_vf(adev) &&
 666                     !amdgpu_sriov_runtime(adev) &&
 667                     adev->gfx.rlc.rlcg_reg_access_supported &&
 668                     amdgpu_virt_get_rlcg_reg_access_flag(adev, acc_flags,
 669                                                          GC_HWIP, true,
 670                                                          &rlcg_flag)) {
 671                         amdgpu_virt_rlcg_reg_rw(adev, reg, v, rlcg_flag, xcc_id);
 672                 } else if (!(acc_flags & AMDGPU_REGS_NO_KIQ) &&
 673                     amdgpu_sriov_runtime(adev) &&
 674                     down_read_trylock(&adev->reset_domain->sem)) {
 675                         amdgpu_kiq_wreg(adev, reg, v, xcc_id);
 676                         up_read(&adev->reset_domain->sem);
 677                 } else {
 678                         writel(v, ((void __iomem *)adev->rmmio) + (reg * 4));
 679                 }
 680         } else {
 681                 adev->pcie_wreg(adev, reg * 4, v);
 682         }
 683 }
 684
 685 /**
 686  * amdgpu_device_indirect_rreg - read an indirect register
 687  *
 688  * @adev: amdgpu_device pointer
 689  * @reg_addr: indirect register address to read from
 690  *
 691  * Returns the value of indirect register @reg_addr
 692  */
 693 u32 amdgpu_device_indirect_rreg(struct amdgpu_device *adev,
 694                                 u32 reg_addr)
 695 {
 696         unsigned long flags, pcie_index, pcie_data;
 697         void __iomem *pcie_index_offset;
 698         void __iomem *pcie_data_offset;
 699         u32 r;
 700
 701         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 702         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 703
 704         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 705         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 706         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 707
 708         writel(reg_addr, pcie_index_offset);
 709         readl(pcie_index_offset);
 710         r = readl(pcie_data_offset);
 711         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 712
 713         return r;
 714 }
 715
 716 u32 amdgpu_device_indirect_rreg_ext(struct amdgpu_device *adev,
 717                                     u64 reg_addr)
 718 {
 719         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 720         u32 r;
 721         void __iomem *pcie_index_offset;
 722         void __iomem *pcie_index_hi_offset;
 723         void __iomem *pcie_data_offset;
 724
 725         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 726         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 727         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 728                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 729         else
 730                 pcie_index_hi = 0;
 731
 732         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 733         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 734         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 735         if (pcie_index_hi != 0)
 736                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 737                                 pcie_index_hi * 4;
 738
 739         writel(reg_addr, pcie_index_offset);
 740         readl(pcie_index_offset);
 741         if (pcie_index_hi != 0) {
 742                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 743                 readl(pcie_index_hi_offset);
 744         }
 745         r = readl(pcie_data_offset);
 746
 747         /* clear the high bits */
 748         if (pcie_index_hi != 0) {
 749                 writel(0, pcie_index_hi_offset);
 750                 readl(pcie_index_hi_offset);
 751         }
 752
 753         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 754
 755         return r;
 756 }
 757
 758 /**
 759  * amdgpu_device_indirect_rreg64 - read a 64bits indirect register
 760  *
 761  * @adev: amdgpu_device pointer
 762  * @reg_addr: indirect register address to read from
 763  *
 764  * Returns the value of indirect register @reg_addr
 765  */
 766 u64 amdgpu_device_indirect_rreg64(struct amdgpu_device *adev,
 767                                   u32 reg_addr)
 768 {
 769         unsigned long flags, pcie_index, pcie_data;
 770         void __iomem *pcie_index_offset;
 771         void __iomem *pcie_data_offset;
 772         u64 r;
 773
 774         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 775         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 776
 777         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 778         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 779         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 780
 781         /* read low 32 bits */
 782         writel(reg_addr, pcie_index_offset);
 783         readl(pcie_index_offset);
 784         r = readl(pcie_data_offset);
 785         /* read high 32 bits */
 786         writel(reg_addr + 4, pcie_index_offset);
 787         readl(pcie_index_offset);
 788         r |= ((u64)readl(pcie_data_offset) << 32);
 789         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 790
 791         return r;
 792 }
 793
 794 u64 amdgpu_device_indirect_rreg64_ext(struct amdgpu_device *adev,
 795                                   u64 reg_addr)
 796 {
 797         unsigned long flags, pcie_index, pcie_data;
 798         unsigned long pcie_index_hi = 0;
 799         void __iomem *pcie_index_offset;
 800         void __iomem *pcie_index_hi_offset;
 801         void __iomem *pcie_data_offset;
 802         u64 r;
 803
 804         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 805         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 806         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 807                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 808
 809         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 810         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 811         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 812         if (pcie_index_hi != 0)
 813                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 814                         pcie_index_hi * 4;
 815
 816         /* read low 32 bits */
 817         writel(reg_addr, pcie_index_offset);
 818         readl(pcie_index_offset);
 819         if (pcie_index_hi != 0) {
 820                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 821                 readl(pcie_index_hi_offset);
 822         }
 823         r = readl(pcie_data_offset);
 824         /* read high 32 bits */
 825         writel(reg_addr + 4, pcie_index_offset);
 826         readl(pcie_index_offset);
 827         if (pcie_index_hi != 0) {
 828                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 829                 readl(pcie_index_hi_offset);
 830         }
 831         r |= ((u64)readl(pcie_data_offset) << 32);
 832
 833         /* clear the high bits */
 834         if (pcie_index_hi != 0) {
 835                 writel(0, pcie_index_hi_offset);
 836                 readl(pcie_index_hi_offset);
 837         }
 838
 839         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 840
 841         return r;
 842 }
 843
 844 /**
 845  * amdgpu_device_indirect_wreg - write an indirect register address
 846  *
 847  * @adev: amdgpu_device pointer
 848  * @reg_addr: indirect register offset
 849  * @reg_data: indirect register data
 850  *
 851  */
 852 void amdgpu_device_indirect_wreg(struct amdgpu_device *adev,
 853                                  u32 reg_addr, u32 reg_data)
 854 {
 855         unsigned long flags, pcie_index, pcie_data;
 856         void __iomem *pcie_index_offset;
 857         void __iomem *pcie_data_offset;
 858
 859         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 860         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 861
 862         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 863         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 864         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 865
 866         writel(reg_addr, pcie_index_offset);
 867         readl(pcie_index_offset);
 868         writel(reg_data, pcie_data_offset);
 869         readl(pcie_data_offset);
 870         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 871 }
 872
 873 void amdgpu_device_indirect_wreg_ext(struct amdgpu_device *adev,
 874                                      u64 reg_addr, u32 reg_data)
 875 {
 876         unsigned long flags, pcie_index, pcie_index_hi, pcie_data;
 877         void __iomem *pcie_index_offset;
 878         void __iomem *pcie_index_hi_offset;
 879         void __iomem *pcie_data_offset;
 880
 881         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 882         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 883         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 884                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 885         else
 886                 pcie_index_hi = 0;
 887
 888         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 889         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 890         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 891         if (pcie_index_hi != 0)
 892                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 893                                 pcie_index_hi * 4;
 894
 895         writel(reg_addr, pcie_index_offset);
 896         readl(pcie_index_offset);
 897         if (pcie_index_hi != 0) {
 898                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 899                 readl(pcie_index_hi_offset);
 900         }
 901         writel(reg_data, pcie_data_offset);
 902         readl(pcie_data_offset);
 903
 904         /* clear the high bits */
 905         if (pcie_index_hi != 0) {
 906                 writel(0, pcie_index_hi_offset);
 907                 readl(pcie_index_hi_offset);
 908         }
 909
 910         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 911 }
 912
 913 /**
 914  * amdgpu_device_indirect_wreg64 - write a 64bits indirect register address
 915  *
 916  * @adev: amdgpu_device pointer
 917  * @reg_addr: indirect register offset
 918  * @reg_data: indirect register data
 919  *
 920  */
 921 void amdgpu_device_indirect_wreg64(struct amdgpu_device *adev,
 922                                    u32 reg_addr, u64 reg_data)
 923 {
 924         unsigned long flags, pcie_index, pcie_data;
 925         void __iomem *pcie_index_offset;
 926         void __iomem *pcie_data_offset;
 927
 928         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 929         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 930
 931         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 932         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 933         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 934
 935         /* write low 32 bits */
 936         writel(reg_addr, pcie_index_offset);
 937         readl(pcie_index_offset);
 938         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 939         readl(pcie_data_offset);
 940         /* write high 32 bits */
 941         writel(reg_addr + 4, pcie_index_offset);
 942         readl(pcie_index_offset);
 943         writel((u32)(reg_data >> 32), pcie_data_offset);
 944         readl(pcie_data_offset);
 945         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 946 }
 947
 948 void amdgpu_device_indirect_wreg64_ext(struct amdgpu_device *adev,
 949                                    u64 reg_addr, u64 reg_data)
 950 {
 951         unsigned long flags, pcie_index, pcie_data;
 952         unsigned long pcie_index_hi = 0;
 953         void __iomem *pcie_index_offset;
 954         void __iomem *pcie_index_hi_offset;
 955         void __iomem *pcie_data_offset;
 956
 957         pcie_index = adev->nbio.funcs->get_pcie_index_offset(adev);
 958         pcie_data = adev->nbio.funcs->get_pcie_data_offset(adev);
 959         if ((reg_addr >> 32) && (adev->nbio.funcs->get_pcie_index_hi_offset))
 960                 pcie_index_hi = adev->nbio.funcs->get_pcie_index_hi_offset(adev);
 961
 962         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
 963         pcie_index_offset = (void __iomem *)adev->rmmio + pcie_index * 4;
 964         pcie_data_offset = (void __iomem *)adev->rmmio + pcie_data * 4;
 965         if (pcie_index_hi != 0)
 966                 pcie_index_hi_offset = (void __iomem *)adev->rmmio +
 967                                 pcie_index_hi * 4;
 968
 969         /* write low 32 bits */
 970         writel(reg_addr, pcie_index_offset);
 971         readl(pcie_index_offset);
 972         if (pcie_index_hi != 0) {
 973                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 974                 readl(pcie_index_hi_offset);
 975         }
 976         writel((u32)(reg_data & 0xffffffffULL), pcie_data_offset);
 977         readl(pcie_data_offset);
 978         /* write high 32 bits */
 979         writel(reg_addr + 4, pcie_index_offset);
 980         readl(pcie_index_offset);
 981         if (pcie_index_hi != 0) {
 982                 writel((reg_addr >> 32) & 0xff, pcie_index_hi_offset);
 983                 readl(pcie_index_hi_offset);
 984         }
 985         writel((u32)(reg_data >> 32), pcie_data_offset);
 986         readl(pcie_data_offset);
 987
 988         /* clear the high bits */
 989         if (pcie_index_hi != 0) {
 990                 writel(0, pcie_index_hi_offset);
 991                 readl(pcie_index_hi_offset);
 992         }
 993
 994         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
 995 }
 996
 997 /**
 998  * amdgpu_device_get_rev_id - query device rev_id
 999  *
1000  * @adev: amdgpu_device pointer
1001  *
1002  * Return device rev_id
1003  */
1004 u32 amdgpu_device_get_rev_id(struct amdgpu_device *adev)
1005 {
1006         return adev->nbio.funcs->get_rev_id(adev);
1007 }
1008
1009 /**
1010  * amdgpu_invalid_rreg - dummy reg read function
1011  *
1012  * @adev: amdgpu_device pointer
1013  * @reg: offset of register
1014  *
1015  * Dummy register read function.  Used for register blocks
1016  * that certain asics don't have (all asics).
1017  * Returns the value in the register.
1018  */
1019 static uint32_t amdgpu_invalid_rreg(struct amdgpu_device *adev, uint32_t reg)
1020 {
1021         DRM_ERROR("Invalid callback to read register 0x%04X\n", reg);
1022         BUG();
1023         return 0;
1024 }
1025
1026 static uint32_t amdgpu_invalid_rreg_ext(struct amdgpu_device *adev, uint64_t reg)
1027 {
1028         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1029         BUG();
1030         return 0;
1031 }
1032
1033 /**
1034  * amdgpu_invalid_wreg - dummy reg write function
1035  *
1036  * @adev: amdgpu_device pointer
1037  * @reg: offset of register
1038  * @v: value to write to the register
1039  *
1040  * Dummy register read function.  Used for register blocks
1041  * that certain asics don't have (all asics).
1042  */
1043 static void amdgpu_invalid_wreg(struct amdgpu_device *adev, uint32_t reg, uint32_t v)
1044 {
1045         DRM_ERROR("Invalid callback to write register 0x%04X with 0x%08X\n",
1046                   reg, v);
1047         BUG();
1048 }
1049
1050 static void amdgpu_invalid_wreg_ext(struct amdgpu_device *adev, uint64_t reg, uint32_t v)
1051 {
1052         DRM_ERROR("Invalid callback to write register 0x%llX with 0x%08X\n",
1053                   reg, v);
1054         BUG();
1055 }
1056
1057 /**
1058  * amdgpu_invalid_rreg64 - dummy 64 bit reg read function
1059  *
1060  * @adev: amdgpu_device pointer
1061  * @reg: offset of register
1062  *
1063  * Dummy register read function.  Used for register blocks
1064  * that certain asics don't have (all asics).
1065  * Returns the value in the register.
1066  */
1067 static uint64_t amdgpu_invalid_rreg64(struct amdgpu_device *adev, uint32_t reg)
1068 {
1069         DRM_ERROR("Invalid callback to read 64 bit register 0x%04X\n", reg);
1070         BUG();
1071         return 0;
1072 }
1073
1074 static uint64_t amdgpu_invalid_rreg64_ext(struct amdgpu_device *adev, uint64_t reg)
1075 {
1076         DRM_ERROR("Invalid callback to read register 0x%llX\n", reg);
1077         BUG();
1078         return 0;
1079 }
1080
1081 /**
1082  * amdgpu_invalid_wreg64 - dummy reg write function
1083  *
1084  * @adev: amdgpu_device pointer
1085  * @reg: offset of register
1086  * @v: value to write to the register
1087  *
1088  * Dummy register read function.  Used for register blocks
1089  * that certain asics don't have (all asics).
1090  */
1091 static void amdgpu_invalid_wreg64(struct amdgpu_device *adev, uint32_t reg, uint64_t v)
1092 {
1093         DRM_ERROR("Invalid callback to write 64 bit register 0x%04X with 0x%08llX\n",
1094                   reg, v);
1095         BUG();
1096 }
1097
1098 static void amdgpu_invalid_wreg64_ext(struct amdgpu_device *adev, uint64_t reg, uint64_t v)
1099 {
1100         DRM_ERROR("Invalid callback to write 64 bit register 0x%llX with 0x%08llX\n",
1101                   reg, v);
1102         BUG();
1103 }
1104
1105 /**
1106  * amdgpu_block_invalid_rreg - dummy reg read function
1107  *
1108  * @adev: amdgpu_device pointer
1109  * @block: offset of instance
1110  * @reg: offset of register
1111  *
1112  * Dummy register read function.  Used for register blocks
1113  * that certain asics don't have (all asics).
1114  * Returns the value in the register.
1115  */
1116 static uint32_t amdgpu_block_invalid_rreg(struct amdgpu_device *adev,
1117                                           uint32_t block, uint32_t reg)
1118 {
1119         DRM_ERROR("Invalid callback to read register 0x%04X in block 0x%04X\n",
1120                   reg, block);
1121         BUG();
1122         return 0;
1123 }
1124
1125 /**
1126  * amdgpu_block_invalid_wreg - dummy reg write function
1127  *
1128  * @adev: amdgpu_device pointer
1129  * @block: offset of instance
1130  * @reg: offset of register
1131  * @v: value to write to the register
1132  *
1133  * Dummy register read function.  Used for register blocks
1134  * that certain asics don't have (all asics).
1135  */
1136 static void amdgpu_block_invalid_wreg(struct amdgpu_device *adev,
1137                                       uint32_t block,
1138                                       uint32_t reg, uint32_t v)
1139 {
1140         DRM_ERROR("Invalid block callback to write register 0x%04X in block 0x%04X with 0x%08X\n",
1141                   reg, block, v);
1142         BUG();
1143 }
1144
1145 /**
1146  * amdgpu_device_asic_init - Wrapper for atom asic_init
1147  *
1148  * @adev: amdgpu_device pointer
1149  *
1150  * Does any asic specific work and then calls atom asic init.
1151  */
1152 static int amdgpu_device_asic_init(struct amdgpu_device *adev)
1153 {
1154         int ret;
1155
1156         amdgpu_asic_pre_asic_init(adev);
1157
1158         if (amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(9, 4, 3) ||
1159             amdgpu_ip_version(adev, GC_HWIP, 0) >= IP_VERSION(11, 0, 0)) {
1160                 amdgpu_psp_wait_for_bootloader(adev);
1161                 ret = amdgpu_atomfirmware_asic_init(adev, true);
1162                 /* TODO: check the return val and stop device initialization if boot fails */
1163                 amdgpu_psp_query_boot_status(adev);
1164                 return ret;
1165         } else {
1166                 return amdgpu_atom_asic_init(adev->mode_info.atom_context);
1167         }
1168
1169         return 0;
1170 }
1171
1172 /**
1173  * amdgpu_device_mem_scratch_init - allocate the VRAM scratch page
1174  *
1175  * @adev: amdgpu_device pointer
1176  *
1177  * Allocates a scratch page of VRAM for use by various things in the
1178  * driver.
1179  */
1180 static int amdgpu_device_mem_scratch_init(struct amdgpu_device *adev)
1181 {
1182         return amdgpu_bo_create_kernel(adev, AMDGPU_GPU_PAGE_SIZE, PAGE_SIZE,
1183                                        AMDGPU_GEM_DOMAIN_VRAM |
1184                                        AMDGPU_GEM_DOMAIN_GTT,
1185                                        &adev->mem_scratch.robj,
1186                                        &adev->mem_scratch.gpu_addr,
1187                                        (void **)&adev->mem_scratch.ptr);
1188 }
1189
1190 /**
1191  * amdgpu_device_mem_scratch_fini - Free the VRAM scratch page
1192  *
1193  * @adev: amdgpu_device pointer
1194  *
1195  * Frees the VRAM scratch page.
1196  */
1197 static void amdgpu_device_mem_scratch_fini(struct amdgpu_device *adev)
1198 {
1199         amdgpu_bo_free_kernel(&adev->mem_scratch.robj, NULL, NULL);
1200 }
1201
1202 /**
1203  * amdgpu_device_program_register_sequence - program an array of registers.
1204  *
1205  * @adev: amdgpu_device pointer
1206  * @registers: pointer to the register array
1207  * @array_size: size of the register array
1208  *
1209  * Programs an array or registers with and or masks.
1210  * This is a helper for setting golden registers.
1211  */
1212 void amdgpu_device_program_register_sequence(struct amdgpu_device *adev,
1213                                              const u32 *registers,
1214                                              const u32 array_size)
1215 {
1216         u32 tmp, reg, and_mask, or_mask;
1217         int i;
1218
1219         if (array_size % 3)
1220                 return;
1221
1222         for (i = 0; i < array_size; i += 3) {
1223                 reg = registers[i + 0];
1224                 and_mask = registers[i + 1];
1225                 or_mask = registers[i + 2];
1226
1227                 if (and_mask == 0xffffffff) {
1228                         tmp = or_mask;
1229                 } else {
1230                         tmp = RREG32(reg);
1231                         tmp &= ~and_mask;
1232                         if (adev->family >= AMDGPU_FAMILY_AI)
1233                                 tmp |= (or_mask & and_mask);
1234                         else
1235                                 tmp |= or_mask;
1236                 }
1237                 WREG32(reg, tmp);
1238         }
1239 }
1240
1241 /**
1242  * amdgpu_device_pci_config_reset - reset the GPU
1243  *
1244  * @adev: amdgpu_device pointer
1245  *
1246  * Resets the GPU using the pci config reset sequence.
1247  * Only applicable to asics prior to vega10.
1248  */
1249 void amdgpu_device_pci_config_reset(struct amdgpu_device *adev)
1250 {
1251         pci_write_config_dword(adev->pdev, 0x7c, AMDGPU_ASIC_RESET_DATA);
1252 }
1253
1254 /**
1255  * amdgpu_device_pci_reset - reset the GPU using generic PCI means
1256  *
1257  * @adev: amdgpu_device pointer
1258  *
1259  * Resets the GPU using generic pci reset interfaces (FLR, SBR, etc.).
1260  */
1261 int amdgpu_device_pci_reset(struct amdgpu_device *adev)
1262 {
1263         return pci_reset_function(adev->pdev);
1264 }
1265
1266 /*
1267  * amdgpu_device_wb_*()
1268  * Writeback is the method by which the GPU updates special pages in memory
1269  * with the status of certain GPU events (fences, ring pointers,etc.).
1270  */
1271
1272 /**
1273  * amdgpu_device_wb_fini - Disable Writeback and free memory
1274  *
1275  * @adev: amdgpu_device pointer
1276  *
1277  * Disables Writeback and frees the Writeback memory (all asics).
1278  * Used at driver shutdown.
1279  */
1280 static void amdgpu_device_wb_fini(struct amdgpu_device *adev)
1281 {
1282         if (adev->wb.wb_obj) {
1283                 amdgpu_bo_free_kernel(&adev->wb.wb_obj,
1284                                       &adev->wb.gpu_addr,
1285                                       (void **)&adev->wb.wb);
1286                 adev->wb.wb_obj = NULL;
1287         }
1288 }
1289
1290 /**
1291  * amdgpu_device_wb_init - Init Writeback driver info and allocate memory
1292  *
1293  * @adev: amdgpu_device pointer
1294  *
1295  * Initializes writeback and allocates writeback memory (all asics).
1296  * Used at driver startup.
1297  * Returns 0 on success or an -error on failure.
1298  */
1299 static int amdgpu_device_wb_init(struct amdgpu_device *adev)
1300 {
1301         int r;
1302
1303         if (adev->wb.wb_obj == NULL) {
1304                 /* AMDGPU_MAX_WB * sizeof(uint32_t) * 8 = AMDGPU_MAX_WB 256bit slots */
1305                 r = amdgpu_bo_create_kernel(adev, AMDGPU_MAX_WB * sizeof(uint32_t) * 8,
1306                                             PAGE_SIZE, AMDGPU_GEM_DOMAIN_GTT,
1307                                             &adev->wb.wb_obj, &adev->wb.gpu_addr,
1308                                             (void **)&adev->wb.wb);
1309                 if (r) {
1310                         dev_warn(adev->dev, "(%d) create WB bo failed\n", r);
1311                         return r;
1312                 }
1313
1314                 adev->wb.num_wb = AMDGPU_MAX_WB;
1315                 memset(&adev->wb.used, 0, sizeof(adev->wb.used));
1316
1317                 /* clear wb memory */
1318                 memset((char *)adev->wb.wb, 0, AMDGPU_MAX_WB * sizeof(uint32_t) * 8);
1319         }
1320
1321         return 0;
1322 }
1323
1324 /**
1325  * amdgpu_device_wb_get - Allocate a wb entry
1326  *
1327  * @adev: amdgpu_device pointer
1328  * @wb: wb index
1329  *
1330  * Allocate a wb slot for use by the driver (all asics).
1331  * Returns 0 on success or -EINVAL on failure.
1332  */
1333 int amdgpu_device_wb_get(struct amdgpu_device *adev, u32 *wb)
1334 {
1335         unsigned long offset = find_first_zero_bit(adev->wb.used, adev->wb.num_wb);
1336
1337         if (offset < adev->wb.num_wb) {
1338                 __set_bit(offset, adev->wb.used);
1339                 *wb = offset << 3; /* convert to dw offset */
1340                 return 0;
1341         } else {
1342                 return -EINVAL;
1343         }
1344 }
1345
1346 /**
1347  * amdgpu_device_wb_free - Free a wb entry
1348  *
1349  * @adev: amdgpu_device pointer
1350  * @wb: wb index
1351  *
1352  * Free a wb slot allocated for use by the driver (all asics)
1353  */
1354 void amdgpu_device_wb_free(struct amdgpu_device *adev, u32 wb)
1355 {
1356         wb >>= 3;
1357         if (wb < adev->wb.num_wb)
1358                 __clear_bit(wb, adev->wb.used);
1359 }
1360
1361 /**
1362  * amdgpu_device_resize_fb_bar - try to resize FB BAR
1363  *
1364  * @adev: amdgpu_device pointer
1365  *
1366  * Try to resize FB BAR to make all VRAM CPU accessible. We try very hard not
1367  * to fail, but if any of the BARs is not accessible after the size we abort
1368  * driver loading by returning -ENODEV.
1369  */
1370 int amdgpu_device_resize_fb_bar(struct amdgpu_device *adev)
1371 {
1372         int rbar_size = pci_rebar_bytes_to_size(adev->gmc.real_vram_size);
1373         struct pci_bus *root;
1374         struct resource *res;
1375         unsigned int i;
1376         u16 cmd;
1377         int r;
1378
1379         if (!IS_ENABLED(CONFIG_PHYS_ADDR_T_64BIT))
1380                 return 0;
1381
1382         /* Bypass for VF */
1383         if (amdgpu_sriov_vf(adev))
1384                 return 0;
1385
1386         /* skip if the bios has already enabled large BAR */
1387         if (adev->gmc.real_vram_size &&
1388             (pci_resource_len(adev->pdev, 0) >= adev->gmc.real_vram_size))
1389                 return 0;
1390
1391         /* Check if the root BUS has 64bit memory resources */
1392         root = adev->pdev->bus;
1393         while (root->parent)
1394                 root = root->parent;
1395
1396         pci_bus_for_each_resource(root, res, i) {
1397                 if (res && res->flags & (IORESOURCE_MEM | IORESOURCE_MEM_64) &&
1398                     res->start > 0x100000000ull)
1399                         break;
1400         }
1401
1402         /* Trying to resize is pointless without a root hub window above 4GB */
1403         if (!res)
1404                 return 0;
1405
1406         /* Limit the BAR size to what is available */
1407         rbar_size = min(fls(pci_rebar_get_possible_sizes(adev->pdev, 0)) - 1,
1408                         rbar_size);
1409
1410         /* Disable memory decoding while we change the BAR addresses and size */
1411         pci_read_config_word(adev->pdev, PCI_COMMAND, &cmd);
1412         pci_write_config_word(adev->pdev, PCI_COMMAND,
1413                               cmd & ~PCI_COMMAND_MEMORY);
1414
1415         /* Free the VRAM and doorbell BAR, we most likely need to move both. */
1416         amdgpu_doorbell_fini(adev);
1417         if (adev->asic_type >= CHIP_BONAIRE)
1418                 pci_release_resource(adev->pdev, 2);
1419
1420         pci_release_resource(adev->pdev, 0);
1421
1422         r = pci_resize_resource(adev->pdev, 0, rbar_size);
1423         if (r == -ENOSPC)
1424                 DRM_INFO("Not enough PCI address space for a large BAR.");
1425         else if (r && r != -ENOTSUPP)
1426                 DRM_ERROR("Problem resizing BAR0 (%d).", r);
1427
1428         pci_assign_unassigned_bus_resources(adev->pdev->bus);
1429
1430         /* When the doorbell or fb BAR isn't available we have no chance of
1431          * using the device.
1432          */
1433         r = amdgpu_doorbell_init(adev);
1434         if (r || (pci_resource_flags(adev->pdev, 0) & IORESOURCE_UNSET))
1435                 return -ENODEV;
1436
1437         pci_write_config_word(adev->pdev, PCI_COMMAND, cmd);
1438
1439         return 0;
1440 }
1441
1442 static bool amdgpu_device_read_bios(struct amdgpu_device *adev)
1443 {
1444         if (hweight32(adev->aid_mask) && (adev->flags & AMD_IS_APU))
1445                 return false;
1446
1447         return true;
1448 }
1449
1450 /*
1451  * GPU helpers function.
1452  */
1453 /**
1454  * amdgpu_device_need_post - check if the hw need post or not
1455  *
1456  * @adev: amdgpu_device pointer
1457  *
1458  * Check if the asic has been initialized (all asics) at driver startup
1459  * or post is needed if  hw reset is performed.
1460  * Returns true if need or false if not.
1461  */
1462 bool amdgpu_device_need_post(struct amdgpu_device *adev)
1463 {
1464         uint32_t reg;
1465
1466         if (amdgpu_sriov_vf(adev))
1467                 return false;
1468
1469         if (!amdgpu_device_read_bios(adev))
1470                 return false;
1471
1472         if (amdgpu_passthrough(adev)) {
1473                 /* for FIJI: In whole GPU pass-through virtualization case, after VM reboot
1474                  * some old smc fw still need driver do vPost otherwise gpu hang, while
1475                  * those smc fw version above 22.15 doesn't have this flaw, so we force
1476                  * vpost executed for smc version below 22.15
1477                  */
1478                 if (adev->asic_type == CHIP_FIJI) {
1479                         int err;
1480                         uint32_t fw_ver;
1481
1482                         err = request_firmware(&adev->pm.fw, "amdgpu/fiji_smc.bin", adev->dev);
1483                         /* force vPost if error occured */
1484                         if (err)
1485                                 return true;
1486
1487                         fw_ver = *((uint32_t *)adev->pm.fw->data + 69);
1488                         if (fw_ver < 0x00160e00)
1489                                 return true;
1490                 }
1491         }
1492
1493         /* Don't post if we need to reset whole hive on init */
1494         if (adev->gmc.xgmi.pending_reset)
1495                 return false;
1496
1497         if (adev->has_hw_reset) {
1498                 adev->has_hw_reset = false;
1499                 return true;
1500         }
1501
1502         /* bios scratch used on CIK+ */
1503         if (adev->asic_type >= CHIP_BONAIRE)
1504                 return amdgpu_atombios_scratch_need_asic_init(adev);
1505
1506         /* check MEM_SIZE for older asics */
1507         reg = amdgpu_asic_get_config_memsize(adev);
1508
1509         if ((reg != 0) && (reg != 0xffffffff))
1510                 return false;
1511
1512         return true;
1513 }
1514
1515 /*
1516  * Check whether seamless boot is supported.
1517  *
1518  * So far we only support seamless boot on DCE 3.0 or later.
1519  * If users report that it works on older ASICS as well, we may
1520  * loosen this.
1521  */
1522 bool amdgpu_device_seamless_boot_supported(struct amdgpu_device *adev)
1523 {
1524         switch (amdgpu_seamless) {
1525         case -1:
1526                 break;
1527         case 1:
1528                 return true;
1529         case 0:
1530                 return false;
1531         default:
1532                 DRM_ERROR("Invalid value for amdgpu.seamless: %d\n",
1533                           amdgpu_seamless);
1534                 return false;
1535         }
1536
1537         if (!(adev->flags & AMD_IS_APU))
1538                 return false;
1539
1540         if (adev->mman.keep_stolen_vga_memory)
1541                 return false;
1542
1543         return adev->ip_versions[DCE_HWIP][0] >= IP_VERSION(3, 0, 0);
1544 }
1545
1546 /*
1547  * Intel hosts such as Rocket Lake, Alder Lake, Raptor Lake and Sapphire Rapids
1548  * don't support dynamic speed switching. Until we have confirmation from Intel
1549  * that a specific host supports it, it's safer that we keep it disabled for all.
1550  *
1551  * https://edc.intel.com/content/www/us/en/design/products/platforms/details/raptor-lake-s/13th-generation-core-processors-datasheet-volume-1-of-2/005/pci-express-support/
1552  * https://gitlab.freedesktop.org/drm/amd/-/issues/2663
1553  */
1554 static bool amdgpu_device_pcie_dynamic_switching_supported(void)
1555 {
1556 #if IS_ENABLED(CONFIG_X86)
1557         struct cpuinfo_x86 *c = &cpu_data(0);
1558
1559         if (c->x86_vendor == X86_VENDOR_INTEL)
1560                 return false;
1561 #endif
1562         return true;
1563 }
1564
1565 /**
1566  * amdgpu_device_should_use_aspm - check if the device should program ASPM
1567  *
1568  * @adev: amdgpu_device pointer
1569  *
1570  * Confirm whether the module parameter and pcie bridge agree that ASPM should
1571  * be set for this device.
1572  *
1573  * Returns true if it should be used or false if not.
1574  */
1575 bool amdgpu_device_should_use_aspm(struct amdgpu_device *adev)
1576 {
1577         switch (amdgpu_aspm) {
1578         case -1:
1579                 break;
1580         case 0:
1581                 return false;
1582         case 1:
1583                 return true;
1584         default:
1585                 return false;
1586         }
1587         if (adev->flags & AMD_IS_APU)
1588                 return false;
1589         if (!(adev->pm.pp_feature & PP_PCIE_DPM_MASK))
1590                 return false;
1591         return pcie_aspm_enabled(adev->pdev);
1592 }
1593
1594 /* if we get transitioned to only one device, take VGA back */
1595 /**
1596  * amdgpu_device_vga_set_decode - enable/disable vga decode
1597  *
1598  * @pdev: PCI device pointer
1599  * @state: enable/disable vga decode
1600  *
1601  * Enable/disable vga decode (all asics).
1602  * Returns VGA resource flags.
1603  */
1604 static unsigned int amdgpu_device_vga_set_decode(struct pci_dev *pdev,
1605                 bool state)
1606 {
1607         struct amdgpu_device *adev = drm_to_adev(pci_get_drvdata(pdev));
1608
1609         amdgpu_asic_set_vga_state(adev, state);
1610         if (state)
1611                 return VGA_RSRC_LEGACY_IO | VGA_RSRC_LEGACY_MEM |
1612                        VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1613         else
1614                 return VGA_RSRC_NORMAL_IO | VGA_RSRC_NORMAL_MEM;
1615 }
1616
1617 /**
1618  * amdgpu_device_check_block_size - validate the vm block size
1619  *
1620  * @adev: amdgpu_device pointer
1621  *
1622  * Validates the vm block size specified via module parameter.
1623  * The vm block size defines number of bits in page table versus page directory,
1624  * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1625  * page table and the remaining bits are in the page directory.
1626  */
1627 static void amdgpu_device_check_block_size(struct amdgpu_device *adev)
1628 {
1629         /* defines number of bits in page table versus page directory,
1630          * a page is 4KB so we have 12 bits offset, minimum 9 bits in the
1631          * page table and the remaining bits are in the page directory
1632          */
1633         if (amdgpu_vm_block_size == -1)
1634                 return;
1635
1636         if (amdgpu_vm_block_size < 9) {
1637                 dev_warn(adev->dev, "VM page table size (%d) too small\n",
1638                          amdgpu_vm_block_size);
1639                 amdgpu_vm_block_size = -1;
1640         }
1641 }
1642
1643 /**
1644  * amdgpu_device_check_vm_size - validate the vm size
1645  *
1646  * @adev: amdgpu_device pointer
1647  *
1648  * Validates the vm size in GB specified via module parameter.
1649  * The VM size is the size of the GPU virtual memory space in GB.
1650  */
1651 static void amdgpu_device_check_vm_size(struct amdgpu_device *adev)
1652 {
1653         /* no need to check the default value */
1654         if (amdgpu_vm_size == -1)
1655                 return;
1656
1657         if (amdgpu_vm_size < 1) {
1658                 dev_warn(adev->dev, "VM size (%d) too small, min is 1GB\n",
1659                          amdgpu_vm_size);
1660                 amdgpu_vm_size = -1;
1661         }
1662 }
1663
1664 static void amdgpu_device_check_smu_prv_buffer_size(struct amdgpu_device *adev)
1665 {
1666         struct sysinfo si;
1667         bool is_os_64 = (sizeof(void *) == 8);
1668         uint64_t total_memory;
1669         uint64_t dram_size_seven_GB = 0x1B8000000;
1670         uint64_t dram_size_three_GB = 0xB8000000;
1671
1672         if (amdgpu_smu_memory_pool_size == 0)
1673                 return;
1674
1675         if (!is_os_64) {
1676                 DRM_WARN("Not 64-bit OS, feature not supported\n");
1677                 goto def_value;
1678         }
1679         si_meminfo(&si);
1680         total_memory = (uint64_t)si.totalram * si.mem_unit;
1681
1682         if ((amdgpu_smu_memory_pool_size == 1) ||
1683                 (amdgpu_smu_memory_pool_size == 2)) {
1684                 if (total_memory < dram_size_three_GB)
1685                         goto def_value1;
1686         } else if ((amdgpu_smu_memory_pool_size == 4) ||
1687                 (amdgpu_smu_memory_pool_size == 8)) {
1688                 if (total_memory < dram_size_seven_GB)
1689                         goto def_value1;
1690         } else {
1691                 DRM_WARN("Smu memory pool size not supported\n");
1692                 goto def_value;
1693         }
1694         adev->pm.smu_prv_buffer_size = amdgpu_smu_memory_pool_size << 28;
1695
1696         return;
1697
1698 def_value1:
1699         DRM_WARN("No enough system memory\n");
1700 def_value:
1701         adev->pm.smu_prv_buffer_size = 0;
1702 }
1703
1704 static int amdgpu_device_init_apu_flags(struct amdgpu_device *adev)
1705 {
1706         if (!(adev->flags & AMD_IS_APU) ||
1707             adev->asic_type < CHIP_RAVEN)
1708                 return 0;
1709
1710         switch (adev->asic_type) {
1711         case CHIP_RAVEN:
1712                 if (adev->pdev->device == 0x15dd)
1713                         adev->apu_flags |= AMD_APU_IS_RAVEN;
1714                 if (adev->pdev->device == 0x15d8)
1715                         adev->apu_flags |= AMD_APU_IS_PICASSO;
1716                 break;
1717         case CHIP_RENOIR:
1718                 if ((adev->pdev->device == 0x1636) ||
1719                     (adev->pdev->device == 0x164c))
1720                         adev->apu_flags |= AMD_APU_IS_RENOIR;
1721                 else
1722                         adev->apu_flags |= AMD_APU_IS_GREEN_SARDINE;
1723                 break;
1724         case CHIP_VANGOGH:
1725                 adev->apu_flags |= AMD_APU_IS_VANGOGH;
1726                 break;
1727         case CHIP_YELLOW_CARP:
1728                 break;
1729         case CHIP_CYAN_SKILLFISH:
1730                 if ((adev->pdev->device == 0x13FE) ||
1731                     (adev->pdev->device == 0x143F))
1732                         adev->apu_flags |= AMD_APU_IS_CYAN_SKILLFISH2;
1733                 break;
1734         default:
1735                 break;
1736         }
1737
1738         return 0;
1739 }
1740
1741 /**
1742  * amdgpu_device_check_arguments - validate module params
1743  *
1744  * @adev: amdgpu_device pointer
1745  *
1746  * Validates certain module parameters and updates
1747  * the associated values used by the driver (all asics).
1748  */
1749 static int amdgpu_device_check_arguments(struct amdgpu_device *adev)
1750 {
1751         if (amdgpu_sched_jobs < 4) {
1752                 dev_warn(adev->dev, "sched jobs (%d) must be at least 4\n",
1753                          amdgpu_sched_jobs);
1754                 amdgpu_sched_jobs = 4;
1755         } else if (!is_power_of_2(amdgpu_sched_jobs)) {
1756                 dev_warn(adev->dev, "sched jobs (%d) must be a power of 2\n",
1757                          amdgpu_sched_jobs);
1758                 amdgpu_sched_jobs = roundup_pow_of_two(amdgpu_sched_jobs);
1759         }
1760
1761         if (amdgpu_gart_size != -1 && amdgpu_gart_size < 32) {
1762                 /* gart size must be greater or equal to 32M */
1763                 dev_warn(adev->dev, "gart size (%d) too small\n",
1764                          amdgpu_gart_size);
1765                 amdgpu_gart_size = -1;
1766         }
1767
1768         if (amdgpu_gtt_size != -1 && amdgpu_gtt_size < 32) {
1769                 /* gtt size must be greater or equal to 32M */
1770                 dev_warn(adev->dev, "gtt size (%d) too small\n",
1771                                  amdgpu_gtt_size);
1772                 amdgpu_gtt_size = -1;
1773         }
1774
1775         /* valid range is between 4 and 9 inclusive */
1776         if (amdgpu_vm_fragment_size != -1 &&
1777             (amdgpu_vm_fragment_size > 9 || amdgpu_vm_fragment_size < 4)) {
1778                 dev_warn(adev->dev, "valid range is between 4 and 9\n");
1779                 amdgpu_vm_fragment_size = -1;
1780         }
1781
1782         if (amdgpu_sched_hw_submission < 2) {
1783                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be at least 2\n",
1784                          amdgpu_sched_hw_submission);
1785                 amdgpu_sched_hw_submission = 2;
1786         } else if (!is_power_of_2(amdgpu_sched_hw_submission)) {
1787                 dev_warn(adev->dev, "sched hw submission jobs (%d) must be a power of 2\n",
1788                          amdgpu_sched_hw_submission);
1789                 amdgpu_sched_hw_submission = roundup_pow_of_two(amdgpu_sched_hw_submission);
1790         }
1791
1792         if (amdgpu_reset_method < -1 || amdgpu_reset_method > 4) {
1793                 dev_warn(adev->dev, "invalid option for reset method, reverting to default\n");
1794                 amdgpu_reset_method = -1;
1795         }
1796
1797         amdgpu_device_check_smu_prv_buffer_size(adev);
1798
1799         amdgpu_device_check_vm_size(adev);
1800
1801         amdgpu_device_check_block_size(adev);
1802
1803         adev->firmware.load_type = amdgpu_ucode_get_load_type(adev, amdgpu_fw_load_type);
1804
1805         return 0;
1806 }
1807
1808 /**
1809  * amdgpu_switcheroo_set_state - set switcheroo state
1810  *
1811  * @pdev: pci dev pointer
1812  * @state: vga_switcheroo state
1813  *
1814  * Callback for the switcheroo driver.  Suspends or resumes
1815  * the asics before or after it is powered up using ACPI methods.
1816  */
1817 static void amdgpu_switcheroo_set_state(struct pci_dev *pdev,
1818                                         enum vga_switcheroo_state state)
1819 {
1820         struct drm_device *dev = pci_get_drvdata(pdev);
1821         int r;
1822
1823         if (amdgpu_device_supports_px(dev) && state == VGA_SWITCHEROO_OFF)
1824                 return;
1825
1826         if (state == VGA_SWITCHEROO_ON) {
1827                 pr_info("switched on\n");
1828                 /* don't suspend or resume card normally */
1829                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1830
1831                 pci_set_power_state(pdev, PCI_D0);
1832                 amdgpu_device_load_pci_state(pdev);
1833                 r = pci_enable_device(pdev);
1834                 if (r)
1835                         DRM_WARN("pci_enable_device failed (%d)\n", r);
1836                 amdgpu_device_resume(dev, true);
1837
1838                 dev->switch_power_state = DRM_SWITCH_POWER_ON;
1839         } else {
1840                 pr_info("switched off\n");
1841                 dev->switch_power_state = DRM_SWITCH_POWER_CHANGING;
1842                 amdgpu_device_prepare(dev);
1843                 amdgpu_device_suspend(dev, true);
1844                 amdgpu_device_cache_pci_state(pdev);
1845                 /* Shut down the device */
1846                 pci_disable_device(pdev);
1847                 pci_set_power_state(pdev, PCI_D3cold);
1848                 dev->switch_power_state = DRM_SWITCH_POWER_OFF;
1849         }
1850 }
1851
1852 /**
1853  * amdgpu_switcheroo_can_switch - see if switcheroo state can change
1854  *
1855  * @pdev: pci dev pointer
1856  *
1857  * Callback for the switcheroo driver.  Check of the switcheroo
1858  * state can be changed.
1859  * Returns true if the state can be changed, false if not.
1860  */
1861 static bool amdgpu_switcheroo_can_switch(struct pci_dev *pdev)
1862 {
1863         struct drm_device *dev = pci_get_drvdata(pdev);
1864
1865        /*
1866         * FIXME: open_count is protected by drm_global_mutex but that would lead to
1867         * locking inversion with the driver load path. And the access here is
1868         * completely racy anyway. So don't bother with locking for now.
1869         */
1870         return atomic_read(&dev->open_count) == 0;
1871 }
1872
1873 static const struct vga_switcheroo_client_ops amdgpu_switcheroo_ops = {
1874         .set_gpu_state = amdgpu_switcheroo_set_state,
1875         .reprobe = NULL,
1876         .can_switch = amdgpu_switcheroo_can_switch,
1877 };
1878
1879 /**
1880  * amdgpu_device_ip_set_clockgating_state - set the CG state
1881  *
1882  * @dev: amdgpu_device pointer
1883  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1884  * @state: clockgating state (gate or ungate)
1885  *
1886  * Sets the requested clockgating state for all instances of
1887  * the hardware IP specified.
1888  * Returns the error code from the last instance.
1889  */
1890 int amdgpu_device_ip_set_clockgating_state(void *dev,
1891                                            enum amd_ip_block_type block_type,
1892                                            enum amd_clockgating_state state)
1893 {
1894         struct amdgpu_device *adev = dev;
1895         int i, r = 0;
1896
1897         for (i = 0; i < adev->num_ip_blocks; i++) {
1898                 if (!adev->ip_blocks[i].status.valid)
1899                         continue;
1900                 if (adev->ip_blocks[i].version->type != block_type)
1901                         continue;
1902                 if (!adev->ip_blocks[i].version->funcs->set_clockgating_state)
1903                         continue;
1904                 r = adev->ip_blocks[i].version->funcs->set_clockgating_state(
1905                         (void *)adev, state);
1906                 if (r)
1907                         DRM_ERROR("set_clockgating_state of IP block <%s> failed %d\n",
1908                                   adev->ip_blocks[i].version->funcs->name, r);
1909         }
1910         return r;
1911 }
1912
1913 /**
1914  * amdgpu_device_ip_set_powergating_state - set the PG state
1915  *
1916  * @dev: amdgpu_device pointer
1917  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1918  * @state: powergating state (gate or ungate)
1919  *
1920  * Sets the requested powergating state for all instances of
1921  * the hardware IP specified.
1922  * Returns the error code from the last instance.
1923  */
1924 int amdgpu_device_ip_set_powergating_state(void *dev,
1925                                            enum amd_ip_block_type block_type,
1926                                            enum amd_powergating_state state)
1927 {
1928         struct amdgpu_device *adev = dev;
1929         int i, r = 0;
1930
1931         for (i = 0; i < adev->num_ip_blocks; i++) {
1932                 if (!adev->ip_blocks[i].status.valid)
1933                         continue;
1934                 if (adev->ip_blocks[i].version->type != block_type)
1935                         continue;
1936                 if (!adev->ip_blocks[i].version->funcs->set_powergating_state)
1937                         continue;
1938                 r = adev->ip_blocks[i].version->funcs->set_powergating_state(
1939                         (void *)adev, state);
1940                 if (r)
1941                         DRM_ERROR("set_powergating_state of IP block <%s> failed %d\n",
1942                                   adev->ip_blocks[i].version->funcs->name, r);
1943         }
1944         return r;
1945 }
1946
1947 /**
1948  * amdgpu_device_ip_get_clockgating_state - get the CG state
1949  *
1950  * @adev: amdgpu_device pointer
1951  * @flags: clockgating feature flags
1952  *
1953  * Walks the list of IPs on the device and updates the clockgating
1954  * flags for each IP.
1955  * Updates @flags with the feature flags for each hardware IP where
1956  * clockgating is enabled.
1957  */
1958 void amdgpu_device_ip_get_clockgating_state(struct amdgpu_device *adev,
1959                                             u64 *flags)
1960 {
1961         int i;
1962
1963         for (i = 0; i < adev->num_ip_blocks; i++) {
1964                 if (!adev->ip_blocks[i].status.valid)
1965                         continue;
1966                 if (adev->ip_blocks[i].version->funcs->get_clockgating_state)
1967                         adev->ip_blocks[i].version->funcs->get_clockgating_state((void *)adev, flags);
1968         }
1969 }
1970
1971 /**
1972  * amdgpu_device_ip_wait_for_idle - wait for idle
1973  *
1974  * @adev: amdgpu_device pointer
1975  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
1976  *
1977  * Waits for the request hardware IP to be idle.
1978  * Returns 0 for success or a negative error code on failure.
1979  */
1980 int amdgpu_device_ip_wait_for_idle(struct amdgpu_device *adev,
1981                                    enum amd_ip_block_type block_type)
1982 {
1983         int i, r;
1984
1985         for (i = 0; i < adev->num_ip_blocks; i++) {
1986                 if (!adev->ip_blocks[i].status.valid)
1987                         continue;
1988                 if (adev->ip_blocks[i].version->type == block_type) {
1989                         r = adev->ip_blocks[i].version->funcs->wait_for_idle((void *)adev);
1990                         if (r)
1991                                 return r;
1992                         break;
1993                 }
1994         }
1995         return 0;
1996
1997 }
1998
1999 /**
2000  * amdgpu_device_ip_is_idle - is the hardware IP idle
2001  *
2002  * @adev: amdgpu_device pointer
2003  * @block_type: Type of hardware IP (SMU, GFX, UVD, etc.)
2004  *
2005  * Check if the hardware IP is idle or not.
2006  * Returns true if it the IP is idle, false if not.
2007  */
2008 bool amdgpu_device_ip_is_idle(struct amdgpu_device *adev,
2009                               enum amd_ip_block_type block_type)
2010 {
2011         int i;
2012
2013         for (i = 0; i < adev->num_ip_blocks; i++) {
2014                 if (!adev->ip_blocks[i].status.valid)
2015                         continue;
2016                 if (adev->ip_blocks[i].version->type == block_type)
2017                         return adev->ip_blocks[i].version->funcs->is_idle((void *)adev);
2018         }
2019         return true;
2020
2021 }
2022
2023 /**
2024  * amdgpu_device_ip_get_ip_block - get a hw IP pointer
2025  *
2026  * @adev: amdgpu_device pointer
2027  * @type: Type of hardware IP (SMU, GFX, UVD, etc.)
2028  *
2029  * Returns a pointer to the hardware IP block structure
2030  * if it exists for the asic, otherwise NULL.
2031  */
2032 struct amdgpu_ip_block *
2033 amdgpu_device_ip_get_ip_block(struct amdgpu_device *adev,
2034                               enum amd_ip_block_type type)
2035 {
2036         int i;
2037
2038         for (i = 0; i < adev->num_ip_blocks; i++)
2039                 if (adev->ip_blocks[i].version->type == type)
2040                         return &adev->ip_blocks[i];
2041
2042         return NULL;
2043 }
2044
2045 /**
2046  * amdgpu_device_ip_block_version_cmp
2047  *
2048  * @adev: amdgpu_device pointer
2049  * @type: enum amd_ip_block_type
2050  * @major: major version
2051  * @minor: minor version
2052  *
2053  * return 0 if equal or greater
2054  * return 1 if smaller or the ip_block doesn't exist
2055  */
2056 int amdgpu_device_ip_block_version_cmp(struct amdgpu_device *adev,
2057                                        enum amd_ip_block_type type,
2058                                        u32 major, u32 minor)
2059 {
2060         struct amdgpu_ip_block *ip_block = amdgpu_device_ip_get_ip_block(adev, type);
2061
2062         if (ip_block && ((ip_block->version->major > major) ||
2063                         ((ip_block->version->major == major) &&
2064                         (ip_block->version->minor >= minor))))
2065                 return 0;
2066
2067         return 1;
2068 }
2069
2070 /**
2071  * amdgpu_device_ip_block_add
2072  *
2073  * @adev: amdgpu_device pointer
2074  * @ip_block_version: pointer to the IP to add
2075  *
2076  * Adds the IP block driver information to the collection of IPs
2077  * on the asic.
2078  */
2079 int amdgpu_device_ip_block_add(struct amdgpu_device *adev,
2080                                const struct amdgpu_ip_block_version *ip_block_version)
2081 {
2082         if (!ip_block_version)
2083                 return -EINVAL;
2084
2085         switch (ip_block_version->type) {
2086         case AMD_IP_BLOCK_TYPE_VCN:
2087                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_VCN_MASK)
2088                         return 0;
2089                 break;
2090         case AMD_IP_BLOCK_TYPE_JPEG:
2091                 if (adev->harvest_ip_mask & AMD_HARVEST_IP_JPEG_MASK)
2092                         return 0;
2093                 break;
2094         default:
2095                 break;
2096         }
2097
2098         DRM_INFO("add ip block number %d <%s>\n", adev->num_ip_blocks,
2099                   ip_block_version->funcs->name);
2100
2101         adev->ip_blocks[adev->num_ip_blocks++].version = ip_block_version;
2102
2103         return 0;
2104 }
2105
2106 /**
2107  * amdgpu_device_enable_virtual_display - enable virtual display feature
2108  *
2109  * @adev: amdgpu_device pointer
2110  *
2111  * Enabled the virtual display feature if the user has enabled it via
2112  * the module parameter virtual_display.  This feature provides a virtual
2113  * display hardware on headless boards or in virtualized environments.
2114  * This function parses and validates the configuration string specified by
2115  * the user and configues the virtual display configuration (number of
2116  * virtual connectors, crtcs, etc.) specified.
2117  */
2118 static void amdgpu_device_enable_virtual_display(struct amdgpu_device *adev)
2119 {
2120         adev->enable_virtual_display = false;
2121
2122         if (amdgpu_virtual_display) {
2123                 const char *pci_address_name = pci_name(adev->pdev);
2124                 char *pciaddstr, *pciaddstr_tmp, *pciaddname_tmp, *pciaddname;
2125
2126                 pciaddstr = kstrdup(amdgpu_virtual_display, GFP_KERNEL);
2127                 pciaddstr_tmp = pciaddstr;
2128                 while ((pciaddname_tmp = strsep(&pciaddstr_tmp, ";"))) {
2129                         pciaddname = strsep(&pciaddname_tmp, ",");
2130                         if (!strcmp("all", pciaddname)
2131                             || !strcmp(pci_address_name, pciaddname)) {
2132                                 long num_crtc;
2133                                 int res = -1;
2134
2135                                 adev->enable_virtual_display = true;
2136
2137                                 if (pciaddname_tmp)
2138                                         res = kstrtol(pciaddname_tmp, 10,
2139                                                       &num_crtc);
2140
2141                                 if (!res) {
2142                                         if (num_crtc < 1)
2143                                                 num_crtc = 1;
2144                                         if (num_crtc > 6)
2145                                                 num_crtc = 6;
2146                                         adev->mode_info.num_crtc = num_crtc;
2147                                 } else {
2148                                         adev->mode_info.num_crtc = 1;
2149                                 }
2150                                 break;
2151                         }
2152                 }
2153
2154                 DRM_INFO("virtual display string:%s, %s:virtual_display:%d, num_crtc:%d\n",
2155                          amdgpu_virtual_display, pci_address_name,
2156                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2157
2158                 kfree(pciaddstr);
2159         }
2160 }
2161
2162 void amdgpu_device_set_sriov_virtual_display(struct amdgpu_device *adev)
2163 {
2164         if (amdgpu_sriov_vf(adev) && !adev->enable_virtual_display) {
2165                 adev->mode_info.num_crtc = 1;
2166                 adev->enable_virtual_display = true;
2167                 DRM_INFO("virtual_display:%d, num_crtc:%d\n",
2168                          adev->enable_virtual_display, adev->mode_info.num_crtc);
2169         }
2170 }
2171
2172 /**
2173  * amdgpu_device_parse_gpu_info_fw - parse gpu info firmware
2174  *
2175  * @adev: amdgpu_device pointer
2176  *
2177  * Parses the asic configuration parameters specified in the gpu info
2178  * firmware and makes them availale to the driver for use in configuring
2179  * the asic.
2180  * Returns 0 on success, -EINVAL on failure.
2181  */
2182 static int amdgpu_device_parse_gpu_info_fw(struct amdgpu_device *adev)
2183 {
2184         const char *chip_name;
2185         char fw_name[40];
2186         int err;
2187         const struct gpu_info_firmware_header_v1_0 *hdr;
2188
2189         adev->firmware.gpu_info_fw = NULL;
2190
2191         if (adev->mman.discovery_bin)
2192                 return 0;
2193
2194         switch (adev->asic_type) {
2195         default:
2196                 return 0;
2197         case CHIP_VEGA10:
2198                 chip_name = "vega10";
2199                 break;
2200         case CHIP_VEGA12:
2201                 chip_name = "vega12";
2202                 break;
2203         case CHIP_RAVEN:
2204                 if (adev->apu_flags & AMD_APU_IS_RAVEN2)
2205                         chip_name = "raven2";
2206                 else if (adev->apu_flags & AMD_APU_IS_PICASSO)
2207                         chip_name = "picasso";
2208                 else
2209                         chip_name = "raven";
2210                 break;
2211         case CHIP_ARCTURUS:
2212                 chip_name = "arcturus";
2213                 break;
2214         case CHIP_NAVI12:
2215                 chip_name = "navi12";
2216                 break;
2217         }
2218
2219         snprintf(fw_name, sizeof(fw_name), "amdgpu/%s_gpu_info.bin", chip_name);
2220         err = amdgpu_ucode_request(adev, &adev->firmware.gpu_info_fw, fw_name);
2221         if (err) {
2222                 dev_err(adev->dev,
2223                         "Failed to get gpu_info firmware \"%s\"\n",
2224                         fw_name);
2225                 goto out;
2226         }
2227
2228         hdr = (const struct gpu_info_firmware_header_v1_0 *)adev->firmware.gpu_info_fw->data;
2229         amdgpu_ucode_print_gpu_info_hdr(&hdr->header);
2230
2231         switch (hdr->version_major) {
2232         case 1:
2233         {
2234                 const struct gpu_info_firmware_v1_0 *gpu_info_fw =
2235                         (const struct gpu_info_firmware_v1_0 *)(adev->firmware.gpu_info_fw->data +
2236                                                                 le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2237
2238                 /*
2239                  * Should be droped when DAL no longer needs it.
2240                  */
2241                 if (adev->asic_type == CHIP_NAVI12)
2242                         goto parse_soc_bounding_box;
2243
2244                 adev->gfx.config.max_shader_engines = le32_to_cpu(gpu_info_fw->gc_num_se);
2245                 adev->gfx.config.max_cu_per_sh = le32_to_cpu(gpu_info_fw->gc_num_cu_per_sh);
2246                 adev->gfx.config.max_sh_per_se = le32_to_cpu(gpu_info_fw->gc_num_sh_per_se);
2247                 adev->gfx.config.max_backends_per_se = le32_to_cpu(gpu_info_fw->gc_num_rb_per_se);
2248                 adev->gfx.config.max_texture_channel_caches =
2249                         le32_to_cpu(gpu_info_fw->gc_num_tccs);
2250                 adev->gfx.config.max_gprs = le32_to_cpu(gpu_info_fw->gc_num_gprs);
2251                 adev->gfx.config.max_gs_threads = le32_to_cpu(gpu_info_fw->gc_num_max_gs_thds);
2252                 adev->gfx.config.gs_vgt_table_depth = le32_to_cpu(gpu_info_fw->gc_gs_table_depth);
2253                 adev->gfx.config.gs_prim_buffer_depth = le32_to_cpu(gpu_info_fw->gc_gsprim_buff_depth);
2254                 adev->gfx.config.double_offchip_lds_buf =
2255                         le32_to_cpu(gpu_info_fw->gc_double_offchip_lds_buffer);
2256                 adev->gfx.cu_info.wave_front_size = le32_to_cpu(gpu_info_fw->gc_wave_size);
2257                 adev->gfx.cu_info.max_waves_per_simd =
2258                         le32_to_cpu(gpu_info_fw->gc_max_waves_per_simd);
2259                 adev->gfx.cu_info.max_scratch_slots_per_cu =
2260                         le32_to_cpu(gpu_info_fw->gc_max_scratch_slots_per_cu);
2261                 adev->gfx.cu_info.lds_size = le32_to_cpu(gpu_info_fw->gc_lds_size);
2262                 if (hdr->version_minor >= 1) {
2263                         const struct gpu_info_firmware_v1_1 *gpu_info_fw =
2264                                 (const struct gpu_info_firmware_v1_1 *)(adev->firmware.gpu_info_fw->data +
2265                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2266                         adev->gfx.config.num_sc_per_sh =
2267                                 le32_to_cpu(gpu_info_fw->num_sc_per_sh);
2268                         adev->gfx.config.num_packer_per_sc =
2269                                 le32_to_cpu(gpu_info_fw->num_packer_per_sc);
2270                 }
2271
2272 parse_soc_bounding_box:
2273                 /*
2274                  * soc bounding box info is not integrated in disocovery table,
2275                  * we always need to parse it from gpu info firmware if needed.
2276                  */
2277                 if (hdr->version_minor == 2) {
2278                         const struct gpu_info_firmware_v1_2 *gpu_info_fw =
2279                                 (const struct gpu_info_firmware_v1_2 *)(adev->firmware.gpu_info_fw->data +
2280                                                                         le32_to_cpu(hdr->header.ucode_array_offset_bytes));
2281                         adev->dm.soc_bounding_box = &gpu_info_fw->soc_bounding_box;
2282                 }
2283                 break;
2284         }
2285         default:
2286                 dev_err(adev->dev,
2287                         "Unsupported gpu_info table %d\n", hdr->header.ucode_version);
2288                 err = -EINVAL;
2289                 goto out;
2290         }
2291 out:
2292         return err;
2293 }
2294
2295 /**
2296  * amdgpu_device_ip_early_init - run early init for hardware IPs
2297  *
2298  * @adev: amdgpu_device pointer
2299  *
2300  * Early initialization pass for hardware IPs.  The hardware IPs that make
2301  * up each asic are discovered each IP's early_init callback is run.  This
2302  * is the first stage in initializing the asic.
2303  * Returns 0 on success, negative error code on failure.
2304  */
2305 static int amdgpu_device_ip_early_init(struct amdgpu_device *adev)
2306 {
2307         struct pci_dev *parent;
2308         int i, r;
2309         bool total;
2310
2311         amdgpu_device_enable_virtual_display(adev);
2312
2313         if (amdgpu_sriov_vf(adev)) {
2314                 r = amdgpu_virt_request_full_gpu(adev, true);
2315                 if (r)
2316                         return r;
2317         }
2318
2319         switch (adev->asic_type) {
2320 #ifdef CONFIG_DRM_AMDGPU_SI
2321         case CHIP_VERDE:
2322         case CHIP_TAHITI:
2323         case CHIP_PITCAIRN:
2324         case CHIP_OLAND:
2325         case CHIP_HAINAN:
2326                 adev->family = AMDGPU_FAMILY_SI;
2327                 r = si_set_ip_blocks(adev);
2328                 if (r)
2329                         return r;
2330                 break;
2331 #endif
2332 #ifdef CONFIG_DRM_AMDGPU_CIK
2333         case CHIP_BONAIRE:
2334         case CHIP_HAWAII:
2335         case CHIP_KAVERI:
2336         case CHIP_KABINI:
2337         case CHIP_MULLINS:
2338                 if (adev->flags & AMD_IS_APU)
2339                         adev->family = AMDGPU_FAMILY_KV;
2340                 else
2341                         adev->family = AMDGPU_FAMILY_CI;
2342
2343                 r = cik_set_ip_blocks(adev);
2344                 if (r)
2345                         return r;
2346                 break;
2347 #endif
2348         case CHIP_TOPAZ:
2349         case CHIP_TONGA:
2350         case CHIP_FIJI:
2351         case CHIP_POLARIS10:
2352         case CHIP_POLARIS11:
2353         case CHIP_POLARIS12:
2354         case CHIP_VEGAM:
2355         case CHIP_CARRIZO:
2356         case CHIP_STONEY:
2357                 if (adev->flags & AMD_IS_APU)
2358                         adev->family = AMDGPU_FAMILY_CZ;
2359                 else
2360                         adev->family = AMDGPU_FAMILY_VI;
2361
2362                 r = vi_set_ip_blocks(adev);
2363                 if (r)
2364                         return r;
2365                 break;
2366         default:
2367                 r = amdgpu_discovery_set_ip_blocks(adev);
2368                 if (r)
2369                         return r;
2370                 break;
2371         }
2372
2373         if (amdgpu_has_atpx() &&
2374             (amdgpu_is_atpx_hybrid() ||
2375              amdgpu_has_atpx_dgpu_power_cntl()) &&
2376             ((adev->flags & AMD_IS_APU) == 0) &&
2377             !dev_is_removable(&adev->pdev->dev))
2378                 adev->flags |= AMD_IS_PX;
2379
2380         if (!(adev->flags & AMD_IS_APU)) {
2381                 parent = pcie_find_root_port(adev->pdev);
2382                 adev->has_pr3 = parent ? pci_pr3_present(parent) : false;
2383         }
2384
2385
2386         adev->pm.pp_feature = amdgpu_pp_feature_mask;
2387         if (amdgpu_sriov_vf(adev) || sched_policy == KFD_SCHED_POLICY_NO_HWS)
2388                 adev->pm.pp_feature &= ~PP_GFXOFF_MASK;
2389         if (amdgpu_sriov_vf(adev) && adev->asic_type == CHIP_SIENNA_CICHLID)
2390                 adev->pm.pp_feature &= ~PP_OVERDRIVE_MASK;
2391         if (!amdgpu_device_pcie_dynamic_switching_supported())
2392                 adev->pm.pp_feature &= ~PP_PCIE_DPM_MASK;
2393
2394         total = true;
2395         for (i = 0; i < adev->num_ip_blocks; i++) {
2396                 if ((amdgpu_ip_block_mask & (1 << i)) == 0) {
2397                         DRM_WARN("disabled ip block: %d <%s>\n",
2398                                   i, adev->ip_blocks[i].version->funcs->name);
2399                         adev->ip_blocks[i].status.valid = false;
2400                 } else {
2401                         if (adev->ip_blocks[i].version->funcs->early_init) {
2402                                 r = adev->ip_blocks[i].version->funcs->early_init((void *)adev);
2403                                 if (r == -ENOENT) {
2404                                         adev->ip_blocks[i].status.valid = false;
2405                                 } else if (r) {
2406                                         DRM_ERROR("early_init of IP block <%s> failed %d\n",
2407                                                   adev->ip_blocks[i].version->funcs->name, r);
2408                                         total = false;
2409                                 } else {
2410                                         adev->ip_blocks[i].status.valid = true;
2411                                 }
2412                         } else {
2413                                 adev->ip_blocks[i].status.valid = true;
2414                         }
2415                 }
2416                 /* get the vbios after the asic_funcs are set up */
2417                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2418                         r = amdgpu_device_parse_gpu_info_fw(adev);
2419                         if (r)
2420                                 return r;
2421
2422                         /* Read BIOS */
2423                         if (amdgpu_device_read_bios(adev)) {
2424                                 if (!amdgpu_get_bios(adev))
2425                                         return -EINVAL;
2426
2427                                 r = amdgpu_atombios_init(adev);
2428                                 if (r) {
2429                                         dev_err(adev->dev, "amdgpu_atombios_init failed\n");
2430                                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_INIT_FAIL, 0, 0);
2431                                         return r;
2432                                 }
2433                         }
2434
2435                         /*get pf2vf msg info at it's earliest time*/
2436                         if (amdgpu_sriov_vf(adev))
2437                                 amdgpu_virt_init_data_exchange(adev);
2438
2439                 }
2440         }
2441         if (!total)
2442                 return -ENODEV;
2443
2444         amdgpu_amdkfd_device_probe(adev);
2445         adev->cg_flags &= amdgpu_cg_mask;
2446         adev->pg_flags &= amdgpu_pg_mask;
2447
2448         return 0;
2449 }
2450
2451 static int amdgpu_device_ip_hw_init_phase1(struct amdgpu_device *adev)
2452 {
2453         int i, r;
2454
2455         for (i = 0; i < adev->num_ip_blocks; i++) {
2456                 if (!adev->ip_blocks[i].status.sw)
2457                         continue;
2458                 if (adev->ip_blocks[i].status.hw)
2459                         continue;
2460                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
2461                     (amdgpu_sriov_vf(adev) && (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)) ||
2462                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH) {
2463                         r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2464                         if (r) {
2465                                 DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2466                                           adev->ip_blocks[i].version->funcs->name, r);
2467                                 return r;
2468                         }
2469                         adev->ip_blocks[i].status.hw = true;
2470                 }
2471         }
2472
2473         return 0;
2474 }
2475
2476 static int amdgpu_device_ip_hw_init_phase2(struct amdgpu_device *adev)
2477 {
2478         int i, r;
2479
2480         for (i = 0; i < adev->num_ip_blocks; i++) {
2481                 if (!adev->ip_blocks[i].status.sw)
2482                         continue;
2483                 if (adev->ip_blocks[i].status.hw)
2484                         continue;
2485                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2486                 if (r) {
2487                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2488                                   adev->ip_blocks[i].version->funcs->name, r);
2489                         return r;
2490                 }
2491                 adev->ip_blocks[i].status.hw = true;
2492         }
2493
2494         return 0;
2495 }
2496
2497 static int amdgpu_device_fw_loading(struct amdgpu_device *adev)
2498 {
2499         int r = 0;
2500         int i;
2501         uint32_t smu_version;
2502
2503         if (adev->asic_type >= CHIP_VEGA10) {
2504                 for (i = 0; i < adev->num_ip_blocks; i++) {
2505                         if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_PSP)
2506                                 continue;
2507
2508                         if (!adev->ip_blocks[i].status.sw)
2509                                 continue;
2510
2511                         /* no need to do the fw loading again if already done*/
2512                         if (adev->ip_blocks[i].status.hw == true)
2513                                 break;
2514
2515                         if (amdgpu_in_reset(adev) || adev->in_suspend) {
2516                                 r = adev->ip_blocks[i].version->funcs->resume(adev);
2517                                 if (r) {
2518                                         DRM_ERROR("resume of IP block <%s> failed %d\n",
2519                                                           adev->ip_blocks[i].version->funcs->name, r);
2520                                         return r;
2521                                 }
2522                         } else {
2523                                 r = adev->ip_blocks[i].version->funcs->hw_init(adev);
2524                                 if (r) {
2525                                         DRM_ERROR("hw_init of IP block <%s> failed %d\n",
2526                                                           adev->ip_blocks[i].version->funcs->name, r);
2527                                         return r;
2528                                 }
2529                         }
2530
2531                         adev->ip_blocks[i].status.hw = true;
2532                         break;
2533                 }
2534         }
2535
2536         if (!amdgpu_sriov_vf(adev) || adev->asic_type == CHIP_TONGA)
2537                 r = amdgpu_pm_load_smu_firmware(adev, &smu_version);
2538
2539         return r;
2540 }
2541
2542 static int amdgpu_device_init_schedulers(struct amdgpu_device *adev)
2543 {
2544         long timeout;
2545         int r, i;
2546
2547         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
2548                 struct amdgpu_ring *ring = adev->rings[i];
2549
2550                 /* No need to setup the GPU scheduler for rings that don't need it */
2551                 if (!ring || ring->no_scheduler)
2552                         continue;
2553
2554                 switch (ring->funcs->type) {
2555                 case AMDGPU_RING_TYPE_GFX:
2556                         timeout = adev->gfx_timeout;
2557                         break;
2558                 case AMDGPU_RING_TYPE_COMPUTE:
2559                         timeout = adev->compute_timeout;
2560                         break;
2561                 case AMDGPU_RING_TYPE_SDMA:
2562                         timeout = adev->sdma_timeout;
2563                         break;
2564                 default:
2565                         timeout = adev->video_timeout;
2566                         break;
2567                 }
2568
2569                 r = drm_sched_init(&ring->sched, &amdgpu_sched_ops,
2570                                    DRM_SCHED_PRIORITY_COUNT,
2571                                    ring->num_hw_submission, 0,
2572                                    timeout, adev->reset_domain->wq,
2573                                    ring->sched_score, ring->name,
2574                                    adev->dev);
2575                 if (r) {
2576                         DRM_ERROR("Failed to create scheduler on ring %s.\n",
2577                                   ring->name);
2578                         return r;
2579                 }
2580                 r = amdgpu_uvd_entity_init(adev, ring);
2581                 if (r) {
2582                         DRM_ERROR("Failed to create UVD scheduling entity on ring %s.\n",
2583                                   ring->name);
2584                         return r;
2585                 }
2586                 r = amdgpu_vce_entity_init(adev, ring);
2587                 if (r) {
2588                         DRM_ERROR("Failed to create VCE scheduling entity on ring %s.\n",
2589                                   ring->name);
2590                         return r;
2591                 }
2592         }
2593
2594         amdgpu_xcp_update_partition_sched_list(adev);
2595
2596         return 0;
2597 }
2598
2599
2600 /**
2601  * amdgpu_device_ip_init - run init for hardware IPs
2602  *
2603  * @adev: amdgpu_device pointer
2604  *
2605  * Main initialization pass for hardware IPs.  The list of all the hardware
2606  * IPs that make up the asic is walked and the sw_init and hw_init callbacks
2607  * are run.  sw_init initializes the software state associated with each IP
2608  * and hw_init initializes the hardware associated with each IP.
2609  * Returns 0 on success, negative error code on failure.
2610  */
2611 static int amdgpu_device_ip_init(struct amdgpu_device *adev)
2612 {
2613         int i, r;
2614
2615         r = amdgpu_ras_init(adev);
2616         if (r)
2617                 return r;
2618
2619         for (i = 0; i < adev->num_ip_blocks; i++) {
2620                 if (!adev->ip_blocks[i].status.valid)
2621                         continue;
2622                 r = adev->ip_blocks[i].version->funcs->sw_init((void *)adev);
2623                 if (r) {
2624                         DRM_ERROR("sw_init of IP block <%s> failed %d\n",
2625                                   adev->ip_blocks[i].version->funcs->name, r);
2626                         goto init_failed;
2627                 }
2628                 adev->ip_blocks[i].status.sw = true;
2629
2630                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON) {
2631                         /* need to do common hw init early so everything is set up for gmc */
2632                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2633                         if (r) {
2634                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2635                                 goto init_failed;
2636                         }
2637                         adev->ip_blocks[i].status.hw = true;
2638                 } else if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
2639                         /* need to do gmc hw init early so we can allocate gpu mem */
2640                         /* Try to reserve bad pages early */
2641                         if (amdgpu_sriov_vf(adev))
2642                                 amdgpu_virt_exchange_data(adev);
2643
2644                         r = amdgpu_device_mem_scratch_init(adev);
2645                         if (r) {
2646                                 DRM_ERROR("amdgpu_mem_scratch_init failed %d\n", r);
2647                                 goto init_failed;
2648                         }
2649                         r = adev->ip_blocks[i].version->funcs->hw_init((void *)adev);
2650                         if (r) {
2651                                 DRM_ERROR("hw_init %d failed %d\n", i, r);
2652                                 goto init_failed;
2653                         }
2654                         r = amdgpu_device_wb_init(adev);
2655                         if (r) {
2656                                 DRM_ERROR("amdgpu_device_wb_init failed %d\n", r);
2657                                 goto init_failed;
2658                         }
2659                         adev->ip_blocks[i].status.hw = true;
2660
2661                         /* right after GMC hw init, we create CSA */
2662                         if (adev->gfx.mcbp) {
2663                                 r = amdgpu_allocate_static_csa(adev, &adev->virt.csa_obj,
2664                                                                AMDGPU_GEM_DOMAIN_VRAM |
2665                                                                AMDGPU_GEM_DOMAIN_GTT,
2666                                                                AMDGPU_CSA_SIZE);
2667                                 if (r) {
2668                                         DRM_ERROR("allocate CSA failed %d\n", r);
2669                                         goto init_failed;
2670                                 }
2671                         }
2672                 }
2673         }
2674
2675         if (amdgpu_sriov_vf(adev))
2676                 amdgpu_virt_init_data_exchange(adev);
2677
2678         r = amdgpu_ib_pool_init(adev);
2679         if (r) {
2680                 dev_err(adev->dev, "IB initialization failed (%d).\n", r);
2681                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_IB_INIT_FAIL, 0, r);
2682                 goto init_failed;
2683         }
2684
2685         r = amdgpu_ucode_create_bo(adev); /* create ucode bo when sw_init complete*/
2686         if (r)
2687                 goto init_failed;
2688
2689         r = amdgpu_device_ip_hw_init_phase1(adev);
2690         if (r)
2691                 goto init_failed;
2692
2693         r = amdgpu_device_fw_loading(adev);
2694         if (r)
2695                 goto init_failed;
2696
2697         r = amdgpu_device_ip_hw_init_phase2(adev);
2698         if (r)
2699                 goto init_failed;
2700
2701         /*
2702          * retired pages will be loaded from eeprom and reserved here,
2703          * it should be called after amdgpu_device_ip_hw_init_phase2  since
2704          * for some ASICs the RAS EEPROM code relies on SMU fully functioning
2705          * for I2C communication which only true at this point.
2706          *
2707          * amdgpu_ras_recovery_init may fail, but the upper only cares the
2708          * failure from bad gpu situation and stop amdgpu init process
2709          * accordingly. For other failed cases, it will still release all
2710          * the resource and print error message, rather than returning one
2711          * negative value to upper level.
2712          *
2713          * Note: theoretically, this should be called before all vram allocations
2714          * to protect retired page from abusing
2715          */
2716         r = amdgpu_ras_recovery_init(adev);
2717         if (r)
2718                 goto init_failed;
2719
2720         /**
2721          * In case of XGMI grab extra reference for reset domain for this device
2722          */
2723         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2724                 if (amdgpu_xgmi_add_device(adev) == 0) {
2725                         if (!amdgpu_sriov_vf(adev)) {
2726                                 struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
2727
2728                                 if (WARN_ON(!hive)) {
2729                                         r = -ENOENT;
2730                                         goto init_failed;
2731                                 }
2732
2733                                 if (!hive->reset_domain ||
2734                                     !amdgpu_reset_get_reset_domain(hive->reset_domain)) {
2735                                         r = -ENOENT;
2736                                         amdgpu_put_xgmi_hive(hive);
2737                                         goto init_failed;
2738                                 }
2739
2740                                 /* Drop the early temporary reset domain we created for device */
2741                                 amdgpu_reset_put_reset_domain(adev->reset_domain);
2742                                 adev->reset_domain = hive->reset_domain;
2743                                 amdgpu_put_xgmi_hive(hive);
2744                         }
2745                 }
2746         }
2747
2748         r = amdgpu_device_init_schedulers(adev);
2749         if (r)
2750                 goto init_failed;
2751
2752         if (adev->mman.buffer_funcs_ring->sched.ready)
2753                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
2754
2755         /* Don't init kfd if whole hive need to be reset during init */
2756         if (!adev->gmc.xgmi.pending_reset) {
2757                 kgd2kfd_init_zone_device(adev);
2758                 amdgpu_amdkfd_device_init(adev);
2759         }
2760
2761         amdgpu_fru_get_product_info(adev);
2762
2763 init_failed:
2764
2765         return r;
2766 }
2767
2768 /**
2769  * amdgpu_device_fill_reset_magic - writes reset magic to gart pointer
2770  *
2771  * @adev: amdgpu_device pointer
2772  *
2773  * Writes a reset magic value to the gart pointer in VRAM.  The driver calls
2774  * this function before a GPU reset.  If the value is retained after a
2775  * GPU reset, VRAM has not been lost.  Some GPU resets may destry VRAM contents.
2776  */
2777 static void amdgpu_device_fill_reset_magic(struct amdgpu_device *adev)
2778 {
2779         memcpy(adev->reset_magic, adev->gart.ptr, AMDGPU_RESET_MAGIC_NUM);
2780 }
2781
2782 /**
2783  * amdgpu_device_check_vram_lost - check if vram is valid
2784  *
2785  * @adev: amdgpu_device pointer
2786  *
2787  * Checks the reset magic value written to the gart pointer in VRAM.
2788  * The driver calls this after a GPU reset to see if the contents of
2789  * VRAM is lost or now.
2790  * returns true if vram is lost, false if not.
2791  */
2792 static bool amdgpu_device_check_vram_lost(struct amdgpu_device *adev)
2793 {
2794         if (memcmp(adev->gart.ptr, adev->reset_magic,
2795                         AMDGPU_RESET_MAGIC_NUM))
2796                 return true;
2797
2798         if (!amdgpu_in_reset(adev))
2799                 return false;
2800
2801         /*
2802          * For all ASICs with baco/mode1 reset, the VRAM is
2803          * always assumed to be lost.
2804          */
2805         switch (amdgpu_asic_reset_method(adev)) {
2806         case AMD_RESET_METHOD_BACO:
2807         case AMD_RESET_METHOD_MODE1:
2808                 return true;
2809         default:
2810                 return false;
2811         }
2812 }
2813
2814 /**
2815  * amdgpu_device_set_cg_state - set clockgating for amdgpu device
2816  *
2817  * @adev: amdgpu_device pointer
2818  * @state: clockgating state (gate or ungate)
2819  *
2820  * The list of all the hardware IPs that make up the asic is walked and the
2821  * set_clockgating_state callbacks are run.
2822  * Late initialization pass enabling clockgating for hardware IPs.
2823  * Fini or suspend, pass disabling clockgating for hardware IPs.
2824  * Returns 0 on success, negative error code on failure.
2825  */
2826
2827 int amdgpu_device_set_cg_state(struct amdgpu_device *adev,
2828                                enum amd_clockgating_state state)
2829 {
2830         int i, j, r;
2831
2832         if (amdgpu_emu_mode == 1)
2833                 return 0;
2834
2835         for (j = 0; j < adev->num_ip_blocks; j++) {
2836                 i = state == AMD_CG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2837                 if (!adev->ip_blocks[i].status.late_initialized)
2838                         continue;
2839                 /* skip CG for GFX, SDMA on S0ix */
2840                 if (adev->in_s0ix &&
2841                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2842                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2843                         continue;
2844                 /* skip CG for VCE/UVD, it's handled specially */
2845                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2846                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2847                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2848                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2849                     adev->ip_blocks[i].version->funcs->set_clockgating_state) {
2850                         /* enable clockgating to save power */
2851                         r = adev->ip_blocks[i].version->funcs->set_clockgating_state((void *)adev,
2852                                                                                      state);
2853                         if (r) {
2854                                 DRM_ERROR("set_clockgating_state(gate) of IP block <%s> failed %d\n",
2855                                           adev->ip_blocks[i].version->funcs->name, r);
2856                                 return r;
2857                         }
2858                 }
2859         }
2860
2861         return 0;
2862 }
2863
2864 int amdgpu_device_set_pg_state(struct amdgpu_device *adev,
2865                                enum amd_powergating_state state)
2866 {
2867         int i, j, r;
2868
2869         if (amdgpu_emu_mode == 1)
2870                 return 0;
2871
2872         for (j = 0; j < adev->num_ip_blocks; j++) {
2873                 i = state == AMD_PG_STATE_GATE ? j : adev->num_ip_blocks - j - 1;
2874                 if (!adev->ip_blocks[i].status.late_initialized)
2875                         continue;
2876                 /* skip PG for GFX, SDMA on S0ix */
2877                 if (adev->in_s0ix &&
2878                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
2879                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SDMA))
2880                         continue;
2881                 /* skip CG for VCE/UVD, it's handled specially */
2882                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_UVD &&
2883                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCE &&
2884                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_VCN &&
2885                     adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_JPEG &&
2886                     adev->ip_blocks[i].version->funcs->set_powergating_state) {
2887                         /* enable powergating to save power */
2888                         r = adev->ip_blocks[i].version->funcs->set_powergating_state((void *)adev,
2889                                                                                         state);
2890                         if (r) {
2891                                 DRM_ERROR("set_powergating_state(gate) of IP block <%s> failed %d\n",
2892                                           adev->ip_blocks[i].version->funcs->name, r);
2893                                 return r;
2894                         }
2895                 }
2896         }
2897         return 0;
2898 }
2899
2900 static int amdgpu_device_enable_mgpu_fan_boost(void)
2901 {
2902         struct amdgpu_gpu_instance *gpu_ins;
2903         struct amdgpu_device *adev;
2904         int i, ret = 0;
2905
2906         mutex_lock(&mgpu_info.mutex);
2907
2908         /*
2909          * MGPU fan boost feature should be enabled
2910          * only when there are two or more dGPUs in
2911          * the system
2912          */
2913         if (mgpu_info.num_dgpu < 2)
2914                 goto out;
2915
2916         for (i = 0; i < mgpu_info.num_dgpu; i++) {
2917                 gpu_ins = &(mgpu_info.gpu_ins[i]);
2918                 adev = gpu_ins->adev;
2919                 if (!(adev->flags & AMD_IS_APU) &&
2920                     !gpu_ins->mgpu_fan_enabled) {
2921                         ret = amdgpu_dpm_enable_mgpu_fan_boost(adev);
2922                         if (ret)
2923                                 break;
2924
2925                         gpu_ins->mgpu_fan_enabled = 1;
2926                 }
2927         }
2928
2929 out:
2930         mutex_unlock(&mgpu_info.mutex);
2931
2932         return ret;
2933 }
2934
2935 /**
2936  * amdgpu_device_ip_late_init - run late init for hardware IPs
2937  *
2938  * @adev: amdgpu_device pointer
2939  *
2940  * Late initialization pass for hardware IPs.  The list of all the hardware
2941  * IPs that make up the asic is walked and the late_init callbacks are run.
2942  * late_init covers any special initialization that an IP requires
2943  * after all of the have been initialized or something that needs to happen
2944  * late in the init process.
2945  * Returns 0 on success, negative error code on failure.
2946  */
2947 static int amdgpu_device_ip_late_init(struct amdgpu_device *adev)
2948 {
2949         struct amdgpu_gpu_instance *gpu_instance;
2950         int i = 0, r;
2951
2952         for (i = 0; i < adev->num_ip_blocks; i++) {
2953                 if (!adev->ip_blocks[i].status.hw)
2954                         continue;
2955                 if (adev->ip_blocks[i].version->funcs->late_init) {
2956                         r = adev->ip_blocks[i].version->funcs->late_init((void *)adev);
2957                         if (r) {
2958                                 DRM_ERROR("late_init of IP block <%s> failed %d\n",
2959                                           adev->ip_blocks[i].version->funcs->name, r);
2960                                 return r;
2961                         }
2962                 }
2963                 adev->ip_blocks[i].status.late_initialized = true;
2964         }
2965
2966         r = amdgpu_ras_late_init(adev);
2967         if (r) {
2968                 DRM_ERROR("amdgpu_ras_late_init failed %d", r);
2969                 return r;
2970         }
2971
2972         amdgpu_ras_set_error_query_ready(adev, true);
2973
2974         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_GATE);
2975         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_GATE);
2976
2977         amdgpu_device_fill_reset_magic(adev);
2978
2979         r = amdgpu_device_enable_mgpu_fan_boost();
2980         if (r)
2981                 DRM_ERROR("enable mgpu fan boost failed (%d).\n", r);
2982
2983         /* For passthrough configuration on arcturus and aldebaran, enable special handling SBR */
2984         if (amdgpu_passthrough(adev) &&
2985             ((adev->asic_type == CHIP_ARCTURUS && adev->gmc.xgmi.num_physical_nodes > 1) ||
2986              adev->asic_type == CHIP_ALDEBARAN))
2987                 amdgpu_dpm_handle_passthrough_sbr(adev, true);
2988
2989         if (adev->gmc.xgmi.num_physical_nodes > 1) {
2990                 mutex_lock(&mgpu_info.mutex);
2991
2992                 /*
2993                  * Reset device p-state to low as this was booted with high.
2994                  *
2995                  * This should be performed only after all devices from the same
2996                  * hive get initialized.
2997                  *
2998                  * However, it's unknown how many device in the hive in advance.
2999                  * As this is counted one by one during devices initializations.
3000                  *
3001                  * So, we wait for all XGMI interlinked devices initialized.
3002                  * This may bring some delays as those devices may come from
3003                  * different hives. But that should be OK.
3004                  */
3005                 if (mgpu_info.num_dgpu == adev->gmc.xgmi.num_physical_nodes) {
3006                         for (i = 0; i < mgpu_info.num_gpu; i++) {
3007                                 gpu_instance = &(mgpu_info.gpu_ins[i]);
3008                                 if (gpu_instance->adev->flags & AMD_IS_APU)
3009                                         continue;
3010
3011                                 r = amdgpu_xgmi_set_pstate(gpu_instance->adev,
3012                                                 AMDGPU_XGMI_PSTATE_MIN);
3013                                 if (r) {
3014                                         DRM_ERROR("pstate setting failed (%d).\n", r);
3015                                         break;
3016                                 }
3017                         }
3018                 }
3019
3020                 mutex_unlock(&mgpu_info.mutex);
3021         }
3022
3023         return 0;
3024 }
3025
3026 /**
3027  * amdgpu_device_smu_fini_early - smu hw_fini wrapper
3028  *
3029  * @adev: amdgpu_device pointer
3030  *
3031  * For ASICs need to disable SMC first
3032  */
3033 static void amdgpu_device_smu_fini_early(struct amdgpu_device *adev)
3034 {
3035         int i, r;
3036
3037         if (amdgpu_ip_version(adev, GC_HWIP, 0) > IP_VERSION(9, 0, 0))
3038                 return;
3039
3040         for (i = 0; i < adev->num_ip_blocks; i++) {
3041                 if (!adev->ip_blocks[i].status.hw)
3042                         continue;
3043                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3044                         r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3045                         /* XXX handle errors */
3046                         if (r) {
3047                                 DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3048                                           adev->ip_blocks[i].version->funcs->name, r);
3049                         }
3050                         adev->ip_blocks[i].status.hw = false;
3051                         break;
3052                 }
3053         }
3054 }
3055
3056 static int amdgpu_device_ip_fini_early(struct amdgpu_device *adev)
3057 {
3058         int i, r;
3059
3060         for (i = 0; i < adev->num_ip_blocks; i++) {
3061                 if (!adev->ip_blocks[i].version->funcs->early_fini)
3062                         continue;
3063
3064                 r = adev->ip_blocks[i].version->funcs->early_fini((void *)adev);
3065                 if (r) {
3066                         DRM_DEBUG("early_fini of IP block <%s> failed %d\n",
3067                                   adev->ip_blocks[i].version->funcs->name, r);
3068                 }
3069         }
3070
3071         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3072         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3073
3074         amdgpu_amdkfd_suspend(adev, false);
3075
3076         /* Workaroud for ASICs need to disable SMC first */
3077         amdgpu_device_smu_fini_early(adev);
3078
3079         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3080                 if (!adev->ip_blocks[i].status.hw)
3081                         continue;
3082
3083                 r = adev->ip_blocks[i].version->funcs->hw_fini((void *)adev);
3084                 /* XXX handle errors */
3085                 if (r) {
3086                         DRM_DEBUG("hw_fini of IP block <%s> failed %d\n",
3087                                   adev->ip_blocks[i].version->funcs->name, r);
3088                 }
3089
3090                 adev->ip_blocks[i].status.hw = false;
3091         }
3092
3093         if (amdgpu_sriov_vf(adev)) {
3094                 if (amdgpu_virt_release_full_gpu(adev, false))
3095                         DRM_ERROR("failed to release exclusive mode on fini\n");
3096         }
3097
3098         return 0;
3099 }
3100
3101 /**
3102  * amdgpu_device_ip_fini - run fini for hardware IPs
3103  *
3104  * @adev: amdgpu_device pointer
3105  *
3106  * Main teardown pass for hardware IPs.  The list of all the hardware
3107  * IPs that make up the asic is walked and the hw_fini and sw_fini callbacks
3108  * are run.  hw_fini tears down the hardware associated with each IP
3109  * and sw_fini tears down any software state associated with each IP.
3110  * Returns 0 on success, negative error code on failure.
3111  */
3112 static int amdgpu_device_ip_fini(struct amdgpu_device *adev)
3113 {
3114         int i, r;
3115
3116         if (amdgpu_sriov_vf(adev) && adev->virt.ras_init_done)
3117                 amdgpu_virt_release_ras_err_handler_data(adev);
3118
3119         if (adev->gmc.xgmi.num_physical_nodes > 1)
3120                 amdgpu_xgmi_remove_device(adev);
3121
3122         amdgpu_amdkfd_device_fini_sw(adev);
3123
3124         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3125                 if (!adev->ip_blocks[i].status.sw)
3126                         continue;
3127
3128                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) {
3129                         amdgpu_ucode_free_bo(adev);
3130                         amdgpu_free_static_csa(&adev->virt.csa_obj);
3131                         amdgpu_device_wb_fini(adev);
3132                         amdgpu_device_mem_scratch_fini(adev);
3133                         amdgpu_ib_pool_fini(adev);
3134                 }
3135
3136                 r = adev->ip_blocks[i].version->funcs->sw_fini((void *)adev);
3137                 /* XXX handle errors */
3138                 if (r) {
3139                         DRM_DEBUG("sw_fini of IP block <%s> failed %d\n",
3140                                   adev->ip_blocks[i].version->funcs->name, r);
3141                 }
3142                 adev->ip_blocks[i].status.sw = false;
3143                 adev->ip_blocks[i].status.valid = false;
3144         }
3145
3146         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3147                 if (!adev->ip_blocks[i].status.late_initialized)
3148                         continue;
3149                 if (adev->ip_blocks[i].version->funcs->late_fini)
3150                         adev->ip_blocks[i].version->funcs->late_fini((void *)adev);
3151                 adev->ip_blocks[i].status.late_initialized = false;
3152         }
3153
3154         amdgpu_ras_fini(adev);
3155
3156         return 0;
3157 }
3158
3159 /**
3160  * amdgpu_device_delayed_init_work_handler - work handler for IB tests
3161  *
3162  * @work: work_struct.
3163  */
3164 static void amdgpu_device_delayed_init_work_handler(struct work_struct *work)
3165 {
3166         struct amdgpu_device *adev =
3167                 container_of(work, struct amdgpu_device, delayed_init_work.work);
3168         int r;
3169
3170         r = amdgpu_ib_ring_tests(adev);
3171         if (r)
3172                 DRM_ERROR("ib ring test failed (%d).\n", r);
3173 }
3174
3175 static void amdgpu_device_delay_enable_gfx_off(struct work_struct *work)
3176 {
3177         struct amdgpu_device *adev =
3178                 container_of(work, struct amdgpu_device, gfx.gfx_off_delay_work.work);
3179
3180         WARN_ON_ONCE(adev->gfx.gfx_off_state);
3181         WARN_ON_ONCE(adev->gfx.gfx_off_req_count);
3182
3183         if (!amdgpu_dpm_set_powergating_by_smu(adev, AMD_IP_BLOCK_TYPE_GFX, true))
3184                 adev->gfx.gfx_off_state = true;
3185 }
3186
3187 /**
3188  * amdgpu_device_ip_suspend_phase1 - run suspend for hardware IPs (phase 1)
3189  *
3190  * @adev: amdgpu_device pointer
3191  *
3192  * Main suspend function for hardware IPs.  The list of all the hardware
3193  * IPs that make up the asic is walked, clockgating is disabled and the
3194  * suspend callbacks are run.  suspend puts the hardware and software state
3195  * in each IP into a state suitable for suspend.
3196  * Returns 0 on success, negative error code on failure.
3197  */
3198 static int amdgpu_device_ip_suspend_phase1(struct amdgpu_device *adev)
3199 {
3200         int i, r;
3201
3202         amdgpu_device_set_pg_state(adev, AMD_PG_STATE_UNGATE);
3203         amdgpu_device_set_cg_state(adev, AMD_CG_STATE_UNGATE);
3204
3205         /*
3206          * Per PMFW team's suggestion, driver needs to handle gfxoff
3207          * and df cstate features disablement for gpu reset(e.g. Mode1Reset)
3208          * scenario. Add the missing df cstate disablement here.
3209          */
3210         if (amdgpu_dpm_set_df_cstate(adev, DF_CSTATE_DISALLOW))
3211                 dev_warn(adev->dev, "Failed to disallow df cstate");
3212
3213         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3214                 if (!adev->ip_blocks[i].status.valid)
3215                         continue;
3216
3217                 /* displays are handled separately */
3218                 if (adev->ip_blocks[i].version->type != AMD_IP_BLOCK_TYPE_DCE)
3219                         continue;
3220
3221                 /* XXX handle errors */
3222                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3223                 /* XXX handle errors */
3224                 if (r) {
3225                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3226                                   adev->ip_blocks[i].version->funcs->name, r);
3227                         return r;
3228                 }
3229
3230                 adev->ip_blocks[i].status.hw = false;
3231         }
3232
3233         return 0;
3234 }
3235
3236 /**
3237  * amdgpu_device_ip_suspend_phase2 - run suspend for hardware IPs (phase 2)
3238  *
3239  * @adev: amdgpu_device pointer
3240  *
3241  * Main suspend function for hardware IPs.  The list of all the hardware
3242  * IPs that make up the asic is walked, clockgating is disabled and the
3243  * suspend callbacks are run.  suspend puts the hardware and software state
3244  * in each IP into a state suitable for suspend.
3245  * Returns 0 on success, negative error code on failure.
3246  */
3247 static int amdgpu_device_ip_suspend_phase2(struct amdgpu_device *adev)
3248 {
3249         int i, r;
3250
3251         if (adev->in_s0ix)
3252                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D3Entry);
3253
3254         for (i = adev->num_ip_blocks - 1; i >= 0; i--) {
3255                 if (!adev->ip_blocks[i].status.valid)
3256                         continue;
3257                 /* displays are handled in phase1 */
3258                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE)
3259                         continue;
3260                 /* PSP lost connection when err_event_athub occurs */
3261                 if (amdgpu_ras_intr_triggered() &&
3262                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
3263                         adev->ip_blocks[i].status.hw = false;
3264                         continue;
3265                 }
3266
3267                 /* skip unnecessary suspend if we do not initialize them yet */
3268                 if (adev->gmc.xgmi.pending_reset &&
3269                     !(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3270                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC ||
3271                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3272                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH)) {
3273                         adev->ip_blocks[i].status.hw = false;
3274                         continue;
3275                 }
3276
3277                 /* skip suspend of gfx/mes and psp for S0ix
3278                  * gfx is in gfxoff state, so on resume it will exit gfxoff just
3279                  * like at runtime. PSP is also part of the always on hardware
3280                  * so no need to suspend it.
3281                  */
3282                 if (adev->in_s0ix &&
3283                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP ||
3284                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GFX ||
3285                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_MES))
3286                         continue;
3287
3288                 /* SDMA 5.x+ is part of GFX power domain so it's covered by GFXOFF */
3289                 if (adev->in_s0ix &&
3290                     (amdgpu_ip_version(adev, SDMA0_HWIP, 0) >=
3291                      IP_VERSION(5, 0, 0)) &&
3292                     (adev->ip_blocks[i].version->type ==
3293                      AMD_IP_BLOCK_TYPE_SDMA))
3294                         continue;
3295
3296                 /* Once swPSP provides the IMU, RLC FW binaries to TOS during cold-boot.
3297                  * These are in TMR, hence are expected to be reused by PSP-TOS to reload
3298                  * from this location and RLC Autoload automatically also gets loaded
3299                  * from here based on PMFW -> PSP message during re-init sequence.
3300                  * Therefore, the psp suspend & resume should be skipped to avoid destroy
3301                  * the TMR and reload FWs again for IMU enabled APU ASICs.
3302                  */
3303                 if (amdgpu_in_reset(adev) &&
3304                     (adev->flags & AMD_IS_APU) && adev->gfx.imu.funcs &&
3305                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3306                         continue;
3307
3308                 /* XXX handle errors */
3309                 r = adev->ip_blocks[i].version->funcs->suspend(adev);
3310                 /* XXX handle errors */
3311                 if (r) {
3312                         DRM_ERROR("suspend of IP block <%s> failed %d\n",
3313                                   adev->ip_blocks[i].version->funcs->name, r);
3314                 }
3315                 adev->ip_blocks[i].status.hw = false;
3316                 /* handle putting the SMC in the appropriate state */
3317                 if (!amdgpu_sriov_vf(adev)) {
3318                         if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) {
3319                                 r = amdgpu_dpm_set_mp1_state(adev, adev->mp1_state);
3320                                 if (r) {
3321                                         DRM_ERROR("SMC failed to set mp1 state %d, %d\n",
3322                                                         adev->mp1_state, r);
3323                                         return r;
3324                                 }
3325                         }
3326                 }
3327         }
3328
3329         return 0;
3330 }
3331
3332 /**
3333  * amdgpu_device_ip_suspend - run suspend for hardware IPs
3334  *
3335  * @adev: amdgpu_device pointer
3336  *
3337  * Main suspend function for hardware IPs.  The list of all the hardware
3338  * IPs that make up the asic is walked, clockgating is disabled and the
3339  * suspend callbacks are run.  suspend puts the hardware and software state
3340  * in each IP into a state suitable for suspend.
3341  * Returns 0 on success, negative error code on failure.
3342  */
3343 int amdgpu_device_ip_suspend(struct amdgpu_device *adev)
3344 {
3345         int r;
3346
3347         if (amdgpu_sriov_vf(adev)) {
3348                 amdgpu_virt_fini_data_exchange(adev);
3349                 amdgpu_virt_request_full_gpu(adev, false);
3350         }
3351
3352         amdgpu_ttm_set_buffer_funcs_status(adev, false);
3353
3354         r = amdgpu_device_ip_suspend_phase1(adev);
3355         if (r)
3356                 return r;
3357         r = amdgpu_device_ip_suspend_phase2(adev);
3358
3359         if (amdgpu_sriov_vf(adev))
3360                 amdgpu_virt_release_full_gpu(adev, false);
3361
3362         return r;
3363 }
3364
3365 static int amdgpu_device_ip_reinit_early_sriov(struct amdgpu_device *adev)
3366 {
3367         int i, r;
3368
3369         static enum amd_ip_block_type ip_order[] = {
3370                 AMD_IP_BLOCK_TYPE_COMMON,
3371                 AMD_IP_BLOCK_TYPE_GMC,
3372                 AMD_IP_BLOCK_TYPE_PSP,
3373                 AMD_IP_BLOCK_TYPE_IH,
3374         };
3375
3376         for (i = 0; i < adev->num_ip_blocks; i++) {
3377                 int j;
3378                 struct amdgpu_ip_block *block;
3379
3380                 block = &adev->ip_blocks[i];
3381                 block->status.hw = false;
3382
3383                 for (j = 0; j < ARRAY_SIZE(ip_order); j++) {
3384
3385                         if (block->version->type != ip_order[j] ||
3386                                 !block->status.valid)
3387                                 continue;
3388
3389                         r = block->version->funcs->hw_init(adev);
3390                         DRM_INFO("RE-INIT-early: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3391                         if (r)
3392                                 return r;
3393                         block->status.hw = true;
3394                 }
3395         }
3396
3397         return 0;
3398 }
3399
3400 static int amdgpu_device_ip_reinit_late_sriov(struct amdgpu_device *adev)
3401 {
3402         int i, r;
3403
3404         static enum amd_ip_block_type ip_order[] = {
3405                 AMD_IP_BLOCK_TYPE_SMC,
3406                 AMD_IP_BLOCK_TYPE_DCE,
3407                 AMD_IP_BLOCK_TYPE_GFX,
3408                 AMD_IP_BLOCK_TYPE_SDMA,
3409                 AMD_IP_BLOCK_TYPE_MES,
3410                 AMD_IP_BLOCK_TYPE_UVD,
3411                 AMD_IP_BLOCK_TYPE_VCE,
3412                 AMD_IP_BLOCK_TYPE_VCN,
3413                 AMD_IP_BLOCK_TYPE_JPEG
3414         };
3415
3416         for (i = 0; i < ARRAY_SIZE(ip_order); i++) {
3417                 int j;
3418                 struct amdgpu_ip_block *block;
3419
3420                 for (j = 0; j < adev->num_ip_blocks; j++) {
3421                         block = &adev->ip_blocks[j];
3422
3423                         if (block->version->type != ip_order[i] ||
3424                                 !block->status.valid ||
3425                                 block->status.hw)
3426                                 continue;
3427
3428                         if (block->version->type == AMD_IP_BLOCK_TYPE_SMC)
3429                                 r = block->version->funcs->resume(adev);
3430                         else
3431                                 r = block->version->funcs->hw_init(adev);
3432
3433                         DRM_INFO("RE-INIT-late: %s %s\n", block->version->funcs->name, r?"failed":"succeeded");
3434                         if (r)
3435                                 return r;
3436                         block->status.hw = true;
3437                 }
3438         }
3439
3440         return 0;
3441 }
3442
3443 /**
3444  * amdgpu_device_ip_resume_phase1 - run resume for hardware IPs
3445  *
3446  * @adev: amdgpu_device pointer
3447  *
3448  * First resume function for hardware IPs.  The list of all the hardware
3449  * IPs that make up the asic is walked and the resume callbacks are run for
3450  * COMMON, GMC, and IH.  resume puts the hardware into a functional state
3451  * after a suspend and updates the software state as necessary.  This
3452  * function is also used for restoring the GPU after a GPU reset.
3453  * Returns 0 on success, negative error code on failure.
3454  */
3455 static int amdgpu_device_ip_resume_phase1(struct amdgpu_device *adev)
3456 {
3457         int i, r;
3458
3459         for (i = 0; i < adev->num_ip_blocks; i++) {
3460                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3461                         continue;
3462                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3463                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3464                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3465                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP && amdgpu_sriov_vf(adev))) {
3466
3467                         r = adev->ip_blocks[i].version->funcs->resume(adev);
3468                         if (r) {
3469                                 DRM_ERROR("resume of IP block <%s> failed %d\n",
3470                                           adev->ip_blocks[i].version->funcs->name, r);
3471                                 return r;
3472                         }
3473                         adev->ip_blocks[i].status.hw = true;
3474                 }
3475         }
3476
3477         return 0;
3478 }
3479
3480 /**
3481  * amdgpu_device_ip_resume_phase2 - run resume for hardware IPs
3482  *
3483  * @adev: amdgpu_device pointer
3484  *
3485  * First resume function for hardware IPs.  The list of all the hardware
3486  * IPs that make up the asic is walked and the resume callbacks are run for
3487  * all blocks except COMMON, GMC, and IH.  resume puts the hardware into a
3488  * functional state after a suspend and updates the software state as
3489  * necessary.  This function is also used for restoring the GPU after a GPU
3490  * reset.
3491  * Returns 0 on success, negative error code on failure.
3492  */
3493 static int amdgpu_device_ip_resume_phase2(struct amdgpu_device *adev)
3494 {
3495         int i, r;
3496
3497         for (i = 0; i < adev->num_ip_blocks; i++) {
3498                 if (!adev->ip_blocks[i].status.valid || adev->ip_blocks[i].status.hw)
3499                         continue;
3500                 if (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
3501                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
3502                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
3503                     adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP)
3504                         continue;
3505                 r = adev->ip_blocks[i].version->funcs->resume(adev);
3506                 if (r) {
3507                         DRM_ERROR("resume of IP block <%s> failed %d\n",
3508                                   adev->ip_blocks[i].version->funcs->name, r);
3509                         return r;
3510                 }
3511                 adev->ip_blocks[i].status.hw = true;
3512         }
3513
3514         return 0;
3515 }
3516
3517 /**
3518  * amdgpu_device_ip_resume - run resume for hardware IPs
3519  *
3520  * @adev: amdgpu_device pointer
3521  *
3522  * Main resume function for hardware IPs.  The hardware IPs
3523  * are split into two resume functions because they are
3524  * also used in recovering from a GPU reset and some additional
3525  * steps need to be take between them.  In this case (S3/S4) they are
3526  * run sequentially.
3527  * Returns 0 on success, negative error code on failure.
3528  */
3529 static int amdgpu_device_ip_resume(struct amdgpu_device *adev)
3530 {
3531         int r;
3532
3533         r = amdgpu_device_ip_resume_phase1(adev);
3534         if (r)
3535                 return r;
3536
3537         r = amdgpu_device_fw_loading(adev);
3538         if (r)
3539                 return r;
3540
3541         r = amdgpu_device_ip_resume_phase2(adev);
3542
3543         if (adev->mman.buffer_funcs_ring->sched.ready)
3544                 amdgpu_ttm_set_buffer_funcs_status(adev, true);
3545
3546         return r;
3547 }
3548
3549 /**
3550  * amdgpu_device_detect_sriov_bios - determine if the board supports SR-IOV
3551  *
3552  * @adev: amdgpu_device pointer
3553  *
3554  * Query the VBIOS data tables to determine if the board supports SR-IOV.
3555  */
3556 static void amdgpu_device_detect_sriov_bios(struct amdgpu_device *adev)
3557 {
3558         if (amdgpu_sriov_vf(adev)) {
3559                 if (adev->is_atom_fw) {
3560                         if (amdgpu_atomfirmware_gpu_virtualization_supported(adev))
3561                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3562                 } else {
3563                         if (amdgpu_atombios_has_gpu_virtualization_table(adev))
3564                                 adev->virt.caps |= AMDGPU_SRIOV_CAPS_SRIOV_VBIOS;
3565                 }
3566
3567                 if (!(adev->virt.caps & AMDGPU_SRIOV_CAPS_SRIOV_VBIOS))
3568                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_NO_VBIOS, 0, 0);
3569         }
3570 }
3571
3572 /**
3573  * amdgpu_device_asic_has_dc_support - determine if DC supports the asic
3574  *
3575  * @asic_type: AMD asic type
3576  *
3577  * Check if there is DC (new modesetting infrastructre) support for an asic.
3578  * returns true if DC has support, false if not.
3579  */
3580 bool amdgpu_device_asic_has_dc_support(enum amd_asic_type asic_type)
3581 {
3582         switch (asic_type) {
3583 #ifdef CONFIG_DRM_AMDGPU_SI
3584         case CHIP_HAINAN:
3585 #endif
3586         case CHIP_TOPAZ:
3587                 /* chips with no display hardware */
3588                 return false;
3589 #if defined(CONFIG_DRM_AMD_DC)
3590         case CHIP_TAHITI:
3591         case CHIP_PITCAIRN:
3592         case CHIP_VERDE:
3593         case CHIP_OLAND:
3594                 /*
3595                  * We have systems in the wild with these ASICs that require
3596                  * LVDS and VGA support which is not supported with DC.
3597                  *
3598                  * Fallback to the non-DC driver here by default so as not to
3599                  * cause regressions.
3600                  */
3601 #if defined(CONFIG_DRM_AMD_DC_SI)
3602                 return amdgpu_dc > 0;
3603 #else
3604                 return false;
3605 #endif
3606         case CHIP_BONAIRE:
3607         case CHIP_KAVERI:
3608         case CHIP_KABINI:
3609         case CHIP_MULLINS:
3610                 /*
3611                  * We have systems in the wild with these ASICs that require
3612                  * VGA support which is not supported with DC.
3613                  *
3614                  * Fallback to the non-DC driver here by default so as not to
3615                  * cause regressions.
3616                  */
3617                 return amdgpu_dc > 0;
3618         default:
3619                 return amdgpu_dc != 0;
3620 #else
3621         default:
3622                 if (amdgpu_dc > 0)
3623                         DRM_INFO_ONCE("Display Core has been requested via kernel parameter but isn't supported by ASIC, ignoring\n");
3624                 return false;
3625 #endif
3626         }
3627 }
3628
3629 /**
3630  * amdgpu_device_has_dc_support - check if dc is supported
3631  *
3632  * @adev: amdgpu_device pointer
3633  *
3634  * Returns true for supported, false for not supported
3635  */
3636 bool amdgpu_device_has_dc_support(struct amdgpu_device *adev)
3637 {
3638         if (adev->enable_virtual_display ||
3639             (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
3640                 return false;
3641
3642         return amdgpu_device_asic_has_dc_support(adev->asic_type);
3643 }
3644
3645 static void amdgpu_device_xgmi_reset_func(struct work_struct *__work)
3646 {
3647         struct amdgpu_device *adev =
3648                 container_of(__work, struct amdgpu_device, xgmi_reset_work);
3649         struct amdgpu_hive_info *hive = amdgpu_get_xgmi_hive(adev);
3650
3651         /* It's a bug to not have a hive within this function */
3652         if (WARN_ON(!hive))
3653                 return;
3654
3655         /*
3656          * Use task barrier to synchronize all xgmi reset works across the
3657          * hive. task_barrier_enter and task_barrier_exit will block
3658          * until all the threads running the xgmi reset works reach
3659          * those points. task_barrier_full will do both blocks.
3660          */
3661         if (amdgpu_asic_reset_method(adev) == AMD_RESET_METHOD_BACO) {
3662
3663                 task_barrier_enter(&hive->tb);
3664                 adev->asic_reset_res = amdgpu_device_baco_enter(adev_to_drm(adev));
3665
3666                 if (adev->asic_reset_res)
3667                         goto fail;
3668
3669                 task_barrier_exit(&hive->tb);
3670                 adev->asic_reset_res = amdgpu_device_baco_exit(adev_to_drm(adev));
3671
3672                 if (adev->asic_reset_res)
3673                         goto fail;
3674
3675                 amdgpu_ras_reset_error_count(adev, AMDGPU_RAS_BLOCK__MMHUB);
3676         } else {
3677
3678                 task_barrier_full(&hive->tb);
3679                 adev->asic_reset_res =  amdgpu_asic_reset(adev);
3680         }
3681
3682 fail:
3683         if (adev->asic_reset_res)
3684                 DRM_WARN("ASIC reset failed with error, %d for drm dev, %s",
3685                          adev->asic_reset_res, adev_to_drm(adev)->unique);
3686         amdgpu_put_xgmi_hive(hive);
3687 }
3688
3689 static int amdgpu_device_get_job_timeout_settings(struct amdgpu_device *adev)
3690 {
3691         char *input = amdgpu_lockup_timeout;
3692         char *timeout_setting = NULL;
3693         int index = 0;
3694         long timeout;
3695         int ret = 0;
3696
3697         /*
3698          * By default timeout for non compute jobs is 10000
3699          * and 60000 for compute jobs.
3700          * In SR-IOV or passthrough mode, timeout for compute
3701          * jobs are 60000 by default.
3702          */
3703         adev->gfx_timeout = msecs_to_jiffies(10000);
3704         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3705         if (amdgpu_sriov_vf(adev))
3706                 adev->compute_timeout = amdgpu_sriov_is_pp_one_vf(adev) ?
3707                                         msecs_to_jiffies(60000) : msecs_to_jiffies(10000);
3708         else
3709                 adev->compute_timeout =  msecs_to_jiffies(60000);
3710
3711         if (strnlen(input, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3712                 while ((timeout_setting = strsep(&input, ",")) &&
3713                                 strnlen(timeout_setting, AMDGPU_MAX_TIMEOUT_PARAM_LENGTH)) {
3714                         ret = kstrtol(timeout_setting, 0, &timeout);
3715                         if (ret)
3716                                 return ret;
3717
3718                         if (timeout == 0) {
3719                                 index++;
3720                                 continue;
3721                         } else if (timeout < 0) {
3722                                 timeout = MAX_SCHEDULE_TIMEOUT;
3723                                 dev_warn(adev->dev, "lockup timeout disabled");
3724                                 add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
3725                         } else {
3726                                 timeout = msecs_to_jiffies(timeout);
3727                         }
3728
3729                         switch (index++) {
3730                         case 0:
3731                                 adev->gfx_timeout = timeout;
3732                                 break;
3733                         case 1:
3734                                 adev->compute_timeout = timeout;
3735                                 break;
3736                         case 2:
3737                                 adev->sdma_timeout = timeout;
3738                                 break;
3739                         case 3:
3740                                 adev->video_timeout = timeout;
3741                                 break;
3742                         default:
3743                                 break;
3744                         }
3745                 }
3746                 /*
3747                  * There is only one value specified and
3748                  * it should apply to all non-compute jobs.
3749                  */
3750                 if (index == 1) {
3751                         adev->sdma_timeout = adev->video_timeout = adev->gfx_timeout;
3752                         if (amdgpu_sriov_vf(adev) || amdgpu_passthrough(adev))
3753                                 adev->compute_timeout = adev->gfx_timeout;
3754                 }
3755         }
3756
3757         return ret;
3758 }
3759
3760 /**
3761  * amdgpu_device_check_iommu_direct_map - check if RAM direct mapped to GPU
3762  *
3763  * @adev: amdgpu_device pointer
3764  *
3765  * RAM direct mapped to GPU if IOMMU is not enabled or is pass through mode
3766  */
3767 static void amdgpu_device_check_iommu_direct_map(struct amdgpu_device *adev)
3768 {
3769         struct iommu_domain *domain;
3770
3771         domain = iommu_get_domain_for_dev(adev->dev);
3772         if (!domain || domain->type == IOMMU_DOMAIN_IDENTITY)
3773                 adev->ram_is_direct_mapped = true;
3774 }
3775
3776 static const struct attribute *amdgpu_dev_attributes[] = {
3777         &dev_attr_pcie_replay_count.attr,
3778         NULL
3779 };
3780
3781 static void amdgpu_device_set_mcbp(struct amdgpu_device *adev)
3782 {
3783         if (amdgpu_mcbp == 1)
3784                 adev->gfx.mcbp = true;
3785         else if (amdgpu_mcbp == 0)
3786                 adev->gfx.mcbp = false;
3787
3788         if (amdgpu_sriov_vf(adev))
3789                 adev->gfx.mcbp = true;
3790
3791         if (adev->gfx.mcbp)
3792                 DRM_INFO("MCBP is enabled\n");
3793 }
3794
3795 /**
3796  * amdgpu_device_init - initialize the driver
3797  *
3798  * @adev: amdgpu_device pointer
3799  * @flags: driver flags
3800  *
3801  * Initializes the driver info and hw (all asics).
3802  * Returns 0 for success or an error on failure.
3803  * Called at driver startup.
3804  */
3805 int amdgpu_device_init(struct amdgpu_device *adev,
3806                        uint32_t flags)
3807 {
3808         struct drm_device *ddev = adev_to_drm(adev);
3809         struct pci_dev *pdev = adev->pdev;
3810         int r, i;
3811         bool px = false;
3812         u32 max_MBps;
3813         int tmp;
3814
3815         adev->shutdown = false;
3816         adev->flags = flags;
3817
3818         if (amdgpu_force_asic_type >= 0 && amdgpu_force_asic_type < CHIP_LAST)
3819                 adev->asic_type = amdgpu_force_asic_type;
3820         else
3821                 adev->asic_type = flags & AMD_ASIC_MASK;
3822
3823         adev->usec_timeout = AMDGPU_MAX_USEC_TIMEOUT;
3824         if (amdgpu_emu_mode == 1)
3825                 adev->usec_timeout *= 10;
3826         adev->gmc.gart_size = 512 * 1024 * 1024;
3827         adev->accel_working = false;
3828         adev->num_rings = 0;
3829         RCU_INIT_POINTER(adev->gang_submit, dma_fence_get_stub());
3830         adev->mman.buffer_funcs = NULL;
3831         adev->mman.buffer_funcs_ring = NULL;
3832         adev->vm_manager.vm_pte_funcs = NULL;
3833         adev->vm_manager.vm_pte_num_scheds = 0;
3834         adev->gmc.gmc_funcs = NULL;
3835         adev->harvest_ip_mask = 0x0;
3836         adev->fence_context = dma_fence_context_alloc(AMDGPU_MAX_RINGS);
3837         bitmap_zero(adev->gfx.pipe_reserve_bitmap, AMDGPU_MAX_COMPUTE_QUEUES);
3838
3839         adev->smc_rreg = &amdgpu_invalid_rreg;
3840         adev->smc_wreg = &amdgpu_invalid_wreg;
3841         adev->pcie_rreg = &amdgpu_invalid_rreg;
3842         adev->pcie_wreg = &amdgpu_invalid_wreg;
3843         adev->pcie_rreg_ext = &amdgpu_invalid_rreg_ext;
3844         adev->pcie_wreg_ext = &amdgpu_invalid_wreg_ext;
3845         adev->pciep_rreg = &amdgpu_invalid_rreg;
3846         adev->pciep_wreg = &amdgpu_invalid_wreg;
3847         adev->pcie_rreg64 = &amdgpu_invalid_rreg64;
3848         adev->pcie_wreg64 = &amdgpu_invalid_wreg64;
3849         adev->pcie_rreg64_ext = &amdgpu_invalid_rreg64_ext;
3850         adev->pcie_wreg64_ext = &amdgpu_invalid_wreg64_ext;
3851         adev->uvd_ctx_rreg = &amdgpu_invalid_rreg;
3852         adev->uvd_ctx_wreg = &amdgpu_invalid_wreg;
3853         adev->didt_rreg = &amdgpu_invalid_rreg;
3854         adev->didt_wreg = &amdgpu_invalid_wreg;
3855         adev->gc_cac_rreg = &amdgpu_invalid_rreg;
3856         adev->gc_cac_wreg = &amdgpu_invalid_wreg;
3857         adev->audio_endpt_rreg = &amdgpu_block_invalid_rreg;
3858         adev->audio_endpt_wreg = &amdgpu_block_invalid_wreg;
3859
3860         DRM_INFO("initializing kernel modesetting (%s 0x%04X:0x%04X 0x%04X:0x%04X 0x%02X).\n",
3861                  amdgpu_asic_name[adev->asic_type], pdev->vendor, pdev->device,
3862                  pdev->subsystem_vendor, pdev->subsystem_device, pdev->revision);
3863
3864         /* mutex initialization are all done here so we
3865          * can recall function without having locking issues
3866          */
3867         mutex_init(&adev->firmware.mutex);
3868         mutex_init(&adev->pm.mutex);
3869         mutex_init(&adev->gfx.gpu_clock_mutex);
3870         mutex_init(&adev->srbm_mutex);
3871         mutex_init(&adev->gfx.pipe_reserve_mutex);
3872         mutex_init(&adev->gfx.gfx_off_mutex);
3873         mutex_init(&adev->gfx.partition_mutex);
3874         mutex_init(&adev->grbm_idx_mutex);
3875         mutex_init(&adev->mn_lock);
3876         mutex_init(&adev->virt.vf_errors.lock);
3877         hash_init(adev->mn_hash);
3878         mutex_init(&adev->psp.mutex);
3879         mutex_init(&adev->notifier_lock);
3880         mutex_init(&adev->pm.stable_pstate_ctx_lock);
3881         mutex_init(&adev->benchmark_mutex);
3882
3883         amdgpu_device_init_apu_flags(adev);
3884
3885         r = amdgpu_device_check_arguments(adev);
3886         if (r)
3887                 return r;
3888
3889         spin_lock_init(&adev->mmio_idx_lock);
3890         spin_lock_init(&adev->smc_idx_lock);
3891         spin_lock_init(&adev->pcie_idx_lock);
3892         spin_lock_init(&adev->uvd_ctx_idx_lock);
3893         spin_lock_init(&adev->didt_idx_lock);
3894         spin_lock_init(&adev->gc_cac_idx_lock);
3895         spin_lock_init(&adev->se_cac_idx_lock);
3896         spin_lock_init(&adev->audio_endpt_idx_lock);
3897         spin_lock_init(&adev->mm_stats.lock);
3898
3899         INIT_LIST_HEAD(&adev->shadow_list);
3900         mutex_init(&adev->shadow_list_lock);
3901
3902         INIT_LIST_HEAD(&adev->reset_list);
3903
3904         INIT_LIST_HEAD(&adev->ras_list);
3905
3906         INIT_LIST_HEAD(&adev->pm.od_kobj_list);
3907
3908         INIT_DELAYED_WORK(&adev->delayed_init_work,
3909                           amdgpu_device_delayed_init_work_handler);
3910         INIT_DELAYED_WORK(&adev->gfx.gfx_off_delay_work,
3911                           amdgpu_device_delay_enable_gfx_off);
3912
3913         INIT_WORK(&adev->xgmi_reset_work, amdgpu_device_xgmi_reset_func);
3914
3915         adev->gfx.gfx_off_req_count = 1;
3916         adev->gfx.gfx_off_residency = 0;
3917         adev->gfx.gfx_off_entrycount = 0;
3918         adev->pm.ac_power = power_supply_is_system_supplied() > 0;
3919
3920         atomic_set(&adev->throttling_logging_enabled, 1);
3921         /*
3922          * If throttling continues, logging will be performed every minute
3923          * to avoid log flooding. "-1" is subtracted since the thermal
3924          * throttling interrupt comes every second. Thus, the total logging
3925          * interval is 59 seconds(retelimited printk interval) + 1(waiting
3926          * for throttling interrupt) = 60 seconds.
3927          */
3928         ratelimit_state_init(&adev->throttling_logging_rs, (60 - 1) * HZ, 1);
3929         ratelimit_set_flags(&adev->throttling_logging_rs, RATELIMIT_MSG_ON_RELEASE);
3930
3931         /* Registers mapping */
3932         /* TODO: block userspace mapping of io register */
3933         if (adev->asic_type >= CHIP_BONAIRE) {
3934                 adev->rmmio_base = pci_resource_start(adev->pdev, 5);
3935                 adev->rmmio_size = pci_resource_len(adev->pdev, 5);
3936         } else {
3937                 adev->rmmio_base = pci_resource_start(adev->pdev, 2);
3938                 adev->rmmio_size = pci_resource_len(adev->pdev, 2);
3939         }
3940
3941         for (i = 0; i < AMD_IP_BLOCK_TYPE_NUM; i++)
3942                 atomic_set(&adev->pm.pwr_state[i], POWER_STATE_UNKNOWN);
3943
3944         adev->rmmio = ioremap(adev->rmmio_base, adev->rmmio_size);
3945         if (!adev->rmmio)
3946                 return -ENOMEM;
3947
3948         DRM_INFO("register mmio base: 0x%08X\n", (uint32_t)adev->rmmio_base);
3949         DRM_INFO("register mmio size: %u\n", (unsigned int)adev->rmmio_size);
3950
3951         /*
3952          * Reset domain needs to be present early, before XGMI hive discovered
3953          * (if any) and intitialized to use reset sem and in_gpu reset flag
3954          * early on during init and before calling to RREG32.
3955          */
3956         adev->reset_domain = amdgpu_reset_create_reset_domain(SINGLE_DEVICE, "amdgpu-reset-dev");
3957         if (!adev->reset_domain)
3958                 return -ENOMEM;
3959
3960         /* detect hw virtualization here */
3961         amdgpu_detect_virtualization(adev);
3962
3963         amdgpu_device_get_pcie_info(adev);
3964
3965         r = amdgpu_device_get_job_timeout_settings(adev);
3966         if (r) {
3967                 dev_err(adev->dev, "invalid lockup_timeout parameter syntax\n");
3968                 return r;
3969         }
3970
3971         /* early init functions */
3972         r = amdgpu_device_ip_early_init(adev);
3973         if (r)
3974                 return r;
3975
3976         amdgpu_device_set_mcbp(adev);
3977
3978         /* Get rid of things like offb */
3979         r = drm_aperture_remove_conflicting_pci_framebuffers(adev->pdev, &amdgpu_kms_driver);
3980         if (r)
3981                 return r;
3982
3983         /* Enable TMZ based on IP_VERSION */
3984         amdgpu_gmc_tmz_set(adev);
3985
3986         amdgpu_gmc_noretry_set(adev);
3987         /* Need to get xgmi info early to decide the reset behavior*/
3988         if (adev->gmc.xgmi.supported) {
3989                 r = adev->gfxhub.funcs->get_xgmi_info(adev);
3990                 if (r)
3991                         return r;
3992         }
3993
3994         /* enable PCIE atomic ops */
3995         if (amdgpu_sriov_vf(adev)) {
3996                 if (adev->virt.fw_reserve.p_pf2vf)
3997                         adev->have_atomics_support = ((struct amd_sriov_msg_pf2vf_info *)
3998                                                       adev->virt.fw_reserve.p_pf2vf)->pcie_atomic_ops_support_flags ==
3999                                 (PCI_EXP_DEVCAP2_ATOMIC_COMP32 | PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4000         /* APUs w/ gfx9 onwards doesn't reply on PCIe atomics, rather it is a
4001          * internal path natively support atomics, set have_atomics_support to true.
4002          */
4003         } else if ((adev->flags & AMD_IS_APU) &&
4004                    (amdgpu_ip_version(adev, GC_HWIP, 0) >
4005                     IP_VERSION(9, 0, 0))) {
4006                 adev->have_atomics_support = true;
4007         } else {
4008                 adev->have_atomics_support =
4009                         !pci_enable_atomic_ops_to_root(adev->pdev,
4010                                           PCI_EXP_DEVCAP2_ATOMIC_COMP32 |
4011                                           PCI_EXP_DEVCAP2_ATOMIC_COMP64);
4012         }
4013
4014         if (!adev->have_atomics_support)
4015                 dev_info(adev->dev, "PCIE atomic ops is not supported\n");
4016
4017         /* doorbell bar mapping and doorbell index init*/
4018         amdgpu_doorbell_init(adev);
4019
4020         if (amdgpu_emu_mode == 1) {
4021                 /* post the asic on emulation mode */
4022                 emu_soc_asic_init(adev);
4023                 goto fence_driver_init;
4024         }
4025
4026         amdgpu_reset_init(adev);
4027
4028         /* detect if we are with an SRIOV vbios */
4029         if (adev->bios)
4030                 amdgpu_device_detect_sriov_bios(adev);
4031
4032         /* check if we need to reset the asic
4033          *  E.g., driver was not cleanly unloaded previously, etc.
4034          */
4035         if (!amdgpu_sriov_vf(adev) && amdgpu_asic_need_reset_on_init(adev)) {
4036                 if (adev->gmc.xgmi.num_physical_nodes) {
4037                         dev_info(adev->dev, "Pending hive reset.\n");
4038                         adev->gmc.xgmi.pending_reset = true;
4039                         /* Only need to init necessary block for SMU to handle the reset */
4040                         for (i = 0; i < adev->num_ip_blocks; i++) {
4041                                 if (!adev->ip_blocks[i].status.valid)
4042                                         continue;
4043                                 if (!(adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC ||
4044                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_COMMON ||
4045                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_IH ||
4046                                       adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC)) {
4047                                         DRM_DEBUG("IP %s disabled for hw_init.\n",
4048                                                 adev->ip_blocks[i].version->funcs->name);
4049                                         adev->ip_blocks[i].status.hw = true;
4050                                 }
4051                         }
4052                 } else {
4053                         switch (amdgpu_ip_version(adev, MP1_HWIP, 0)) {
4054                         case IP_VERSION(13, 0, 0):
4055                         case IP_VERSION(13, 0, 7):
4056                         case IP_VERSION(13, 0, 10):
4057                                 r = psp_gpu_reset(adev);
4058                                 break;
4059                         default:
4060                                 tmp = amdgpu_reset_method;
4061                                 /* It should do a default reset when loading or reloading the driver,
4062                                  * regardless of the module parameter reset_method.
4063                                  */
4064                                 amdgpu_reset_method = AMD_RESET_METHOD_NONE;
4065                                 r = amdgpu_asic_reset(adev);
4066                                 amdgpu_reset_method = tmp;
4067                                 break;
4068                         }
4069
4070                         if (r) {
4071                                 dev_err(adev->dev, "asic reset on init failed\n");
4072                                 goto failed;
4073                         }
4074                 }
4075         }
4076
4077         /* Post card if necessary */
4078         if (amdgpu_device_need_post(adev)) {
4079                 if (!adev->bios) {
4080                         dev_err(adev->dev, "no vBIOS found\n");
4081                         r = -EINVAL;
4082                         goto failed;
4083                 }
4084                 DRM_INFO("GPU posting now...\n");
4085                 r = amdgpu_device_asic_init(adev);
4086                 if (r) {
4087                         dev_err(adev->dev, "gpu post error!\n");
4088                         goto failed;
4089                 }
4090         }
4091
4092         if (adev->bios) {
4093                 if (adev->is_atom_fw) {
4094                         /* Initialize clocks */
4095                         r = amdgpu_atomfirmware_get_clock_info(adev);
4096                         if (r) {
4097                                 dev_err(adev->dev, "amdgpu_atomfirmware_get_clock_info failed\n");
4098                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4099                                 goto failed;
4100                         }
4101                 } else {
4102                         /* Initialize clocks */
4103                         r = amdgpu_atombios_get_clock_info(adev);
4104                         if (r) {
4105                                 dev_err(adev->dev, "amdgpu_atombios_get_clock_info failed\n");
4106                                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_ATOMBIOS_GET_CLOCK_FAIL, 0, 0);
4107                                 goto failed;
4108                         }
4109                         /* init i2c buses */
4110                         if (!amdgpu_device_has_dc_support(adev))
4111                                 amdgpu_atombios_i2c_init(adev);
4112                 }
4113         }
4114
4115 fence_driver_init:
4116         /* Fence driver */
4117         r = amdgpu_fence_driver_sw_init(adev);
4118         if (r) {
4119                 dev_err(adev->dev, "amdgpu_fence_driver_sw_init failed\n");
4120                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_FENCE_INIT_FAIL, 0, 0);
4121                 goto failed;
4122         }
4123
4124         /* init the mode config */
4125         drm_mode_config_init(adev_to_drm(adev));
4126
4127         r = amdgpu_device_ip_init(adev);
4128         if (r) {
4129                 dev_err(adev->dev, "amdgpu_device_ip_init failed\n");
4130                 amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_INIT_FAIL, 0, 0);
4131                 goto release_ras_con;
4132         }
4133
4134         amdgpu_fence_driver_hw_init(adev);
4135
4136         dev_info(adev->dev,
4137                 "SE %d, SH per SE %d, CU per SH %d, active_cu_number %d\n",
4138                         adev->gfx.config.max_shader_engines,
4139                         adev->gfx.config.max_sh_per_se,
4140                         adev->gfx.config.max_cu_per_sh,
4141                         adev->gfx.cu_info.number);
4142
4143         adev->accel_working = true;
4144
4145         amdgpu_vm_check_compute_bug(adev);
4146
4147         /* Initialize the buffer migration limit. */
4148         if (amdgpu_moverate >= 0)
4149                 max_MBps = amdgpu_moverate;
4150         else
4151                 max_MBps = 8; /* Allow 8 MB/s. */
4152         /* Get a log2 for easy divisions. */
4153         adev->mm_stats.log2_max_MBps = ilog2(max(1u, max_MBps));
4154
4155         /*
4156          * Register gpu instance before amdgpu_device_enable_mgpu_fan_boost.
4157          * Otherwise the mgpu fan boost feature will be skipped due to the
4158          * gpu instance is counted less.
4159          */
4160         amdgpu_register_gpu_instance(adev);
4161
4162         /* enable clockgating, etc. after ib tests, etc. since some blocks require
4163          * explicit gating rather than handling it automatically.
4164          */
4165         if (!adev->gmc.xgmi.pending_reset) {
4166                 r = amdgpu_device_ip_late_init(adev);
4167                 if (r) {
4168                         dev_err(adev->dev, "amdgpu_device_ip_late_init failed\n");
4169                         amdgpu_vf_error_put(adev, AMDGIM_ERROR_VF_AMDGPU_LATE_INIT_FAIL, 0, r);
4170                         goto release_ras_con;
4171                 }
4172                 /* must succeed. */
4173                 amdgpu_ras_resume(adev);
4174                 queue_delayed_work(system_wq, &adev->delayed_init_work,
4175                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4176         }
4177
4178         if (amdgpu_sriov_vf(adev)) {
4179                 amdgpu_virt_release_full_gpu(adev, true);
4180                 flush_delayed_work(&adev->delayed_init_work);
4181         }
4182
4183         /*
4184          * Place those sysfs registering after `late_init`. As some of those
4185          * operations performed in `late_init` might affect the sysfs
4186          * interfaces creating.
4187          */
4188         r = amdgpu_atombios_sysfs_init(adev);
4189         if (r)
4190                 drm_err(&adev->ddev,
4191                         "registering atombios sysfs failed (%d).\n", r);
4192
4193         r = amdgpu_pm_sysfs_init(adev);
4194         if (r)
4195                 DRM_ERROR("registering pm sysfs failed (%d).\n", r);
4196
4197         r = amdgpu_ucode_sysfs_init(adev);
4198         if (r) {
4199                 adev->ucode_sysfs_en = false;
4200                 DRM_ERROR("Creating firmware sysfs failed (%d).\n", r);
4201         } else
4202                 adev->ucode_sysfs_en = true;
4203
4204         r = sysfs_create_files(&adev->dev->kobj, amdgpu_dev_attributes);
4205         if (r)
4206                 dev_err(adev->dev, "Could not create amdgpu device attr\n");
4207
4208         r = devm_device_add_group(adev->dev, &amdgpu_board_attrs_group);
4209         if (r)
4210                 dev_err(adev->dev,
4211                         "Could not create amdgpu board attributes\n");
4212
4213         amdgpu_fru_sysfs_init(adev);
4214
4215         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4216                 r = amdgpu_pmu_init(adev);
4217         if (r)
4218                 dev_err(adev->dev, "amdgpu_pmu_init failed\n");
4219
4220         /* Have stored pci confspace at hand for restore in sudden PCI error */
4221         if (amdgpu_device_cache_pci_state(adev->pdev))
4222                 pci_restore_state(pdev);
4223
4224         /* if we have > 1 VGA cards, then disable the amdgpu VGA resources */
4225         /* this will fail for cards that aren't VGA class devices, just
4226          * ignore it
4227          */
4228         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4229                 vga_client_register(adev->pdev, amdgpu_device_vga_set_decode);
4230
4231         px = amdgpu_device_supports_px(ddev);
4232
4233         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4234                                 apple_gmux_detect(NULL, NULL)))
4235                 vga_switcheroo_register_client(adev->pdev,
4236                                                &amdgpu_switcheroo_ops, px);
4237
4238         if (px)
4239                 vga_switcheroo_init_domain_pm_ops(adev->dev, &adev->vga_pm_domain);
4240
4241         if (adev->gmc.xgmi.pending_reset)
4242                 queue_delayed_work(system_wq, &mgpu_info.delayed_reset_work,
4243                                    msecs_to_jiffies(AMDGPU_RESUME_MS));
4244
4245         amdgpu_device_check_iommu_direct_map(adev);
4246
4247         return 0;
4248
4249 release_ras_con:
4250         if (amdgpu_sriov_vf(adev))
4251                 amdgpu_virt_release_full_gpu(adev, true);
4252
4253         /* failed in exclusive mode due to timeout */
4254         if (amdgpu_sriov_vf(adev) &&
4255                 !amdgpu_sriov_runtime(adev) &&
4256                 amdgpu_virt_mmio_blocked(adev) &&
4257                 !amdgpu_virt_wait_reset(adev)) {
4258                 dev_err(adev->dev, "VF exclusive mode timeout\n");
4259                 /* Don't send request since VF is inactive. */
4260                 adev->virt.caps &= ~AMDGPU_SRIOV_CAPS_RUNTIME;
4261                 adev->virt.ops = NULL;
4262                 r = -EAGAIN;
4263         }
4264         amdgpu_release_ras_context(adev);
4265
4266 failed:
4267         amdgpu_vf_error_trans_all(adev);
4268
4269         return r;
4270 }
4271
4272 static void amdgpu_device_unmap_mmio(struct amdgpu_device *adev)
4273 {
4274
4275         /* Clear all CPU mappings pointing to this device */
4276         unmap_mapping_range(adev->ddev.anon_inode->i_mapping, 0, 0, 1);
4277
4278         /* Unmap all mapped bars - Doorbell, registers and VRAM */
4279         amdgpu_doorbell_fini(adev);
4280
4281         iounmap(adev->rmmio);
4282         adev->rmmio = NULL;
4283         if (adev->mman.aper_base_kaddr)
4284                 iounmap(adev->mman.aper_base_kaddr);
4285         adev->mman.aper_base_kaddr = NULL;
4286
4287         /* Memory manager related */
4288         if (!adev->gmc.xgmi.connected_to_cpu && !adev->gmc.is_app_apu) {
4289                 arch_phys_wc_del(adev->gmc.vram_mtrr);
4290                 arch_io_free_memtype_wc(adev->gmc.aper_base, adev->gmc.aper_size);
4291         }
4292 }
4293
4294 /**
4295  * amdgpu_device_fini_hw - tear down the driver
4296  *
4297  * @adev: amdgpu_device pointer
4298  *
4299  * Tear down the driver info (all asics).
4300  * Called at driver shutdown.
4301  */
4302 void amdgpu_device_fini_hw(struct amdgpu_device *adev)
4303 {
4304         dev_info(adev->dev, "amdgpu: finishing device.\n");
4305         flush_delayed_work(&adev->delayed_init_work);
4306         adev->shutdown = true;
4307
4308         /* make sure IB test finished before entering exclusive mode
4309          * to avoid preemption on IB test
4310          */
4311         if (amdgpu_sriov_vf(adev)) {
4312                 amdgpu_virt_request_full_gpu(adev, false);
4313                 amdgpu_virt_fini_data_exchange(adev);
4314         }
4315
4316         /* disable all interrupts */
4317         amdgpu_irq_disable_all(adev);
4318         if (adev->mode_info.mode_config_initialized) {
4319                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(adev)))
4320                         drm_helper_force_disable_all(adev_to_drm(adev));
4321                 else
4322                         drm_atomic_helper_shutdown(adev_to_drm(adev));
4323         }
4324         amdgpu_fence_driver_hw_fini(adev);
4325
4326         if (adev->mman.initialized)
4327                 drain_workqueue(adev->mman.bdev.wq);
4328
4329         if (adev->pm.sysfs_initialized)
4330                 amdgpu_pm_sysfs_fini(adev);
4331         if (adev->ucode_sysfs_en)
4332                 amdgpu_ucode_sysfs_fini(adev);
4333         sysfs_remove_files(&adev->dev->kobj, amdgpu_dev_attributes);
4334         amdgpu_fru_sysfs_fini(adev);
4335
4336         /* disable ras feature must before hw fini */
4337         amdgpu_ras_pre_fini(adev);
4338
4339         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4340
4341         amdgpu_device_ip_fini_early(adev);
4342
4343         amdgpu_irq_fini_hw(adev);
4344
4345         if (adev->mman.initialized)
4346                 ttm_device_clear_dma_mappings(&adev->mman.bdev);
4347
4348         amdgpu_gart_dummy_page_fini(adev);
4349
4350         if (drm_dev_is_unplugged(adev_to_drm(adev)))
4351                 amdgpu_device_unmap_mmio(adev);
4352
4353 }
4354
4355 void amdgpu_device_fini_sw(struct amdgpu_device *adev)
4356 {
4357         int idx;
4358         bool px;
4359
4360         amdgpu_fence_driver_sw_fini(adev);
4361         amdgpu_device_ip_fini(adev);
4362         amdgpu_ucode_release(&adev->firmware.gpu_info_fw);
4363         adev->accel_working = false;
4364         dma_fence_put(rcu_dereference_protected(adev->gang_submit, true));
4365
4366         amdgpu_reset_fini(adev);
4367
4368         /* free i2c buses */
4369         if (!amdgpu_device_has_dc_support(adev))
4370                 amdgpu_i2c_fini(adev);
4371
4372         if (amdgpu_emu_mode != 1)
4373                 amdgpu_atombios_fini(adev);
4374
4375         kfree(adev->bios);
4376         adev->bios = NULL;
4377
4378         kfree(adev->fru_info);
4379         adev->fru_info = NULL;
4380
4381         px = amdgpu_device_supports_px(adev_to_drm(adev));
4382
4383         if (px || (!dev_is_removable(&adev->pdev->dev) &&
4384                                 apple_gmux_detect(NULL, NULL)))
4385                 vga_switcheroo_unregister_client(adev->pdev);
4386
4387         if (px)
4388                 vga_switcheroo_fini_domain_pm_ops(adev->dev);
4389
4390         if ((adev->pdev->class >> 8) == PCI_CLASS_DISPLAY_VGA)
4391                 vga_client_unregister(adev->pdev);
4392
4393         if (drm_dev_enter(adev_to_drm(adev), &idx)) {
4394
4395                 iounmap(adev->rmmio);
4396                 adev->rmmio = NULL;
4397                 amdgpu_doorbell_fini(adev);
4398                 drm_dev_exit(idx);
4399         }
4400
4401         if (IS_ENABLED(CONFIG_PERF_EVENTS))
4402                 amdgpu_pmu_fini(adev);
4403         if (adev->mman.discovery_bin)
4404                 amdgpu_discovery_fini(adev);
4405
4406         amdgpu_reset_put_reset_domain(adev->reset_domain);
4407         adev->reset_domain = NULL;
4408
4409         kfree(adev->pci_state);
4410
4411 }
4412
4413 /**
4414  * amdgpu_device_evict_resources - evict device resources
4415  * @adev: amdgpu device object
4416  *
4417  * Evicts all ttm device resources(vram BOs, gart table) from the lru list
4418  * of the vram memory type. Mainly used for evicting device resources
4419  * at suspend time.
4420  *
4421  */
4422 static int amdgpu_device_evict_resources(struct amdgpu_device *adev)
4423 {
4424         int ret;
4425
4426         /* No need to evict vram on APUs for suspend to ram or s2idle */
4427         if ((adev->in_s3 || adev->in_s0ix) && (adev->flags & AMD_IS_APU))
4428                 return 0;
4429
4430         ret = amdgpu_ttm_evict_resources(adev, TTM_PL_VRAM);
4431         if (ret)
4432                 DRM_WARN("evicting device resources failed\n");
4433         return ret;
4434 }
4435
4436 /*
4437  * Suspend & resume.
4438  */
4439 /**
4440  * amdgpu_device_prepare - prepare for device suspend
4441  *
4442  * @dev: drm dev pointer
4443  *
4444  * Prepare to put the hw in the suspend state (all asics).
4445  * Returns 0 for success or an error on failure.
4446  * Called at driver suspend.
4447  */
4448 int amdgpu_device_prepare(struct drm_device *dev)
4449 {
4450         struct amdgpu_device *adev = drm_to_adev(dev);
4451         int i, r;
4452
4453         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4454                 return 0;
4455
4456         /* Evict the majority of BOs before starting suspend sequence */
4457         r = amdgpu_device_evict_resources(adev);
4458         if (r)
4459                 return r;
4460
4461         for (i = 0; i < adev->num_ip_blocks; i++) {
4462                 if (!adev->ip_blocks[i].status.valid)
4463                         continue;
4464                 if (!adev->ip_blocks[i].version->funcs->prepare_suspend)
4465                         continue;
4466                 r = adev->ip_blocks[i].version->funcs->prepare_suspend((void *)adev);
4467                 if (r)
4468                         return r;
4469         }
4470
4471         return 0;
4472 }
4473
4474 /**
4475  * amdgpu_device_suspend - initiate device suspend
4476  *
4477  * @dev: drm dev pointer
4478  * @fbcon : notify the fbdev of suspend
4479  *
4480  * Puts the hw in the suspend state (all asics).
4481  * Returns 0 for success or an error on failure.
4482  * Called at driver suspend.
4483  */
4484 int amdgpu_device_suspend(struct drm_device *dev, bool fbcon)
4485 {
4486         struct amdgpu_device *adev = drm_to_adev(dev);
4487         int r = 0;
4488
4489         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4490                 return 0;
4491
4492         adev->in_suspend = true;
4493
4494         if (amdgpu_sriov_vf(adev)) {
4495                 amdgpu_virt_fini_data_exchange(adev);
4496                 r = amdgpu_virt_request_full_gpu(adev, false);
4497                 if (r)
4498                         return r;
4499         }
4500
4501         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D3))
4502                 DRM_WARN("smart shift update failed\n");
4503
4504         if (fbcon)
4505                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, true);
4506
4507         cancel_delayed_work_sync(&adev->delayed_init_work);
4508         flush_delayed_work(&adev->gfx.gfx_off_delay_work);
4509
4510         amdgpu_ras_suspend(adev);
4511
4512         amdgpu_device_ip_suspend_phase1(adev);
4513
4514         if (!adev->in_s0ix)
4515                 amdgpu_amdkfd_suspend(adev, adev->in_runpm);
4516
4517         r = amdgpu_device_evict_resources(adev);
4518         if (r)
4519                 return r;
4520
4521         amdgpu_ttm_set_buffer_funcs_status(adev, false);
4522
4523         amdgpu_fence_driver_hw_fini(adev);
4524
4525         amdgpu_device_ip_suspend_phase2(adev);
4526
4527         if (amdgpu_sriov_vf(adev))
4528                 amdgpu_virt_release_full_gpu(adev, false);
4529
4530         r = amdgpu_dpm_notify_rlc_state(adev, false);
4531         if (r)
4532                 return r;
4533
4534         return 0;
4535 }
4536
4537 /**
4538  * amdgpu_device_resume - initiate device resume
4539  *
4540  * @dev: drm dev pointer
4541  * @fbcon : notify the fbdev of resume
4542  *
4543  * Bring the hw back to operating state (all asics).
4544  * Returns 0 for success or an error on failure.
4545  * Called at driver resume.
4546  */
4547 int amdgpu_device_resume(struct drm_device *dev, bool fbcon)
4548 {
4549         struct amdgpu_device *adev = drm_to_adev(dev);
4550         int r = 0;
4551
4552         if (amdgpu_sriov_vf(adev)) {
4553                 r = amdgpu_virt_request_full_gpu(adev, true);
4554                 if (r)
4555                         return r;
4556         }
4557
4558         if (dev->switch_power_state == DRM_SWITCH_POWER_OFF)
4559                 return 0;
4560
4561         if (adev->in_s0ix)
4562                 amdgpu_dpm_gfx_state_change(adev, sGpuChangeState_D0Entry);
4563
4564         /* post card */
4565         if (amdgpu_device_need_post(adev)) {
4566                 r = amdgpu_device_asic_init(adev);
4567                 if (r)
4568                         dev_err(adev->dev, "amdgpu asic init failed\n");
4569         }
4570
4571         r = amdgpu_device_ip_resume(adev);
4572
4573         if (r) {
4574                 dev_err(adev->dev, "amdgpu_device_ip_resume failed (%d).\n", r);
4575                 goto exit;
4576         }
4577         amdgpu_fence_driver_hw_init(adev);
4578
4579         if (!adev->in_s0ix) {
4580                 r = amdgpu_amdkfd_resume(adev, adev->in_runpm);
4581                 if (r)
4582                         goto exit;
4583         }
4584
4585         r = amdgpu_device_ip_late_init(adev);
4586         if (r)
4587                 goto exit;
4588
4589         queue_delayed_work(system_wq, &adev->delayed_init_work,
4590                            msecs_to_jiffies(AMDGPU_RESUME_MS));
4591 exit:
4592         if (amdgpu_sriov_vf(adev)) {
4593                 amdgpu_virt_init_data_exchange(adev);
4594                 amdgpu_virt_release_full_gpu(adev, true);
4595         }
4596
4597         if (r)
4598                 return r;
4599
4600         /* Make sure IB tests flushed */
4601         flush_delayed_work(&adev->delayed_init_work);
4602
4603         if (fbcon)
4604                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(adev)->fb_helper, false);
4605
4606         amdgpu_ras_resume(adev);
4607
4608         if (adev->mode_info.num_crtc) {
4609                 /*
4610                  * Most of the connector probing functions try to acquire runtime pm
4611                  * refs to ensure that the GPU is powered on when connector polling is
4612                  * performed. Since we're calling this from a runtime PM callback,
4613                  * trying to acquire rpm refs will cause us to deadlock.
4614                  *
4615                  * Since we're guaranteed to be holding the rpm lock, it's safe to
4616                  * temporarily disable the rpm helpers so this doesn't deadlock us.
4617                  */
4618 #ifdef CONFIG_PM
4619                 dev->dev->power.disable_depth++;
4620 #endif
4621                 if (!adev->dc_enabled)
4622                         drm_helper_hpd_irq_event(dev);
4623                 else
4624                         drm_kms_helper_hotplug_event(dev);
4625 #ifdef CONFIG_PM
4626                 dev->dev->power.disable_depth--;
4627 #endif
4628         }
4629         adev->in_suspend = false;
4630
4631         if (adev->enable_mes)
4632                 amdgpu_mes_self_test(adev);
4633
4634         if (amdgpu_acpi_smart_shift_update(dev, AMDGPU_SS_DEV_D0))
4635                 DRM_WARN("smart shift update failed\n");
4636
4637         return 0;
4638 }
4639
4640 /**
4641  * amdgpu_device_ip_check_soft_reset - did soft reset succeed
4642  *
4643  * @adev: amdgpu_device pointer
4644  *
4645  * The list of all the hardware IPs that make up the asic is walked and
4646  * the check_soft_reset callbacks are run.  check_soft_reset determines
4647  * if the asic is still hung or not.
4648  * Returns true if any of the IPs are still in a hung state, false if not.
4649  */
4650 static bool amdgpu_device_ip_check_soft_reset(struct amdgpu_device *adev)
4651 {
4652         int i;
4653         bool asic_hang = false;
4654
4655         if (amdgpu_sriov_vf(adev))
4656                 return true;
4657
4658         if (amdgpu_asic_need_full_reset(adev))
4659                 return true;
4660
4661         for (i = 0; i < adev->num_ip_blocks; i++) {
4662                 if (!adev->ip_blocks[i].status.valid)
4663                         continue;
4664                 if (adev->ip_blocks[i].version->funcs->check_soft_reset)
4665                         adev->ip_blocks[i].status.hang =
4666                                 adev->ip_blocks[i].version->funcs->check_soft_reset(adev);
4667                 if (adev->ip_blocks[i].status.hang) {
4668                         dev_info(adev->dev, "IP block:%s is hung!\n", adev->ip_blocks[i].version->funcs->name);
4669                         asic_hang = true;
4670                 }
4671         }
4672         return asic_hang;
4673 }
4674
4675 /**
4676  * amdgpu_device_ip_pre_soft_reset - prepare for soft reset
4677  *
4678  * @adev: amdgpu_device pointer
4679  *
4680  * The list of all the hardware IPs that make up the asic is walked and the
4681  * pre_soft_reset callbacks are run if the block is hung.  pre_soft_reset
4682  * handles any IP specific hardware or software state changes that are
4683  * necessary for a soft reset to succeed.
4684  * Returns 0 on success, negative error code on failure.
4685  */
4686 static int amdgpu_device_ip_pre_soft_reset(struct amdgpu_device *adev)
4687 {
4688         int i, r = 0;
4689
4690         for (i = 0; i < adev->num_ip_blocks; i++) {
4691                 if (!adev->ip_blocks[i].status.valid)
4692                         continue;
4693                 if (adev->ip_blocks[i].status.hang &&
4694                     adev->ip_blocks[i].version->funcs->pre_soft_reset) {
4695                         r = adev->ip_blocks[i].version->funcs->pre_soft_reset(adev);
4696                         if (r)
4697                                 return r;
4698                 }
4699         }
4700
4701         return 0;
4702 }
4703
4704 /**
4705  * amdgpu_device_ip_need_full_reset - check if a full asic reset is needed
4706  *
4707  * @adev: amdgpu_device pointer
4708  *
4709  * Some hardware IPs cannot be soft reset.  If they are hung, a full gpu
4710  * reset is necessary to recover.
4711  * Returns true if a full asic reset is required, false if not.
4712  */
4713 static bool amdgpu_device_ip_need_full_reset(struct amdgpu_device *adev)
4714 {
4715         int i;
4716
4717         if (amdgpu_asic_need_full_reset(adev))
4718                 return true;
4719
4720         for (i = 0; i < adev->num_ip_blocks; i++) {
4721                 if (!adev->ip_blocks[i].status.valid)
4722                         continue;
4723                 if ((adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_GMC) ||
4724                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_SMC) ||
4725                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_ACP) ||
4726                     (adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_DCE) ||
4727                      adev->ip_blocks[i].version->type == AMD_IP_BLOCK_TYPE_PSP) {
4728                         if (adev->ip_blocks[i].status.hang) {
4729                                 dev_info(adev->dev, "Some block need full reset!\n");
4730                                 return true;
4731                         }
4732                 }
4733         }
4734         return false;
4735 }
4736
4737 /**
4738  * amdgpu_device_ip_soft_reset - do a soft reset
4739  *
4740  * @adev: amdgpu_device pointer
4741  *
4742  * The list of all the hardware IPs that make up the asic is walked and the
4743  * soft_reset callbacks are run if the block is hung.  soft_reset handles any
4744  * IP specific hardware or software state changes that are necessary to soft
4745  * reset the IP.
4746  * Returns 0 on success, negative error code on failure.
4747  */
4748 static int amdgpu_device_ip_soft_reset(struct amdgpu_device *adev)
4749 {
4750         int i, r = 0;
4751
4752         for (i = 0; i < adev->num_ip_blocks; i++) {
4753                 if (!adev->ip_blocks[i].status.valid)
4754                         continue;
4755                 if (adev->ip_blocks[i].status.hang &&
4756                     adev->ip_blocks[i].version->funcs->soft_reset) {
4757                         r = adev->ip_blocks[i].version->funcs->soft_reset(adev);
4758                         if (r)
4759                                 return r;
4760                 }
4761         }
4762
4763         return 0;
4764 }
4765
4766 /**
4767  * amdgpu_device_ip_post_soft_reset - clean up from soft reset
4768  *
4769  * @adev: amdgpu_device pointer
4770  *
4771  * The list of all the hardware IPs that make up the asic is walked and the
4772  * post_soft_reset callbacks are run if the asic was hung.  post_soft_reset
4773  * handles any IP specific hardware or software state changes that are
4774  * necessary after the IP has been soft reset.
4775  * Returns 0 on success, negative error code on failure.
4776  */
4777 static int amdgpu_device_ip_post_soft_reset(struct amdgpu_device *adev)
4778 {
4779         int i, r = 0;
4780
4781         for (i = 0; i < adev->num_ip_blocks; i++) {
4782                 if (!adev->ip_blocks[i].status.valid)
4783                         continue;
4784                 if (adev->ip_blocks[i].status.hang &&
4785                     adev->ip_blocks[i].version->funcs->post_soft_reset)
4786                         r = adev->ip_blocks[i].version->funcs->post_soft_reset(adev);
4787                 if (r)
4788                         return r;
4789         }
4790
4791         return 0;
4792 }
4793
4794 /**
4795  * amdgpu_device_recover_vram - Recover some VRAM contents
4796  *
4797  * @adev: amdgpu_device pointer
4798  *
4799  * Restores the contents of VRAM buffers from the shadows in GTT.  Used to
4800  * restore things like GPUVM page tables after a GPU reset where
4801  * the contents of VRAM might be lost.
4802  *
4803  * Returns:
4804  * 0 on success, negative error code on failure.
4805  */
4806 static int amdgpu_device_recover_vram(struct amdgpu_device *adev)
4807 {
4808         struct dma_fence *fence = NULL, *next = NULL;
4809         struct amdgpu_bo *shadow;
4810         struct amdgpu_bo_vm *vmbo;
4811         long r = 1, tmo;
4812
4813         if (amdgpu_sriov_runtime(adev))
4814                 tmo = msecs_to_jiffies(8000);
4815         else
4816                 tmo = msecs_to_jiffies(100);
4817
4818         dev_info(adev->dev, "recover vram bo from shadow start\n");
4819         mutex_lock(&adev->shadow_list_lock);
4820         list_for_each_entry(vmbo, &adev->shadow_list, shadow_list) {
4821                 /* If vm is compute context or adev is APU, shadow will be NULL */
4822                 if (!vmbo->shadow)
4823                         continue;
4824                 shadow = vmbo->shadow;
4825
4826                 /* No need to recover an evicted BO */
4827                 if (shadow->tbo.resource->mem_type != TTM_PL_TT ||
4828                     shadow->tbo.resource->start == AMDGPU_BO_INVALID_OFFSET ||
4829                     shadow->parent->tbo.resource->mem_type != TTM_PL_VRAM)
4830                         continue;
4831
4832                 r = amdgpu_bo_restore_shadow(shadow, &next);
4833                 if (r)
4834                         break;
4835
4836                 if (fence) {
4837                         tmo = dma_fence_wait_timeout(fence, false, tmo);
4838                         dma_fence_put(fence);
4839                         fence = next;
4840                         if (tmo == 0) {
4841                                 r = -ETIMEDOUT;
4842                                 break;
4843                         } else if (tmo < 0) {
4844                                 r = tmo;
4845                                 break;
4846                         }
4847                 } else {
4848                         fence = next;
4849                 }
4850         }
4851         mutex_unlock(&adev->shadow_list_lock);
4852
4853         if (fence)
4854                 tmo = dma_fence_wait_timeout(fence, false, tmo);
4855         dma_fence_put(fence);
4856
4857         if (r < 0 || tmo <= 0) {
4858                 dev_err(adev->dev, "recover vram bo from shadow failed, r is %ld, tmo is %ld\n", r, tmo);
4859                 return -EIO;
4860         }
4861
4862         dev_info(adev->dev, "recover vram bo from shadow done\n");
4863         return 0;
4864 }
4865
4866
4867 /**
4868  * amdgpu_device_reset_sriov - reset ASIC for SR-IOV vf
4869  *
4870  * @adev: amdgpu_device pointer
4871  * @from_hypervisor: request from hypervisor
4872  *
4873  * do VF FLR and reinitialize Asic
4874  * return 0 means succeeded otherwise failed
4875  */
4876 static int amdgpu_device_reset_sriov(struct amdgpu_device *adev,
4877                                      bool from_hypervisor)
4878 {
4879         int r;
4880         struct amdgpu_hive_info *hive = NULL;
4881         int retry_limit = 0;
4882
4883 retry:
4884         amdgpu_amdkfd_pre_reset(adev);
4885
4886         if (from_hypervisor)
4887                 r = amdgpu_virt_request_full_gpu(adev, true);
4888         else
4889                 r = amdgpu_virt_reset_gpu(adev);
4890         if (r)
4891                 return r;
4892         amdgpu_irq_gpu_reset_resume_helper(adev);
4893
4894         /* some sw clean up VF needs to do before recover */
4895         amdgpu_virt_post_reset(adev);
4896
4897         /* Resume IP prior to SMC */
4898         r = amdgpu_device_ip_reinit_early_sriov(adev);
4899         if (r)
4900                 goto error;
4901
4902         amdgpu_virt_init_data_exchange(adev);
4903
4904         r = amdgpu_device_fw_loading(adev);
4905         if (r)
4906                 return r;
4907
4908         /* now we are okay to resume SMC/CP/SDMA */
4909         r = amdgpu_device_ip_reinit_late_sriov(adev);
4910         if (r)
4911                 goto error;
4912
4913         hive = amdgpu_get_xgmi_hive(adev);
4914         /* Update PSP FW topology after reset */
4915         if (hive && adev->gmc.xgmi.num_physical_nodes > 1)
4916                 r = amdgpu_xgmi_update_topology(hive, adev);
4917
4918         if (hive)
4919                 amdgpu_put_xgmi_hive(hive);
4920
4921         if (!r) {
4922                 r = amdgpu_ib_ring_tests(adev);
4923
4924                 amdgpu_amdkfd_post_reset(adev);
4925         }
4926
4927 error:
4928         if (!r && adev->virt.gim_feature & AMDGIM_FEATURE_GIM_FLR_VRAMLOST) {
4929                 amdgpu_inc_vram_lost(adev);
4930                 r = amdgpu_device_recover_vram(adev);
4931         }
4932         amdgpu_virt_release_full_gpu(adev, true);
4933
4934         if (AMDGPU_RETRY_SRIOV_RESET(r)) {
4935                 if (retry_limit < AMDGPU_MAX_RETRY_LIMIT) {
4936                         retry_limit++;
4937                         goto retry;
4938                 } else
4939                         DRM_ERROR("GPU reset retry is beyond the retry limit\n");
4940         }
4941
4942         return r;
4943 }
4944
4945 /**
4946  * amdgpu_device_has_job_running - check if there is any job in mirror list
4947  *
4948  * @adev: amdgpu_device pointer
4949  *
4950  * check if there is any job in mirror list
4951  */
4952 bool amdgpu_device_has_job_running(struct amdgpu_device *adev)
4953 {
4954         int i;
4955         struct drm_sched_job *job;
4956
4957         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
4958                 struct amdgpu_ring *ring = adev->rings[i];
4959
4960                 if (!ring || !ring->sched.thread)
4961                         continue;
4962
4963                 spin_lock(&ring->sched.job_list_lock);
4964                 job = list_first_entry_or_null(&ring->sched.pending_list,
4965                                                struct drm_sched_job, list);
4966                 spin_unlock(&ring->sched.job_list_lock);
4967                 if (job)
4968                         return true;
4969         }
4970         return false;
4971 }
4972
4973 /**
4974  * amdgpu_device_should_recover_gpu - check if we should try GPU recovery
4975  *
4976  * @adev: amdgpu_device pointer
4977  *
4978  * Check amdgpu_gpu_recovery and SRIOV status to see if we should try to recover
4979  * a hung GPU.
4980  */
4981 bool amdgpu_device_should_recover_gpu(struct amdgpu_device *adev)
4982 {
4983
4984         if (amdgpu_gpu_recovery == 0)
4985                 goto disabled;
4986
4987         /* Skip soft reset check in fatal error mode */
4988         if (!amdgpu_ras_is_poison_mode_supported(adev))
4989                 return true;
4990
4991         if (amdgpu_sriov_vf(adev))
4992                 return true;
4993
4994         if (amdgpu_gpu_recovery == -1) {
4995                 switch (adev->asic_type) {
4996 #ifdef CONFIG_DRM_AMDGPU_SI
4997                 case CHIP_VERDE:
4998                 case CHIP_TAHITI:
4999                 case CHIP_PITCAIRN:
5000                 case CHIP_OLAND:
5001                 case CHIP_HAINAN:
5002 #endif
5003 #ifdef CONFIG_DRM_AMDGPU_CIK
5004                 case CHIP_KAVERI:
5005                 case CHIP_KABINI:
5006                 case CHIP_MULLINS:
5007 #endif
5008                 case CHIP_CARRIZO:
5009                 case CHIP_STONEY:
5010                 case CHIP_CYAN_SKILLFISH:
5011                         goto disabled;
5012                 default:
5013                         break;
5014                 }
5015         }
5016
5017         return true;
5018
5019 disabled:
5020                 dev_info(adev->dev, "GPU recovery disabled.\n");
5021                 return false;
5022 }
5023
5024 int amdgpu_device_mode1_reset(struct amdgpu_device *adev)
5025 {
5026         u32 i;
5027         int ret = 0;
5028
5029         amdgpu_atombios_scratch_regs_engine_hung(adev, true);
5030
5031         dev_info(adev->dev, "GPU mode1 reset\n");
5032
5033         /* disable BM */
5034         pci_clear_master(adev->pdev);
5035
5036         amdgpu_device_cache_pci_state(adev->pdev);
5037
5038         if (amdgpu_dpm_is_mode1_reset_supported(adev)) {
5039                 dev_info(adev->dev, "GPU smu mode1 reset\n");
5040                 ret = amdgpu_dpm_mode1_reset(adev);
5041         } else {
5042                 dev_info(adev->dev, "GPU psp mode1 reset\n");
5043                 ret = psp_gpu_reset(adev);
5044         }
5045
5046         if (ret)
5047                 goto mode1_reset_failed;
5048
5049         amdgpu_device_load_pci_state(adev->pdev);
5050         ret = amdgpu_psp_wait_for_bootloader(adev);
5051         if (ret)
5052                 goto mode1_reset_failed;
5053
5054         /* wait for asic to come out of reset */
5055         for (i = 0; i < adev->usec_timeout; i++) {
5056                 u32 memsize = adev->nbio.funcs->get_memsize(adev);
5057
5058                 if (memsize != 0xffffffff)
5059                         break;
5060                 udelay(1);
5061         }
5062
5063         if (i >= adev->usec_timeout) {
5064                 ret = -ETIMEDOUT;
5065                 goto mode1_reset_failed;
5066         }
5067
5068         amdgpu_atombios_scratch_regs_engine_hung(adev, false);
5069
5070         return 0;
5071
5072 mode1_reset_failed:
5073         dev_err(adev->dev, "GPU mode1 reset failed\n");
5074         return ret;
5075 }
5076
5077 int amdgpu_device_pre_asic_reset(struct amdgpu_device *adev,
5078                                  struct amdgpu_reset_context *reset_context)
5079 {
5080         int i, r = 0;
5081         struct amdgpu_job *job = NULL;
5082         bool need_full_reset =
5083                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5084
5085         if (reset_context->reset_req_dev == adev)
5086                 job = reset_context->job;
5087
5088         if (amdgpu_sriov_vf(adev)) {
5089                 /* stop the data exchange thread */
5090                 amdgpu_virt_fini_data_exchange(adev);
5091         }
5092
5093         amdgpu_fence_driver_isr_toggle(adev, true);
5094
5095         /* block all schedulers and reset given job's ring */
5096         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5097                 struct amdgpu_ring *ring = adev->rings[i];
5098
5099                 if (!ring || !ring->sched.thread)
5100                         continue;
5101
5102                 /* Clear job fence from fence drv to avoid force_completion
5103                  * leave NULL and vm flush fence in fence drv
5104                  */
5105                 amdgpu_fence_driver_clear_job_fences(ring);
5106
5107                 /* after all hw jobs are reset, hw fence is meaningless, so force_completion */
5108                 amdgpu_fence_driver_force_completion(ring);
5109         }
5110
5111         amdgpu_fence_driver_isr_toggle(adev, false);
5112
5113         if (job && job->vm)
5114                 drm_sched_increase_karma(&job->base);
5115
5116         r = amdgpu_reset_prepare_hwcontext(adev, reset_context);
5117         /* If reset handler not implemented, continue; otherwise return */
5118         if (r == -EOPNOTSUPP)
5119                 r = 0;
5120         else
5121                 return r;
5122
5123         /* Don't suspend on bare metal if we are not going to HW reset the ASIC */
5124         if (!amdgpu_sriov_vf(adev)) {
5125
5126                 if (!need_full_reset)
5127                         need_full_reset = amdgpu_device_ip_need_full_reset(adev);
5128
5129                 if (!need_full_reset && amdgpu_gpu_recovery &&
5130                     amdgpu_device_ip_check_soft_reset(adev)) {
5131                         amdgpu_device_ip_pre_soft_reset(adev);
5132                         r = amdgpu_device_ip_soft_reset(adev);
5133                         amdgpu_device_ip_post_soft_reset(adev);
5134                         if (r || amdgpu_device_ip_check_soft_reset(adev)) {
5135                                 dev_info(adev->dev, "soft reset failed, will fallback to full reset!\n");
5136                                 need_full_reset = true;
5137                         }
5138                 }
5139
5140                 if (need_full_reset)
5141                         r = amdgpu_device_ip_suspend(adev);
5142                 if (need_full_reset)
5143                         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5144                 else
5145                         clear_bit(AMDGPU_NEED_FULL_RESET,
5146                                   &reset_context->flags);
5147         }
5148
5149         return r;
5150 }
5151
5152 static int amdgpu_reset_reg_dumps(struct amdgpu_device *adev)
5153 {
5154         int i;
5155
5156         lockdep_assert_held(&adev->reset_domain->sem);
5157
5158         for (i = 0; i < adev->reset_info.num_regs; i++) {
5159                 adev->reset_info.reset_dump_reg_value[i] =
5160                         RREG32(adev->reset_info.reset_dump_reg_list[i]);
5161
5162                 trace_amdgpu_reset_reg_dumps(adev->reset_info.reset_dump_reg_list[i],
5163                                              adev->reset_info.reset_dump_reg_value[i]);
5164         }
5165
5166         return 0;
5167 }
5168
5169 int amdgpu_do_asic_reset(struct list_head *device_list_handle,
5170                          struct amdgpu_reset_context *reset_context)
5171 {
5172         struct amdgpu_device *tmp_adev = NULL;
5173         bool need_full_reset, skip_hw_reset, vram_lost = false;
5174         int r = 0;
5175         bool gpu_reset_for_dev_remove = 0;
5176
5177         /* Try reset handler method first */
5178         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5179                                     reset_list);
5180         amdgpu_reset_reg_dumps(tmp_adev);
5181
5182         reset_context->reset_device_list = device_list_handle;
5183         r = amdgpu_reset_perform_reset(tmp_adev, reset_context);
5184         /* If reset handler not implemented, continue; otherwise return */
5185         if (r == -EOPNOTSUPP)
5186                 r = 0;
5187         else
5188                 return r;
5189
5190         /* Reset handler not implemented, use the default method */
5191         need_full_reset =
5192                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5193         skip_hw_reset = test_bit(AMDGPU_SKIP_HW_RESET, &reset_context->flags);
5194
5195         gpu_reset_for_dev_remove =
5196                 test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5197                         test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5198
5199         /*
5200          * ASIC reset has to be done on all XGMI hive nodes ASAP
5201          * to allow proper links negotiation in FW (within 1 sec)
5202          */
5203         if (!skip_hw_reset && need_full_reset) {
5204                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5205                         /* For XGMI run all resets in parallel to speed up the process */
5206                         if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5207                                 tmp_adev->gmc.xgmi.pending_reset = false;
5208                                 if (!queue_work(system_unbound_wq, &tmp_adev->xgmi_reset_work))
5209                                         r = -EALREADY;
5210                         } else
5211                                 r = amdgpu_asic_reset(tmp_adev);
5212
5213                         if (r) {
5214                                 dev_err(tmp_adev->dev, "ASIC reset failed with error, %d for drm dev, %s",
5215                                          r, adev_to_drm(tmp_adev)->unique);
5216                                 goto out;
5217                         }
5218                 }
5219
5220                 /* For XGMI wait for all resets to complete before proceed */
5221                 if (!r) {
5222                         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5223                                 if (tmp_adev->gmc.xgmi.num_physical_nodes > 1) {
5224                                         flush_work(&tmp_adev->xgmi_reset_work);
5225                                         r = tmp_adev->asic_reset_res;
5226                                         if (r)
5227                                                 break;
5228                                 }
5229                         }
5230                 }
5231         }
5232
5233         if (!r && amdgpu_ras_intr_triggered()) {
5234                 list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5235                         amdgpu_ras_reset_error_count(tmp_adev, AMDGPU_RAS_BLOCK__MMHUB);
5236                 }
5237
5238                 amdgpu_ras_intr_cleared();
5239         }
5240
5241         /* Since the mode1 reset affects base ip blocks, the
5242          * phase1 ip blocks need to be resumed. Otherwise there
5243          * will be a BIOS signature error and the psp bootloader
5244          * can't load kdb on the next amdgpu install.
5245          */
5246         if (gpu_reset_for_dev_remove) {
5247                 list_for_each_entry(tmp_adev, device_list_handle, reset_list)
5248                         amdgpu_device_ip_resume_phase1(tmp_adev);
5249
5250                 goto end;
5251         }
5252
5253         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5254                 if (need_full_reset) {
5255                         /* post card */
5256                         r = amdgpu_device_asic_init(tmp_adev);
5257                         if (r) {
5258                                 dev_warn(tmp_adev->dev, "asic atom init failed!");
5259                         } else {
5260                                 dev_info(tmp_adev->dev, "GPU reset succeeded, trying to resume\n");
5261
5262                                 r = amdgpu_device_ip_resume_phase1(tmp_adev);
5263                                 if (r)
5264                                         goto out;
5265
5266                                 vram_lost = amdgpu_device_check_vram_lost(tmp_adev);
5267
5268                                 amdgpu_coredump(tmp_adev, vram_lost, reset_context);
5269
5270                                 if (vram_lost) {
5271                                         DRM_INFO("VRAM is lost due to GPU reset!\n");
5272                                         amdgpu_inc_vram_lost(tmp_adev);
5273                                 }
5274
5275                                 r = amdgpu_device_fw_loading(tmp_adev);
5276                                 if (r)
5277                                         return r;
5278
5279                                 r = amdgpu_xcp_restore_partition_mode(
5280                                         tmp_adev->xcp_mgr);
5281                                 if (r)
5282                                         goto out;
5283
5284                                 r = amdgpu_device_ip_resume_phase2(tmp_adev);
5285                                 if (r)
5286                                         goto out;
5287
5288                                 if (tmp_adev->mman.buffer_funcs_ring->sched.ready)
5289                                         amdgpu_ttm_set_buffer_funcs_status(tmp_adev, true);
5290
5291                                 if (vram_lost)
5292                                         amdgpu_device_fill_reset_magic(tmp_adev);
5293
5294                                 /*
5295                                  * Add this ASIC as tracked as reset was already
5296                                  * complete successfully.
5297                                  */
5298                                 amdgpu_register_gpu_instance(tmp_adev);
5299
5300                                 if (!reset_context->hive &&
5301                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5302                                         amdgpu_xgmi_add_device(tmp_adev);
5303
5304                                 r = amdgpu_device_ip_late_init(tmp_adev);
5305                                 if (r)
5306                                         goto out;
5307
5308                                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, false);
5309
5310                                 /*
5311                                  * The GPU enters bad state once faulty pages
5312                                  * by ECC has reached the threshold, and ras
5313                                  * recovery is scheduled next. So add one check
5314                                  * here to break recovery if it indeed exceeds
5315                                  * bad page threshold, and remind user to
5316                                  * retire this GPU or setting one bigger
5317                                  * bad_page_threshold value to fix this once
5318                                  * probing driver again.
5319                                  */
5320                                 if (!amdgpu_ras_eeprom_check_err_threshold(tmp_adev)) {
5321                                         /* must succeed. */
5322                                         amdgpu_ras_resume(tmp_adev);
5323                                 } else {
5324                                         r = -EINVAL;
5325                                         goto out;
5326                                 }
5327
5328                                 /* Update PSP FW topology after reset */
5329                                 if (reset_context->hive &&
5330                                     tmp_adev->gmc.xgmi.num_physical_nodes > 1)
5331                                         r = amdgpu_xgmi_update_topology(
5332                                                 reset_context->hive, tmp_adev);
5333                         }
5334                 }
5335
5336 out:
5337                 if (!r) {
5338                         amdgpu_irq_gpu_reset_resume_helper(tmp_adev);
5339                         r = amdgpu_ib_ring_tests(tmp_adev);
5340                         if (r) {
5341                                 dev_err(tmp_adev->dev, "ib ring test failed (%d).\n", r);
5342                                 need_full_reset = true;
5343                                 r = -EAGAIN;
5344                                 goto end;
5345                         }
5346                 }
5347
5348                 if (!r)
5349                         r = amdgpu_device_recover_vram(tmp_adev);
5350                 else
5351                         tmp_adev->asic_reset_res = r;
5352         }
5353
5354 end:
5355         if (need_full_reset)
5356                 set_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5357         else
5358                 clear_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5359         return r;
5360 }
5361
5362 static void amdgpu_device_set_mp1_state(struct amdgpu_device *adev)
5363 {
5364
5365         switch (amdgpu_asic_reset_method(adev)) {
5366         case AMD_RESET_METHOD_MODE1:
5367                 adev->mp1_state = PP_MP1_STATE_SHUTDOWN;
5368                 break;
5369         case AMD_RESET_METHOD_MODE2:
5370                 adev->mp1_state = PP_MP1_STATE_RESET;
5371                 break;
5372         default:
5373                 adev->mp1_state = PP_MP1_STATE_NONE;
5374                 break;
5375         }
5376 }
5377
5378 static void amdgpu_device_unset_mp1_state(struct amdgpu_device *adev)
5379 {
5380         amdgpu_vf_error_trans_all(adev);
5381         adev->mp1_state = PP_MP1_STATE_NONE;
5382 }
5383
5384 static void amdgpu_device_resume_display_audio(struct amdgpu_device *adev)
5385 {
5386         struct pci_dev *p = NULL;
5387
5388         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5389                         adev->pdev->bus->number, 1);
5390         if (p) {
5391                 pm_runtime_enable(&(p->dev));
5392                 pm_runtime_resume(&(p->dev));
5393         }
5394
5395         pci_dev_put(p);
5396 }
5397
5398 static int amdgpu_device_suspend_display_audio(struct amdgpu_device *adev)
5399 {
5400         enum amd_reset_method reset_method;
5401         struct pci_dev *p = NULL;
5402         u64 expires;
5403
5404         /*
5405          * For now, only BACO and mode1 reset are confirmed
5406          * to suffer the audio issue without proper suspended.
5407          */
5408         reset_method = amdgpu_asic_reset_method(adev);
5409         if ((reset_method != AMD_RESET_METHOD_BACO) &&
5410              (reset_method != AMD_RESET_METHOD_MODE1))
5411                 return -EINVAL;
5412
5413         p = pci_get_domain_bus_and_slot(pci_domain_nr(adev->pdev->bus),
5414                         adev->pdev->bus->number, 1);
5415         if (!p)
5416                 return -ENODEV;
5417
5418         expires = pm_runtime_autosuspend_expiration(&(p->dev));
5419         if (!expires)
5420                 /*
5421                  * If we cannot get the audio device autosuspend delay,
5422                  * a fixed 4S interval will be used. Considering 3S is
5423                  * the audio controller default autosuspend delay setting.
5424                  * 4S used here is guaranteed to cover that.
5425                  */
5426                 expires = ktime_get_mono_fast_ns() + NSEC_PER_SEC * 4ULL;
5427
5428         while (!pm_runtime_status_suspended(&(p->dev))) {
5429                 if (!pm_runtime_suspend(&(p->dev)))
5430                         break;
5431
5432                 if (expires < ktime_get_mono_fast_ns()) {
5433                         dev_warn(adev->dev, "failed to suspend display audio\n");
5434                         pci_dev_put(p);
5435                         /* TODO: abort the succeeding gpu reset? */
5436                         return -ETIMEDOUT;
5437                 }
5438         }
5439
5440         pm_runtime_disable(&(p->dev));
5441
5442         pci_dev_put(p);
5443         return 0;
5444 }
5445
5446 static inline void amdgpu_device_stop_pending_resets(struct amdgpu_device *adev)
5447 {
5448         struct amdgpu_ras *con = amdgpu_ras_get_context(adev);
5449
5450 #if defined(CONFIG_DEBUG_FS)
5451         if (!amdgpu_sriov_vf(adev))
5452                 cancel_work(&adev->reset_work);
5453 #endif
5454
5455         if (adev->kfd.dev)
5456                 cancel_work(&adev->kfd.reset_work);
5457
5458         if (amdgpu_sriov_vf(adev))
5459                 cancel_work(&adev->virt.flr_work);
5460
5461         if (con && adev->ras_enabled)
5462                 cancel_work(&con->recovery_work);
5463
5464 }
5465
5466 /**
5467  * amdgpu_device_gpu_recover - reset the asic and recover scheduler
5468  *
5469  * @adev: amdgpu_device pointer
5470  * @job: which job trigger hang
5471  * @reset_context: amdgpu reset context pointer
5472  *
5473  * Attempt to reset the GPU if it has hung (all asics).
5474  * Attempt to do soft-reset or full-reset and reinitialize Asic
5475  * Returns 0 for success or an error on failure.
5476  */
5477
5478 int amdgpu_device_gpu_recover(struct amdgpu_device *adev,
5479                               struct amdgpu_job *job,
5480                               struct amdgpu_reset_context *reset_context)
5481 {
5482         struct list_head device_list, *device_list_handle =  NULL;
5483         bool job_signaled = false;
5484         struct amdgpu_hive_info *hive = NULL;
5485         struct amdgpu_device *tmp_adev = NULL;
5486         int i, r = 0;
5487         bool need_emergency_restart = false;
5488         bool audio_suspended = false;
5489         bool gpu_reset_for_dev_remove = false;
5490
5491         gpu_reset_for_dev_remove =
5492                         test_bit(AMDGPU_RESET_FOR_DEVICE_REMOVE, &reset_context->flags) &&
5493                                 test_bit(AMDGPU_NEED_FULL_RESET, &reset_context->flags);
5494
5495         /*
5496          * Special case: RAS triggered and full reset isn't supported
5497          */
5498         need_emergency_restart = amdgpu_ras_need_emergency_restart(adev);
5499
5500         /*
5501          * Flush RAM to disk so that after reboot
5502          * the user can read log and see why the system rebooted.
5503          */
5504         if (need_emergency_restart && amdgpu_ras_get_context(adev) &&
5505                 amdgpu_ras_get_context(adev)->reboot) {
5506                 DRM_WARN("Emergency reboot.");
5507
5508                 ksys_sync_helper();
5509                 emergency_restart();
5510         }
5511
5512         dev_info(adev->dev, "GPU %s begin!\n",
5513                 need_emergency_restart ? "jobs stop":"reset");
5514
5515         if (!amdgpu_sriov_vf(adev))
5516                 hive = amdgpu_get_xgmi_hive(adev);
5517         if (hive)
5518                 mutex_lock(&hive->hive_lock);
5519
5520         reset_context->job = job;
5521         reset_context->hive = hive;
5522         /*
5523          * Build list of devices to reset.
5524          * In case we are in XGMI hive mode, resort the device list
5525          * to put adev in the 1st position.
5526          */
5527         INIT_LIST_HEAD(&device_list);
5528         if (!amdgpu_sriov_vf(adev) && (adev->gmc.xgmi.num_physical_nodes > 1)) {
5529                 list_for_each_entry(tmp_adev, &hive->device_list, gmc.xgmi.head) {
5530                         list_add_tail(&tmp_adev->reset_list, &device_list);
5531                         if (gpu_reset_for_dev_remove && adev->shutdown)
5532                                 tmp_adev->shutdown = true;
5533                 }
5534                 if (!list_is_first(&adev->reset_list, &device_list))
5535                         list_rotate_to_front(&adev->reset_list, &device_list);
5536                 device_list_handle = &device_list;
5537         } else {
5538                 list_add_tail(&adev->reset_list, &device_list);
5539                 device_list_handle = &device_list;
5540         }
5541
5542         /* We need to lock reset domain only once both for XGMI and single device */
5543         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5544                                     reset_list);
5545         amdgpu_device_lock_reset_domain(tmp_adev->reset_domain);
5546
5547         /* block all schedulers and reset given job's ring */
5548         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5549
5550                 amdgpu_device_set_mp1_state(tmp_adev);
5551
5552                 /*
5553                  * Try to put the audio codec into suspend state
5554                  * before gpu reset started.
5555                  *
5556                  * Due to the power domain of the graphics device
5557                  * is shared with AZ power domain. Without this,
5558                  * we may change the audio hardware from behind
5559                  * the audio driver's back. That will trigger
5560                  * some audio codec errors.
5561                  */
5562                 if (!amdgpu_device_suspend_display_audio(tmp_adev))
5563                         audio_suspended = true;
5564
5565                 amdgpu_ras_set_error_query_ready(tmp_adev, false);
5566
5567                 cancel_delayed_work_sync(&tmp_adev->delayed_init_work);
5568
5569                 if (!amdgpu_sriov_vf(tmp_adev))
5570                         amdgpu_amdkfd_pre_reset(tmp_adev);
5571
5572                 /*
5573                  * Mark these ASICs to be reseted as untracked first
5574                  * And add them back after reset completed
5575                  */
5576                 amdgpu_unregister_gpu_instance(tmp_adev);
5577
5578                 drm_fb_helper_set_suspend_unlocked(adev_to_drm(tmp_adev)->fb_helper, true);
5579
5580                 /* disable ras on ALL IPs */
5581                 if (!need_emergency_restart &&
5582                       amdgpu_device_ip_need_full_reset(tmp_adev))
5583                         amdgpu_ras_suspend(tmp_adev);
5584
5585                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5586                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5587
5588                         if (!ring || !ring->sched.thread)
5589                                 continue;
5590
5591                         drm_sched_stop(&ring->sched, job ? &job->base : NULL);
5592
5593                         if (need_emergency_restart)
5594                                 amdgpu_job_stop_all_jobs_on_sched(&ring->sched);
5595                 }
5596                 atomic_inc(&tmp_adev->gpu_reset_counter);
5597         }
5598
5599         if (need_emergency_restart)
5600                 goto skip_sched_resume;
5601
5602         /*
5603          * Must check guilty signal here since after this point all old
5604          * HW fences are force signaled.
5605          *
5606          * job->base holds a reference to parent fence
5607          */
5608         if (job && dma_fence_is_signaled(&job->hw_fence)) {
5609                 job_signaled = true;
5610                 dev_info(adev->dev, "Guilty job already signaled, skipping HW reset");
5611                 goto skip_hw_reset;
5612         }
5613
5614 retry:  /* Rest of adevs pre asic reset from XGMI hive. */
5615         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5616                 if (gpu_reset_for_dev_remove) {
5617                         /* Workaroud for ASICs need to disable SMC first */
5618                         amdgpu_device_smu_fini_early(tmp_adev);
5619                 }
5620                 r = amdgpu_device_pre_asic_reset(tmp_adev, reset_context);
5621                 /*TODO Should we stop ?*/
5622                 if (r) {
5623                         dev_err(tmp_adev->dev, "GPU pre asic reset failed with err, %d for drm dev, %s ",
5624                                   r, adev_to_drm(tmp_adev)->unique);
5625                         tmp_adev->asic_reset_res = r;
5626                 }
5627
5628                 /*
5629                  * Drop all pending non scheduler resets. Scheduler resets
5630                  * were already dropped during drm_sched_stop
5631                  */
5632                 amdgpu_device_stop_pending_resets(tmp_adev);
5633         }
5634
5635         /* Actual ASIC resets if needed.*/
5636         /* Host driver will handle XGMI hive reset for SRIOV */
5637         if (amdgpu_sriov_vf(adev)) {
5638                 r = amdgpu_device_reset_sriov(adev, job ? false : true);
5639                 if (r)
5640                         adev->asic_reset_res = r;
5641
5642                 /* Aldebaran and gfx_11_0_3 support ras in SRIOV, so need resume ras during reset */
5643                 if (amdgpu_ip_version(adev, GC_HWIP, 0) ==
5644                             IP_VERSION(9, 4, 2) ||
5645                     amdgpu_ip_version(adev, GC_HWIP, 0) == IP_VERSION(11, 0, 3))
5646                         amdgpu_ras_resume(adev);
5647         } else {
5648                 r = amdgpu_do_asic_reset(device_list_handle, reset_context);
5649                 if (r && r == -EAGAIN)
5650                         goto retry;
5651
5652                 if (!r && gpu_reset_for_dev_remove)
5653                         goto recover_end;
5654         }
5655
5656 skip_hw_reset:
5657
5658         /* Post ASIC reset for all devs .*/
5659         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5660
5661                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5662                         struct amdgpu_ring *ring = tmp_adev->rings[i];
5663
5664                         if (!ring || !ring->sched.thread)
5665                                 continue;
5666
5667                         drm_sched_start(&ring->sched, true);
5668                 }
5669
5670                 if (!drm_drv_uses_atomic_modeset(adev_to_drm(tmp_adev)) && !job_signaled)
5671                         drm_helper_resume_force_mode(adev_to_drm(tmp_adev));
5672
5673                 if (tmp_adev->asic_reset_res)
5674                         r = tmp_adev->asic_reset_res;
5675
5676                 tmp_adev->asic_reset_res = 0;
5677
5678                 if (r) {
5679                         /* bad news, how to tell it to userspace ? */
5680                         dev_info(tmp_adev->dev, "GPU reset(%d) failed\n", atomic_read(&tmp_adev->gpu_reset_counter));
5681                         amdgpu_vf_error_put(tmp_adev, AMDGIM_ERROR_VF_GPU_RESET_FAIL, 0, r);
5682                 } else {
5683                         dev_info(tmp_adev->dev, "GPU reset(%d) succeeded!\n", atomic_read(&tmp_adev->gpu_reset_counter));
5684                         if (amdgpu_acpi_smart_shift_update(adev_to_drm(tmp_adev), AMDGPU_SS_DEV_D0))
5685                                 DRM_WARN("smart shift update failed\n");
5686                 }
5687         }
5688
5689 skip_sched_resume:
5690         list_for_each_entry(tmp_adev, device_list_handle, reset_list) {
5691                 /* unlock kfd: SRIOV would do it separately */
5692                 if (!need_emergency_restart && !amdgpu_sriov_vf(tmp_adev))
5693                         amdgpu_amdkfd_post_reset(tmp_adev);
5694
5695                 /* kfd_post_reset will do nothing if kfd device is not initialized,
5696                  * need to bring up kfd here if it's not be initialized before
5697                  */
5698                 if (!adev->kfd.init_complete)
5699                         amdgpu_amdkfd_device_init(adev);
5700
5701                 if (audio_suspended)
5702                         amdgpu_device_resume_display_audio(tmp_adev);
5703
5704                 amdgpu_device_unset_mp1_state(tmp_adev);
5705
5706                 amdgpu_ras_set_error_query_ready(tmp_adev, true);
5707         }
5708
5709 recover_end:
5710         tmp_adev = list_first_entry(device_list_handle, struct amdgpu_device,
5711                                             reset_list);
5712         amdgpu_device_unlock_reset_domain(tmp_adev->reset_domain);
5713
5714         if (hive) {
5715                 mutex_unlock(&hive->hive_lock);
5716                 amdgpu_put_xgmi_hive(hive);
5717         }
5718
5719         if (r)
5720                 dev_info(adev->dev, "GPU reset end with ret = %d\n", r);
5721
5722         atomic_set(&adev->reset_domain->reset_res, r);
5723         return r;
5724 }
5725
5726 /**
5727  * amdgpu_device_get_pcie_info - fence pcie info about the PCIE slot
5728  *
5729  * @adev: amdgpu_device pointer
5730  *
5731  * Fetchs and stores in the driver the PCIE capabilities (gen speed
5732  * and lanes) of the slot the device is in. Handles APUs and
5733  * virtualized environments where PCIE config space may not be available.
5734  */
5735 static void amdgpu_device_get_pcie_info(struct amdgpu_device *adev)
5736 {
5737         struct pci_dev *pdev;
5738         enum pci_bus_speed speed_cap, platform_speed_cap;
5739         enum pcie_link_width platform_link_width;
5740
5741         if (amdgpu_pcie_gen_cap)
5742                 adev->pm.pcie_gen_mask = amdgpu_pcie_gen_cap;
5743
5744         if (amdgpu_pcie_lane_cap)
5745                 adev->pm.pcie_mlw_mask = amdgpu_pcie_lane_cap;
5746
5747         /* covers APUs as well */
5748         if (pci_is_root_bus(adev->pdev->bus) && !amdgpu_passthrough(adev)) {
5749                 if (adev->pm.pcie_gen_mask == 0)
5750                         adev->pm.pcie_gen_mask = AMDGPU_DEFAULT_PCIE_GEN_MASK;
5751                 if (adev->pm.pcie_mlw_mask == 0)
5752                         adev->pm.pcie_mlw_mask = AMDGPU_DEFAULT_PCIE_MLW_MASK;
5753                 return;
5754         }
5755
5756         if (adev->pm.pcie_gen_mask && adev->pm.pcie_mlw_mask)
5757                 return;
5758
5759         pcie_bandwidth_available(adev->pdev, NULL,
5760                                  &platform_speed_cap, &platform_link_width);
5761
5762         if (adev->pm.pcie_gen_mask == 0) {
5763                 /* asic caps */
5764                 pdev = adev->pdev;
5765                 speed_cap = pcie_get_speed_cap(pdev);
5766                 if (speed_cap == PCI_SPEED_UNKNOWN) {
5767                         adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5768                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5769                                                   CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5770                 } else {
5771                         if (speed_cap == PCIE_SPEED_32_0GT)
5772                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5773                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5774                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5775                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5776                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN5);
5777                         else if (speed_cap == PCIE_SPEED_16_0GT)
5778                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5779                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5780                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5781                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN4);
5782                         else if (speed_cap == PCIE_SPEED_8_0GT)
5783                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5784                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5785                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN3);
5786                         else if (speed_cap == PCIE_SPEED_5_0GT)
5787                                 adev->pm.pcie_gen_mask |= (CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5788                                                           CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN2);
5789                         else
5790                                 adev->pm.pcie_gen_mask |= CAIL_ASIC_PCIE_LINK_SPEED_SUPPORT_GEN1;
5791                 }
5792                 /* platform caps */
5793                 if (platform_speed_cap == PCI_SPEED_UNKNOWN) {
5794                         adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5795                                                    CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5796                 } else {
5797                         if (platform_speed_cap == PCIE_SPEED_32_0GT)
5798                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5799                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5800                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5801                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4 |
5802                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN5);
5803                         else if (platform_speed_cap == PCIE_SPEED_16_0GT)
5804                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5805                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5806                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3 |
5807                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN4);
5808                         else if (platform_speed_cap == PCIE_SPEED_8_0GT)
5809                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5810                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2 |
5811                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN3);
5812                         else if (platform_speed_cap == PCIE_SPEED_5_0GT)
5813                                 adev->pm.pcie_gen_mask |= (CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1 |
5814                                                            CAIL_PCIE_LINK_SPEED_SUPPORT_GEN2);
5815                         else
5816                                 adev->pm.pcie_gen_mask |= CAIL_PCIE_LINK_SPEED_SUPPORT_GEN1;
5817
5818                 }
5819         }
5820         if (adev->pm.pcie_mlw_mask == 0) {
5821                 if (platform_link_width == PCIE_LNK_WIDTH_UNKNOWN) {
5822                         adev->pm.pcie_mlw_mask |= AMDGPU_DEFAULT_PCIE_MLW_MASK;
5823                 } else {
5824                         switch (platform_link_width) {
5825                         case PCIE_LNK_X32:
5826                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X32 |
5827                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5828                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5829                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5830                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5831                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5832                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5833                                 break;
5834                         case PCIE_LNK_X16:
5835                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X16 |
5836                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5837                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5838                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5839                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5840                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5841                                 break;
5842                         case PCIE_LNK_X12:
5843                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X12 |
5844                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5845                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5846                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5847                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5848                                 break;
5849                         case PCIE_LNK_X8:
5850                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X8 |
5851                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5852                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5853                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5854                                 break;
5855                         case PCIE_LNK_X4:
5856                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X4 |
5857                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5858                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5859                                 break;
5860                         case PCIE_LNK_X2:
5861                                 adev->pm.pcie_mlw_mask = (CAIL_PCIE_LINK_WIDTH_SUPPORT_X2 |
5862                                                           CAIL_PCIE_LINK_WIDTH_SUPPORT_X1);
5863                                 break;
5864                         case PCIE_LNK_X1:
5865                                 adev->pm.pcie_mlw_mask = CAIL_PCIE_LINK_WIDTH_SUPPORT_X1;
5866                                 break;
5867                         default:
5868                                 break;
5869                         }
5870                 }
5871         }
5872 }
5873
5874 /**
5875  * amdgpu_device_is_peer_accessible - Check peer access through PCIe BAR
5876  *
5877  * @adev: amdgpu_device pointer
5878  * @peer_adev: amdgpu_device pointer for peer device trying to access @adev
5879  *
5880  * Return true if @peer_adev can access (DMA) @adev through the PCIe
5881  * BAR, i.e. @adev is "large BAR" and the BAR matches the DMA mask of
5882  * @peer_adev.
5883  */
5884 bool amdgpu_device_is_peer_accessible(struct amdgpu_device *adev,
5885                                       struct amdgpu_device *peer_adev)
5886 {
5887 #ifdef CONFIG_HSA_AMD_P2P
5888         uint64_t address_mask = peer_adev->dev->dma_mask ?
5889                 ~*peer_adev->dev->dma_mask : ~((1ULL << 32) - 1);
5890         resource_size_t aper_limit =
5891                 adev->gmc.aper_base + adev->gmc.aper_size - 1;
5892         bool p2p_access =
5893                 !adev->gmc.xgmi.connected_to_cpu &&
5894                 !(pci_p2pdma_distance(adev->pdev, peer_adev->dev, false) < 0);
5895
5896         return pcie_p2p && p2p_access && (adev->gmc.visible_vram_size &&
5897                 adev->gmc.real_vram_size == adev->gmc.visible_vram_size &&
5898                 !(adev->gmc.aper_base & address_mask ||
5899                   aper_limit & address_mask));
5900 #else
5901         return false;
5902 #endif
5903 }
5904
5905 int amdgpu_device_baco_enter(struct drm_device *dev)
5906 {
5907         struct amdgpu_device *adev = drm_to_adev(dev);
5908         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5909
5910         if (!amdgpu_device_supports_baco(dev))
5911                 return -ENOTSUPP;
5912
5913         if (ras && adev->ras_enabled &&
5914             adev->nbio.funcs->enable_doorbell_interrupt)
5915                 adev->nbio.funcs->enable_doorbell_interrupt(adev, false);
5916
5917         return amdgpu_dpm_baco_enter(adev);
5918 }
5919
5920 int amdgpu_device_baco_exit(struct drm_device *dev)
5921 {
5922         struct amdgpu_device *adev = drm_to_adev(dev);
5923         struct amdgpu_ras *ras = amdgpu_ras_get_context(adev);
5924         int ret = 0;
5925
5926         if (!amdgpu_device_supports_baco(dev))
5927                 return -ENOTSUPP;
5928
5929         ret = amdgpu_dpm_baco_exit(adev);
5930         if (ret)
5931                 return ret;
5932
5933         if (ras && adev->ras_enabled &&
5934             adev->nbio.funcs->enable_doorbell_interrupt)
5935                 adev->nbio.funcs->enable_doorbell_interrupt(adev, true);
5936
5937         if (amdgpu_passthrough(adev) &&
5938             adev->nbio.funcs->clear_doorbell_interrupt)
5939                 adev->nbio.funcs->clear_doorbell_interrupt(adev);
5940
5941         return 0;
5942 }
5943
5944 /**
5945  * amdgpu_pci_error_detected - Called when a PCI error is detected.
5946  * @pdev: PCI device struct
5947  * @state: PCI channel state
5948  *
5949  * Description: Called when a PCI error is detected.
5950  *
5951  * Return: PCI_ERS_RESULT_NEED_RESET or PCI_ERS_RESULT_DISCONNECT.
5952  */
5953 pci_ers_result_t amdgpu_pci_error_detected(struct pci_dev *pdev, pci_channel_state_t state)
5954 {
5955         struct drm_device *dev = pci_get_drvdata(pdev);
5956         struct amdgpu_device *adev = drm_to_adev(dev);
5957         int i;
5958
5959         DRM_INFO("PCI error: detected callback, state(%d)!!\n", state);
5960
5961         if (adev->gmc.xgmi.num_physical_nodes > 1) {
5962                 DRM_WARN("No support for XGMI hive yet...");
5963                 return PCI_ERS_RESULT_DISCONNECT;
5964         }
5965
5966         adev->pci_channel_state = state;
5967
5968         switch (state) {
5969         case pci_channel_io_normal:
5970                 return PCI_ERS_RESULT_CAN_RECOVER;
5971         /* Fatal error, prepare for slot reset */
5972         case pci_channel_io_frozen:
5973                 /*
5974                  * Locking adev->reset_domain->sem will prevent any external access
5975                  * to GPU during PCI error recovery
5976                  */
5977                 amdgpu_device_lock_reset_domain(adev->reset_domain);
5978                 amdgpu_device_set_mp1_state(adev);
5979
5980                 /*
5981                  * Block any work scheduling as we do for regular GPU reset
5982                  * for the duration of the recovery
5983                  */
5984                 for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
5985                         struct amdgpu_ring *ring = adev->rings[i];
5986
5987                         if (!ring || !ring->sched.thread)
5988                                 continue;
5989
5990                         drm_sched_stop(&ring->sched, NULL);
5991                 }
5992                 atomic_inc(&adev->gpu_reset_counter);
5993                 return PCI_ERS_RESULT_NEED_RESET;
5994         case pci_channel_io_perm_failure:
5995                 /* Permanent error, prepare for device removal */
5996                 return PCI_ERS_RESULT_DISCONNECT;
5997         }
5998
5999         return PCI_ERS_RESULT_NEED_RESET;
6000 }
6001
6002 /**
6003  * amdgpu_pci_mmio_enabled - Enable MMIO and dump debug registers
6004  * @pdev: pointer to PCI device
6005  */
6006 pci_ers_result_t amdgpu_pci_mmio_enabled(struct pci_dev *pdev)
6007 {
6008
6009         DRM_INFO("PCI error: mmio enabled callback!!\n");
6010
6011         /* TODO - dump whatever for debugging purposes */
6012
6013         /* This called only if amdgpu_pci_error_detected returns
6014          * PCI_ERS_RESULT_CAN_RECOVER. Read/write to the device still
6015          * works, no need to reset slot.
6016          */
6017
6018         return PCI_ERS_RESULT_RECOVERED;
6019 }
6020
6021 /**
6022  * amdgpu_pci_slot_reset - Called when PCI slot has been reset.
6023  * @pdev: PCI device struct
6024  *
6025  * Description: This routine is called by the pci error recovery
6026  * code after the PCI slot has been reset, just before we
6027  * should resume normal operations.
6028  */
6029 pci_ers_result_t amdgpu_pci_slot_reset(struct pci_dev *pdev)
6030 {
6031         struct drm_device *dev = pci_get_drvdata(pdev);
6032         struct amdgpu_device *adev = drm_to_adev(dev);
6033         int r, i;
6034         struct amdgpu_reset_context reset_context;
6035         u32 memsize;
6036         struct list_head device_list;
6037
6038         DRM_INFO("PCI error: slot reset callback!!\n");
6039
6040         memset(&reset_context, 0, sizeof(reset_context));
6041
6042         INIT_LIST_HEAD(&device_list);
6043         list_add_tail(&adev->reset_list, &device_list);
6044
6045         /* wait for asic to come out of reset */
6046         msleep(500);
6047
6048         /* Restore PCI confspace */
6049         amdgpu_device_load_pci_state(pdev);
6050
6051         /* confirm  ASIC came out of reset */
6052         for (i = 0; i < adev->usec_timeout; i++) {
6053                 memsize = amdgpu_asic_get_config_memsize(adev);
6054
6055                 if (memsize != 0xffffffff)
6056                         break;
6057                 udelay(1);
6058         }
6059         if (memsize == 0xffffffff) {
6060                 r = -ETIME;
6061                 goto out;
6062         }
6063
6064         reset_context.method = AMD_RESET_METHOD_NONE;
6065         reset_context.reset_req_dev = adev;
6066         set_bit(AMDGPU_NEED_FULL_RESET, &reset_context.flags);
6067         set_bit(AMDGPU_SKIP_HW_RESET, &reset_context.flags);
6068
6069         adev->no_hw_access = true;
6070         r = amdgpu_device_pre_asic_reset(adev, &reset_context);
6071         adev->no_hw_access = false;
6072         if (r)
6073                 goto out;
6074
6075         r = amdgpu_do_asic_reset(&device_list, &reset_context);
6076
6077 out:
6078         if (!r) {
6079                 if (amdgpu_device_cache_pci_state(adev->pdev))
6080                         pci_restore_state(adev->pdev);
6081
6082                 DRM_INFO("PCIe error recovery succeeded\n");
6083         } else {
6084                 DRM_ERROR("PCIe error recovery failed, err:%d", r);
6085                 amdgpu_device_unset_mp1_state(adev);
6086                 amdgpu_device_unlock_reset_domain(adev->reset_domain);
6087         }
6088
6089         return r ? PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_RECOVERED;
6090 }
6091
6092 /**
6093  * amdgpu_pci_resume() - resume normal ops after PCI reset
6094  * @pdev: pointer to PCI device
6095  *
6096  * Called when the error recovery driver tells us that its
6097  * OK to resume normal operation.
6098  */
6099 void amdgpu_pci_resume(struct pci_dev *pdev)
6100 {
6101         struct drm_device *dev = pci_get_drvdata(pdev);
6102         struct amdgpu_device *adev = drm_to_adev(dev);
6103         int i;
6104
6105
6106         DRM_INFO("PCI error: resume callback!!\n");
6107
6108         /* Only continue execution for the case of pci_channel_io_frozen */
6109         if (adev->pci_channel_state != pci_channel_io_frozen)
6110                 return;
6111
6112         for (i = 0; i < AMDGPU_MAX_RINGS; ++i) {
6113                 struct amdgpu_ring *ring = adev->rings[i];
6114
6115                 if (!ring || !ring->sched.thread)
6116                         continue;
6117
6118                 drm_sched_start(&ring->sched, true);
6119         }
6120
6121         amdgpu_device_unset_mp1_state(adev);
6122         amdgpu_device_unlock_reset_domain(adev->reset_domain);
6123 }
6124
6125 bool amdgpu_device_cache_pci_state(struct pci_dev *pdev)
6126 {
6127         struct drm_device *dev = pci_get_drvdata(pdev);
6128         struct amdgpu_device *adev = drm_to_adev(dev);
6129         int r;
6130
6131         r = pci_save_state(pdev);
6132         if (!r) {
6133                 kfree(adev->pci_state);
6134
6135                 adev->pci_state = pci_store_saved_state(pdev);
6136
6137                 if (!adev->pci_state) {
6138                         DRM_ERROR("Failed to store PCI saved state");
6139                         return false;
6140                 }
6141         } else {
6142                 DRM_WARN("Failed to save PCI state, err:%d\n", r);
6143                 return false;
6144         }
6145
6146         return true;
6147 }
6148
6149 bool amdgpu_device_load_pci_state(struct pci_dev *pdev)
6150 {
6151         struct drm_device *dev = pci_get_drvdata(pdev);
6152         struct amdgpu_device *adev = drm_to_adev(dev);
6153         int r;
6154
6155         if (!adev->pci_state)
6156                 return false;
6157
6158         r = pci_load_saved_state(pdev, adev->pci_state);
6159
6160         if (!r) {
6161                 pci_restore_state(pdev);
6162         } else {
6163                 DRM_WARN("Failed to load PCI state, err:%d\n", r);
6164                 return false;
6165         }
6166
6167         return true;
6168 }
6169
6170 void amdgpu_device_flush_hdp(struct amdgpu_device *adev,
6171                 struct amdgpu_ring *ring)
6172 {
6173 #ifdef CONFIG_X86_64
6174         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6175                 return;
6176 #endif
6177         if (adev->gmc.xgmi.connected_to_cpu)
6178                 return;
6179
6180         if (ring && ring->funcs->emit_hdp_flush)
6181                 amdgpu_ring_emit_hdp_flush(ring);
6182         else
6183                 amdgpu_asic_flush_hdp(adev, ring);
6184 }
6185
6186 void amdgpu_device_invalidate_hdp(struct amdgpu_device *adev,
6187                 struct amdgpu_ring *ring)
6188 {
6189 #ifdef CONFIG_X86_64
6190         if ((adev->flags & AMD_IS_APU) && !amdgpu_passthrough(adev))
6191                 return;
6192 #endif
6193         if (adev->gmc.xgmi.connected_to_cpu)
6194                 return;
6195
6196         amdgpu_asic_invalidate_hdp(adev, ring);
6197 }
6198
6199 int amdgpu_in_reset(struct amdgpu_device *adev)
6200 {
6201         return atomic_read(&adev->reset_domain->in_gpu_reset);
6202 }
6203
6204 /**
6205  * amdgpu_device_halt() - bring hardware to some kind of halt state
6206  *
6207  * @adev: amdgpu_device pointer
6208  *
6209  * Bring hardware to some kind of halt state so that no one can touch it
6210  * any more. It will help to maintain error context when error occurred.
6211  * Compare to a simple hang, the system will keep stable at least for SSH
6212  * access. Then it should be trivial to inspect the hardware state and
6213  * see what's going on. Implemented as following:
6214  *
6215  * 1. drm_dev_unplug() makes device inaccessible to user space(IOCTLs, etc),
6216  *    clears all CPU mappings to device, disallows remappings through page faults
6217  * 2. amdgpu_irq_disable_all() disables all interrupts
6218  * 3. amdgpu_fence_driver_hw_fini() signals all HW fences
6219  * 4. set adev->no_hw_access to avoid potential crashes after setp 5
6220  * 5. amdgpu_device_unmap_mmio() clears all MMIO mappings
6221  * 6. pci_disable_device() and pci_wait_for_pending_transaction()
6222  *    flush any in flight DMA operations
6223  */
6224 void amdgpu_device_halt(struct amdgpu_device *adev)
6225 {
6226         struct pci_dev *pdev = adev->pdev;
6227         struct drm_device *ddev = adev_to_drm(adev);
6228
6229         amdgpu_xcp_dev_unplug(adev);
6230         drm_dev_unplug(ddev);
6231
6232         amdgpu_irq_disable_all(adev);
6233
6234         amdgpu_fence_driver_hw_fini(adev);
6235
6236         adev->no_hw_access = true;
6237
6238         amdgpu_device_unmap_mmio(adev);
6239
6240         pci_disable_device(pdev);
6241         pci_wait_for_pending_transaction(pdev);
6242 }
6243
6244 u32 amdgpu_device_pcie_port_rreg(struct amdgpu_device *adev,
6245                                 u32 reg)
6246 {
6247         unsigned long flags, address, data;
6248         u32 r;
6249
6250         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6251         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6252
6253         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6254         WREG32(address, reg * 4);
6255         (void)RREG32(address);
6256         r = RREG32(data);
6257         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6258         return r;
6259 }
6260
6261 void amdgpu_device_pcie_port_wreg(struct amdgpu_device *adev,
6262                                 u32 reg, u32 v)
6263 {
6264         unsigned long flags, address, data;
6265
6266         address = adev->nbio.funcs->get_pcie_port_index_offset(adev);
6267         data = adev->nbio.funcs->get_pcie_port_data_offset(adev);
6268
6269         spin_lock_irqsave(&adev->pcie_idx_lock, flags);
6270         WREG32(address, reg * 4);
6271         (void)RREG32(address);
6272         WREG32(data, v);
6273         (void)RREG32(data);
6274         spin_unlock_irqrestore(&adev->pcie_idx_lock, flags);
6275 }
6276
6277 /**
6278  * amdgpu_device_switch_gang - switch to a new gang
6279  * @adev: amdgpu_device pointer
6280  * @gang: the gang to switch to
6281  *
6282  * Try to switch to a new gang.
6283  * Returns: NULL if we switched to the new gang or a reference to the current
6284  * gang leader.
6285  */
6286 struct dma_fence *amdgpu_device_switch_gang(struct amdgpu_device *adev,
6287                                             struct dma_fence *gang)
6288 {
6289         struct dma_fence *old = NULL;
6290
6291         do {
6292                 dma_fence_put(old);
6293                 rcu_read_lock();
6294                 old = dma_fence_get_rcu_safe(&adev->gang_submit);
6295                 rcu_read_unlock();
6296
6297                 if (old == gang)
6298                         break;
6299
6300                 if (!dma_fence_is_signaled(old))
6301                         return old;
6302
6303         } while (cmpxchg((struct dma_fence __force **)&adev->gang_submit,
6304                          old, gang) != old);
6305
6306         dma_fence_put(old);
6307         return NULL;
6308 }
6309
6310 bool amdgpu_device_has_display_hardware(struct amdgpu_device *adev)
6311 {
6312         switch (adev->asic_type) {
6313 #ifdef CONFIG_DRM_AMDGPU_SI
6314         case CHIP_HAINAN:
6315 #endif
6316         case CHIP_TOPAZ:
6317                 /* chips with no display hardware */
6318                 return false;
6319 #ifdef CONFIG_DRM_AMDGPU_SI
6320         case CHIP_TAHITI:
6321         case CHIP_PITCAIRN:
6322         case CHIP_VERDE:
6323         case CHIP_OLAND:
6324 #endif
6325 #ifdef CONFIG_DRM_AMDGPU_CIK
6326         case CHIP_BONAIRE:
6327         case CHIP_HAWAII:
6328         case CHIP_KAVERI:
6329         case CHIP_KABINI:
6330         case CHIP_MULLINS:
6331 #endif
6332         case CHIP_TONGA:
6333         case CHIP_FIJI:
6334         case CHIP_POLARIS10:
6335         case CHIP_POLARIS11:
6336         case CHIP_POLARIS12:
6337         case CHIP_VEGAM:
6338         case CHIP_CARRIZO:
6339         case CHIP_STONEY:
6340                 /* chips with display hardware */
6341                 return true;
6342         default:
6343                 /* IP discovery */
6344                 if (!amdgpu_ip_version(adev, DCE_HWIP, 0) ||
6345                     (adev->harvest_ip_mask & AMD_HARVEST_IP_DMU_MASK))
6346                         return false;
6347                 return true;
6348         }
6349 }
6350
6351 uint32_t amdgpu_device_wait_on_rreg(struct amdgpu_device *adev,
6352                 uint32_t inst, uint32_t reg_addr, char reg_name[],
6353                 uint32_t expected_value, uint32_t mask)
6354 {
6355         uint32_t ret = 0;
6356         uint32_t old_ = 0;
6357         uint32_t tmp_ = RREG32(reg_addr);
6358         uint32_t loop = adev->usec_timeout;
6359
6360         while ((tmp_ & (mask)) != (expected_value)) {
6361                 if (old_ != tmp_) {
6362                         loop = adev->usec_timeout;
6363                         old_ = tmp_;
6364                 } else
6365                         udelay(1);
6366                 tmp_ = RREG32(reg_addr);
6367                 loop--;
6368                 if (!loop) {
6369                         DRM_WARN("Register(%d) [%s] failed to reach value 0x%08x != 0x%08xn",
6370                                   inst, reg_name, (uint32_t)expected_value,
6371                                   (uint32_t)(tmp_ & (mask)));
6372                         ret = -ETIMEDOUT;
6373                         break;
6374                 }
6375         }
6376         return ret;
6377 }