drivers/gpu/drm/radeon/trinity_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/pci.h>
  25 #include <linux/seq_file.h>
  26
  27 #include "r600_dpm.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "trinity_dpm.h"
  31 #include "trinityd.h"
  32 #include "vce.h"
  33
  34 #define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
  35 #define TRINITY_MINIMUM_ENGINE_CLOCK 800
  36 #define SCLK_MIN_DIV_INTV_SHIFT     12
  37 #define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
  38
  39 #ifndef TRINITY_MGCG_SEQUENCE
  40 #define TRINITY_MGCG_SEQUENCE  100
  41
  42 static const u32 trinity_mgcg_shls_default[] = {
  43         /* Register, Value, Mask */
  44         0x0000802c, 0xc0000000, 0xffffffff,
  45         0x00003fc4, 0xc0000000, 0xffffffff,
  46         0x00005448, 0x00000100, 0xffffffff,
  47         0x000055e4, 0x00000100, 0xffffffff,
  48         0x0000160c, 0x00000100, 0xffffffff,
  49         0x00008984, 0x06000100, 0xffffffff,
  50         0x0000c164, 0x00000100, 0xffffffff,
  51         0x00008a18, 0x00000100, 0xffffffff,
  52         0x0000897c, 0x06000100, 0xffffffff,
  53         0x00008b28, 0x00000100, 0xffffffff,
  54         0x00009144, 0x00800200, 0xffffffff,
  55         0x00009a60, 0x00000100, 0xffffffff,
  56         0x00009868, 0x00000100, 0xffffffff,
  57         0x00008d58, 0x00000100, 0xffffffff,
  58         0x00009510, 0x00000100, 0xffffffff,
  59         0x0000949c, 0x00000100, 0xffffffff,
  60         0x00009654, 0x00000100, 0xffffffff,
  61         0x00009030, 0x00000100, 0xffffffff,
  62         0x00009034, 0x00000100, 0xffffffff,
  63         0x00009038, 0x00000100, 0xffffffff,
  64         0x0000903c, 0x00000100, 0xffffffff,
  65         0x00009040, 0x00000100, 0xffffffff,
  66         0x0000a200, 0x00000100, 0xffffffff,
  67         0x0000a204, 0x00000100, 0xffffffff,
  68         0x0000a208, 0x00000100, 0xffffffff,
  69         0x0000a20c, 0x00000100, 0xffffffff,
  70         0x00009744, 0x00000100, 0xffffffff,
  71         0x00003f80, 0x00000100, 0xffffffff,
  72         0x0000a210, 0x00000100, 0xffffffff,
  73         0x0000a214, 0x00000100, 0xffffffff,
  74         0x000004d8, 0x00000100, 0xffffffff,
  75         0x00009664, 0x00000100, 0xffffffff,
  76         0x00009698, 0x00000100, 0xffffffff,
  77         0x000004d4, 0x00000200, 0xffffffff,
  78         0x000004d0, 0x00000000, 0xffffffff,
  79         0x000030cc, 0x00000104, 0xffffffff,
  80         0x0000d0c0, 0x00000100, 0xffffffff,
  81         0x0000d8c0, 0x00000100, 0xffffffff,
  82         0x0000951c, 0x00010000, 0xffffffff,
  83         0x00009160, 0x00030002, 0xffffffff,
  84         0x00009164, 0x00050004, 0xffffffff,
  85         0x00009168, 0x00070006, 0xffffffff,
  86         0x00009178, 0x00070000, 0xffffffff,
  87         0x0000917c, 0x00030002, 0xffffffff,
  88         0x00009180, 0x00050004, 0xffffffff,
  89         0x0000918c, 0x00010006, 0xffffffff,
  90         0x00009190, 0x00090008, 0xffffffff,
  91         0x00009194, 0x00070000, 0xffffffff,
  92         0x00009198, 0x00030002, 0xffffffff,
  93         0x0000919c, 0x00050004, 0xffffffff,
  94         0x000091a8, 0x00010006, 0xffffffff,
  95         0x000091ac, 0x00090008, 0xffffffff,
  96         0x000091b0, 0x00070000, 0xffffffff,
  97         0x000091b4, 0x00030002, 0xffffffff,
  98         0x000091b8, 0x00050004, 0xffffffff,
  99         0x000091c4, 0x00010006, 0xffffffff,
 100         0x000091c8, 0x00090008, 0xffffffff,
 101         0x000091cc, 0x00070000, 0xffffffff,
 102         0x000091d0, 0x00030002, 0xffffffff,
 103         0x000091d4, 0x00050004, 0xffffffff,
 104         0x000091e0, 0x00010006, 0xffffffff,
 105         0x000091e4, 0x00090008, 0xffffffff,
 106         0x000091e8, 0x00000000, 0xffffffff,
 107         0x000091ec, 0x00070000, 0xffffffff,
 108         0x000091f0, 0x00030002, 0xffffffff,
 109         0x000091f4, 0x00050004, 0xffffffff,
 110         0x00009200, 0x00010006, 0xffffffff,
 111         0x00009204, 0x00090008, 0xffffffff,
 112         0x00009208, 0x00070000, 0xffffffff,
 113         0x0000920c, 0x00030002, 0xffffffff,
 114         0x00009210, 0x00050004, 0xffffffff,
 115         0x0000921c, 0x00010006, 0xffffffff,
 116         0x00009220, 0x00090008, 0xffffffff,
 117         0x00009294, 0x00000000, 0xffffffff
 118 };
 119 #endif
 120
 121 #ifndef TRINITY_SYSLS_SEQUENCE
 122 #define TRINITY_SYSLS_SEQUENCE  100
 123
 124 static const u32 trinity_sysls_disable[] = {
 125         /* Register, Value, Mask */
 126         0x0000d0c0, 0x00000000, 0xffffffff,
 127         0x0000d8c0, 0x00000000, 0xffffffff,
 128         0x000055e8, 0x00000000, 0xffffffff,
 129         0x0000d0bc, 0x00000000, 0xffffffff,
 130         0x0000d8bc, 0x00000000, 0xffffffff,
 131         0x000015c0, 0x00041401, 0xffffffff,
 132         0x0000264c, 0x00040400, 0xffffffff,
 133         0x00002648, 0x00040400, 0xffffffff,
 134         0x00002650, 0x00040400, 0xffffffff,
 135         0x000020b8, 0x00040400, 0xffffffff,
 136         0x000020bc, 0x00040400, 0xffffffff,
 137         0x000020c0, 0x00040c80, 0xffffffff,
 138         0x0000f4a0, 0x000000c0, 0xffffffff,
 139         0x0000f4a4, 0x00680000, 0xffffffff,
 140         0x00002f50, 0x00000404, 0xffffffff,
 141         0x000004c8, 0x00000001, 0xffffffff,
 142         0x0000641c, 0x00007ffd, 0xffffffff,
 143         0x00000c7c, 0x0000ff00, 0xffffffff,
 144         0x00006dfc, 0x0000007f, 0xffffffff
 145 };
 146
 147 static const u32 trinity_sysls_enable[] = {
 148         /* Register, Value, Mask */
 149         0x000055e8, 0x00000001, 0xffffffff,
 150         0x0000d0bc, 0x00000100, 0xffffffff,
 151         0x0000d8bc, 0x00000100, 0xffffffff,
 152         0x000015c0, 0x000c1401, 0xffffffff,
 153         0x0000264c, 0x000c0400, 0xffffffff,
 154         0x00002648, 0x000c0400, 0xffffffff,
 155         0x00002650, 0x000c0400, 0xffffffff,
 156         0x000020b8, 0x000c0400, 0xffffffff,
 157         0x000020bc, 0x000c0400, 0xffffffff,
 158         0x000020c0, 0x000c0c80, 0xffffffff,
 159         0x0000f4a0, 0x000000c0, 0xffffffff,
 160         0x0000f4a4, 0x00680fff, 0xffffffff,
 161         0x00002f50, 0x00000903, 0xffffffff,
 162         0x000004c8, 0x00000000, 0xffffffff,
 163         0x0000641c, 0x00000000, 0xffffffff,
 164         0x00000c7c, 0x00000000, 0xffffffff,
 165         0x00006dfc, 0x00000000, 0xffffffff
 166 };
 167 #endif
 168
 169 static const u32 trinity_override_mgpg_sequences[] = {
 170         /* Register, Value */
 171         0x00000200, 0xE030032C,
 172         0x00000204, 0x00000FFF,
 173         0x00000200, 0xE0300058,
 174         0x00000204, 0x00030301,
 175         0x00000200, 0xE0300054,
 176         0x00000204, 0x500010FF,
 177         0x00000200, 0xE0300074,
 178         0x00000204, 0x00030301,
 179         0x00000200, 0xE0300070,
 180         0x00000204, 0x500010FF,
 181         0x00000200, 0xE0300090,
 182         0x00000204, 0x00030301,
 183         0x00000200, 0xE030008C,
 184         0x00000204, 0x500010FF,
 185         0x00000200, 0xE03000AC,
 186         0x00000204, 0x00030301,
 187         0x00000200, 0xE03000A8,
 188         0x00000204, 0x500010FF,
 189         0x00000200, 0xE03000C8,
 190         0x00000204, 0x00030301,
 191         0x00000200, 0xE03000C4,
 192         0x00000204, 0x500010FF,
 193         0x00000200, 0xE03000E4,
 194         0x00000204, 0x00030301,
 195         0x00000200, 0xE03000E0,
 196         0x00000204, 0x500010FF,
 197         0x00000200, 0xE0300100,
 198         0x00000204, 0x00030301,
 199         0x00000200, 0xE03000FC,
 200         0x00000204, 0x500010FF,
 201         0x00000200, 0xE0300058,
 202         0x00000204, 0x00030303,
 203         0x00000200, 0xE0300054,
 204         0x00000204, 0x600010FF,
 205         0x00000200, 0xE0300074,
 206         0x00000204, 0x00030303,
 207         0x00000200, 0xE0300070,
 208         0x00000204, 0x600010FF,
 209         0x00000200, 0xE0300090,
 210         0x00000204, 0x00030303,
 211         0x00000200, 0xE030008C,
 212         0x00000204, 0x600010FF,
 213         0x00000200, 0xE03000AC,
 214         0x00000204, 0x00030303,
 215         0x00000200, 0xE03000A8,
 216         0x00000204, 0x600010FF,
 217         0x00000200, 0xE03000C8,
 218         0x00000204, 0x00030303,
 219         0x00000200, 0xE03000C4,
 220         0x00000204, 0x600010FF,
 221         0x00000200, 0xE03000E4,
 222         0x00000204, 0x00030303,
 223         0x00000200, 0xE03000E0,
 224         0x00000204, 0x600010FF,
 225         0x00000200, 0xE0300100,
 226         0x00000204, 0x00030303,
 227         0x00000200, 0xE03000FC,
 228         0x00000204, 0x600010FF,
 229         0x00000200, 0xE0300058,
 230         0x00000204, 0x00030303,
 231         0x00000200, 0xE0300054,
 232         0x00000204, 0x700010FF,
 233         0x00000200, 0xE0300074,
 234         0x00000204, 0x00030303,
 235         0x00000200, 0xE0300070,
 236         0x00000204, 0x700010FF,
 237         0x00000200, 0xE0300090,
 238         0x00000204, 0x00030303,
 239         0x00000200, 0xE030008C,
 240         0x00000204, 0x700010FF,
 241         0x00000200, 0xE03000AC,
 242         0x00000204, 0x00030303,
 243         0x00000200, 0xE03000A8,
 244         0x00000204, 0x700010FF,
 245         0x00000200, 0xE03000C8,
 246         0x00000204, 0x00030303,
 247         0x00000200, 0xE03000C4,
 248         0x00000204, 0x700010FF,
 249         0x00000200, 0xE03000E4,
 250         0x00000204, 0x00030303,
 251         0x00000200, 0xE03000E0,
 252         0x00000204, 0x700010FF,
 253         0x00000200, 0xE0300100,
 254         0x00000204, 0x00030303,
 255         0x00000200, 0xE03000FC,
 256         0x00000204, 0x700010FF,
 257         0x00000200, 0xE0300058,
 258         0x00000204, 0x00010303,
 259         0x00000200, 0xE0300054,
 260         0x00000204, 0x800010FF,
 261         0x00000200, 0xE0300074,
 262         0x00000204, 0x00010303,
 263         0x00000200, 0xE0300070,
 264         0x00000204, 0x800010FF,
 265         0x00000200, 0xE0300090,
 266         0x00000204, 0x00010303,
 267         0x00000200, 0xE030008C,
 268         0x00000204, 0x800010FF,
 269         0x00000200, 0xE03000AC,
 270         0x00000204, 0x00010303,
 271         0x00000200, 0xE03000A8,
 272         0x00000204, 0x800010FF,
 273         0x00000200, 0xE03000C4,
 274         0x00000204, 0x800010FF,
 275         0x00000200, 0xE03000C8,
 276         0x00000204, 0x00010303,
 277         0x00000200, 0xE03000E4,
 278         0x00000204, 0x00010303,
 279         0x00000200, 0xE03000E0,
 280         0x00000204, 0x800010FF,
 281         0x00000200, 0xE0300100,
 282         0x00000204, 0x00010303,
 283         0x00000200, 0xE03000FC,
 284         0x00000204, 0x800010FF,
 285         0x00000200, 0x0001f198,
 286         0x00000204, 0x0003ffff,
 287         0x00000200, 0x0001f19C,
 288         0x00000204, 0x3fffffff,
 289         0x00000200, 0xE030032C,
 290         0x00000204, 0x00000000,
 291 };
 292
 293 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 294                                                    const u32 *seq, u32 count);
 295 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
 296 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
 297                                              struct radeon_ps *new_rps,
 298                                              struct radeon_ps *old_rps);
 299
 300 static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
 301 {
 302         struct trinity_ps *ps = rps->ps_priv;
 303
 304         return ps;
 305 }
 306
 307 static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
 308 {
 309         struct trinity_power_info *pi = rdev->pm.dpm.priv;
 310
 311         return pi;
 312 }
 313
 314 static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
 315 {
 316         struct trinity_power_info *pi = trinity_get_pi(rdev);
 317         u32 p, u;
 318         u32 value;
 319         struct atom_clock_dividers dividers;
 320         u32 xclk = radeon_get_xclk(rdev);
 321         u32 sssd = 1;
 322         int ret;
 323         u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
 324
 325         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 326                                              25000, false, &dividers);
 327         if (ret)
 328                 return;
 329
 330         value = RREG32_SMC(GFX_POWER_GATING_CNTL);
 331         value &= ~(SSSD_MASK | PDS_DIV_MASK);
 332         if (sssd)
 333                 value |= SSSD(1);
 334         value |= PDS_DIV(dividers.post_div);
 335         WREG32_SMC(GFX_POWER_GATING_CNTL, value);
 336
 337         r600_calculate_u_and_p(500, xclk, 16, &p, &u);
 338
 339         WREG32(CG_PG_CTRL, SP(p) | SU(u));
 340
 341         WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
 342
 343         /* XXX double check hw_rev */
 344         if (pi->override_dynamic_mgpg && (hw_rev == 0))
 345                 trinity_override_dynamic_mg_powergating(rdev);
 346
 347 }
 348
 349 #define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
 350 #define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
 351 #define CGTS_SM_CTRL_REG_DISABLE    0x00600000
 352 #define CGTS_SM_CTRL_REG_ENABLE     0x96944200
 353
 354 static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
 355                                           bool enable)
 356 {
 357         u32 local0;
 358         u32 local1;
 359
 360         if (enable) {
 361                 local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 362                 local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 363
 364                 WREG32_CG(CG_CGTT_LOCAL_0,
 365                           (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK));
 366                 WREG32_CG(CG_CGTT_LOCAL_1,
 367                           (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK));
 368
 369                 WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
 370         } else {
 371                 WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
 372
 373                 local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 374                 local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 375
 376                 WREG32_CG(CG_CGTT_LOCAL_0,
 377                           CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK));
 378                 WREG32_CG(CG_CGTT_LOCAL_1,
 379                           CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK));
 380         }
 381 }
 382
 383 static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
 384 {
 385         u32 count;
 386         const u32 *seq = NULL;
 387
 388         seq = &trinity_mgcg_shls_default[0];
 389         count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
 390
 391         trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 392 }
 393
 394 static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
 395                                            bool enable)
 396 {
 397         if (enable) {
 398                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 399         } else {
 400                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 401                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 402                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 403                 RREG32(GB_ADDR_CONFIG);
 404         }
 405 }
 406
 407 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 408                                                    const u32 *seq, u32 count)
 409 {
 410         u32 i, length = count * 3;
 411
 412         for (i = 0; i < length; i += 3)
 413                 WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
 414 }
 415
 416 static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
 417                                                     const u32 *seq, u32 count)
 418 {
 419         u32  i, length = count * 2;
 420
 421         for (i = 0; i < length; i += 2)
 422                 WREG32(seq[i], seq[i+1]);
 423
 424 }
 425
 426 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
 427 {
 428         u32 count;
 429         const u32 *seq = NULL;
 430
 431         seq = &trinity_override_mgpg_sequences[0];
 432         count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
 433
 434         trinity_program_override_mgpg_sequences(rdev, seq, count);
 435 }
 436
 437 static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
 438                                           bool enable)
 439 {
 440         u32 count;
 441         const u32 *seq = NULL;
 442
 443         if (enable) {
 444                 seq = &trinity_sysls_enable[0];
 445                 count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
 446         } else {
 447                 seq = &trinity_sysls_disable[0];
 448                 count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
 449         }
 450
 451         trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 452 }
 453
 454 static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
 455                                            bool enable)
 456 {
 457         if (enable) {
 458                 if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
 459                         WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
 460
 461                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
 462         } else {
 463                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
 464                 RREG32(GB_ADDR_CONFIG);
 465         }
 466 }
 467
 468 static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
 469                                             bool enable)
 470 {
 471         u32 value;
 472
 473         if (enable) {
 474                 value = RREG32_SMC(PM_I_CNTL_1);
 475                 value &= ~DS_PG_CNTL_MASK;
 476                 value |= DS_PG_CNTL(1);
 477                 WREG32_SMC(PM_I_CNTL_1, value);
 478
 479                 value = RREG32_SMC(SMU_S_PG_CNTL);
 480                 value &= ~DS_PG_EN_MASK;
 481                 value |= DS_PG_EN(1);
 482                 WREG32_SMC(SMU_S_PG_CNTL, value);
 483         } else {
 484                 value = RREG32_SMC(SMU_S_PG_CNTL);
 485                 value &= ~DS_PG_EN_MASK;
 486                 WREG32_SMC(SMU_S_PG_CNTL, value);
 487
 488                 value = RREG32_SMC(PM_I_CNTL_1);
 489                 value &= ~DS_PG_CNTL_MASK;
 490                 WREG32_SMC(PM_I_CNTL_1, value);
 491         }
 492
 493         trinity_gfx_dynamic_mgpg_config(rdev);
 494
 495 }
 496
 497 static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
 498 {
 499         struct trinity_power_info *pi = trinity_get_pi(rdev);
 500
 501         if (pi->enable_gfx_clock_gating)
 502                 sumo_gfx_clockgating_initialize(rdev);
 503         if (pi->enable_mg_clock_gating)
 504                 trinity_mg_clockgating_initialize(rdev);
 505         if (pi->enable_gfx_power_gating)
 506                 trinity_gfx_powergating_initialize(rdev);
 507         if (pi->enable_mg_clock_gating) {
 508                 trinity_ls_clockgating_enable(rdev, true);
 509                 trinity_mg_clockgating_enable(rdev, true);
 510         }
 511         if (pi->enable_gfx_clock_gating)
 512                 trinity_gfx_clockgating_enable(rdev, true);
 513         if (pi->enable_gfx_dynamic_mgpg)
 514                 trinity_gfx_dynamic_mgpg_enable(rdev, true);
 515         if (pi->enable_gfx_power_gating)
 516                 trinity_gfx_powergating_enable(rdev, true);
 517 }
 518
 519 static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
 520 {
 521         struct trinity_power_info *pi = trinity_get_pi(rdev);
 522
 523         if (pi->enable_gfx_power_gating)
 524                 trinity_gfx_powergating_enable(rdev, false);
 525         if (pi->enable_gfx_dynamic_mgpg)
 526                 trinity_gfx_dynamic_mgpg_enable(rdev, false);
 527         if (pi->enable_gfx_clock_gating)
 528                 trinity_gfx_clockgating_enable(rdev, false);
 529         if (pi->enable_mg_clock_gating) {
 530                 trinity_mg_clockgating_enable(rdev, false);
 531                 trinity_ls_clockgating_enable(rdev, false);
 532         }
 533 }
 534
 535 static void trinity_set_divider_value(struct radeon_device *rdev,
 536                                       u32 index, u32 sclk)
 537 {
 538         struct atom_clock_dividers  dividers;
 539         int ret;
 540         u32 value;
 541         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 542
 543         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 544                                              sclk, false, &dividers);
 545         if (ret)
 546                 return;
 547
 548         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 549         value &= ~CLK_DIVIDER_MASK;
 550         value |= CLK_DIVIDER(dividers.post_div);
 551         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 552
 553         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 554                                              sclk/2, false, &dividers);
 555         if (ret)
 556                 return;
 557
 558         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
 559         value &= ~PD_SCLK_DIVIDER_MASK;
 560         value |= PD_SCLK_DIVIDER(dividers.post_div);
 561         WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
 562 }
 563
 564 static void trinity_set_ds_dividers(struct radeon_device *rdev,
 565                                     u32 index, u32 divider)
 566 {
 567         u32 value;
 568         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 569
 570         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 571         value &= ~DS_DIV_MASK;
 572         value |= DS_DIV(divider);
 573         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 574 }
 575
 576 static void trinity_set_ss_dividers(struct radeon_device *rdev,
 577                                     u32 index, u32 divider)
 578 {
 579         u32 value;
 580         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 581
 582         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 583         value &= ~DS_SH_DIV_MASK;
 584         value |= DS_SH_DIV(divider);
 585         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 586 }
 587
 588 static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
 589 {
 590         struct trinity_power_info *pi = trinity_get_pi(rdev);
 591         u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
 592         u32 value;
 593         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 594
 595         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 596         value &= ~VID_MASK;
 597         value |= VID(vid_7bit);
 598         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 599
 600         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 601         value &= ~LVRT_MASK;
 602         value |= LVRT(0);
 603         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 604 }
 605
 606 static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
 607                                        u32 index, u32 gnb_slow)
 608 {
 609         u32 value;
 610         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 611
 612         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 613         value &= ~GNB_SLOW_MASK;
 614         value |= GNB_SLOW(gnb_slow);
 615         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 616 }
 617
 618 static void trinity_set_force_nbp_state(struct radeon_device *rdev,
 619                                         u32 index, u32 force_nbp_state)
 620 {
 621         u32 value;
 622         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 623
 624         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 625         value &= ~FORCE_NBPS1_MASK;
 626         value |= FORCE_NBPS1(force_nbp_state);
 627         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 628 }
 629
 630 static void trinity_set_display_wm(struct radeon_device *rdev,
 631                                    u32 index, u32 wm)
 632 {
 633         u32 value;
 634         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 635
 636         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 637         value &= ~DISPLAY_WM_MASK;
 638         value |= DISPLAY_WM(wm);
 639         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 640 }
 641
 642 static void trinity_set_vce_wm(struct radeon_device *rdev,
 643                                u32 index, u32 wm)
 644 {
 645         u32 value;
 646         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 647
 648         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 649         value &= ~VCE_WM_MASK;
 650         value |= VCE_WM(wm);
 651         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 652 }
 653
 654 static void trinity_set_at(struct radeon_device *rdev,
 655                            u32 index, u32 at)
 656 {
 657         u32 value;
 658         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 659
 660         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
 661         value &= ~AT_MASK;
 662         value |= AT(at);
 663         WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
 664 }
 665
 666 static void trinity_program_power_level(struct radeon_device *rdev,
 667                                         struct trinity_pl *pl, u32 index)
 668 {
 669         struct trinity_power_info *pi = trinity_get_pi(rdev);
 670
 671         if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
 672                 return;
 673
 674         trinity_set_divider_value(rdev, index, pl->sclk);
 675         trinity_set_vid(rdev, index, pl->vddc_index);
 676         trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
 677         trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
 678         trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
 679         trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
 680         trinity_set_display_wm(rdev, index, pl->display_wm);
 681         trinity_set_vce_wm(rdev, index, pl->vce_wm);
 682         trinity_set_at(rdev, index, pi->at[index]);
 683 }
 684
 685 static void trinity_power_level_enable_disable(struct radeon_device *rdev,
 686                                                u32 index, bool enable)
 687 {
 688         u32 value;
 689         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 690
 691         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 692         value &= ~STATE_VALID_MASK;
 693         if (enable)
 694                 value |= STATE_VALID(1);
 695         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 696 }
 697
 698 static bool trinity_dpm_enabled(struct radeon_device *rdev)
 699 {
 700         if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
 701                 return true;
 702         else
 703                 return false;
 704 }
 705
 706 static void trinity_start_dpm(struct radeon_device *rdev)
 707 {
 708         u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 709
 710         value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
 711         value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
 712         WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
 713
 714         WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 715         WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
 716
 717         trinity_dpm_config(rdev, true);
 718 }
 719
 720 static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
 721 {
 722         int i;
 723
 724         for (i = 0; i < rdev->usec_timeout; i++) {
 725                 if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
 726                         break;
 727                 udelay(1);
 728         }
 729         for (i = 0; i < rdev->usec_timeout; i++) {
 730                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
 731                         break;
 732                 udelay(1);
 733         }
 734         for (i = 0; i < rdev->usec_timeout; i++) {
 735                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 736                         break;
 737                 udelay(1);
 738         }
 739 }
 740
 741 static void trinity_stop_dpm(struct radeon_device *rdev)
 742 {
 743         u32 sclk_dpm_cntl;
 744
 745         WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
 746
 747         sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 748         sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
 749         WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
 750
 751         trinity_dpm_config(rdev, false);
 752 }
 753
 754 static void trinity_start_am(struct radeon_device *rdev)
 755 {
 756         WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 757 }
 758
 759 static void trinity_reset_am(struct radeon_device *rdev)
 760 {
 761         WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
 762                  ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 763 }
 764
 765 static void trinity_wait_for_level_0(struct radeon_device *rdev)
 766 {
 767         int i;
 768
 769         for (i = 0; i < rdev->usec_timeout; i++) {
 770                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 771                         break;
 772                 udelay(1);
 773         }
 774 }
 775
 776 static void trinity_enable_power_level_0(struct radeon_device *rdev)
 777 {
 778         trinity_power_level_enable_disable(rdev, 0, true);
 779 }
 780
 781 static void trinity_force_level_0(struct radeon_device *rdev)
 782 {
 783         trinity_dpm_force_state(rdev, 0);
 784 }
 785
 786 static void trinity_unforce_levels(struct radeon_device *rdev)
 787 {
 788         trinity_dpm_no_forced_level(rdev);
 789 }
 790
 791 static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
 792                                                 struct radeon_ps *new_rps,
 793                                                 struct radeon_ps *old_rps)
 794 {
 795         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 796         struct trinity_ps *old_ps = trinity_get_ps(old_rps);
 797         u32 i;
 798         u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
 799
 800         for (i = 0; i < new_ps->num_levels; i++) {
 801                 trinity_program_power_level(rdev, &new_ps->levels[i], i);
 802                 trinity_power_level_enable_disable(rdev, i, true);
 803         }
 804
 805         for (i = new_ps->num_levels; i < n_current_state_levels; i++)
 806                 trinity_power_level_enable_disable(rdev, i, false);
 807 }
 808
 809 static void trinity_program_bootup_state(struct radeon_device *rdev)
 810 {
 811         struct trinity_power_info *pi = trinity_get_pi(rdev);
 812         u32 i;
 813
 814         trinity_program_power_level(rdev, &pi->boot_pl, 0);
 815         trinity_power_level_enable_disable(rdev, 0, true);
 816
 817         for (i = 1; i < 8; i++)
 818                 trinity_power_level_enable_disable(rdev, i, false);
 819 }
 820
 821 static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
 822                                           struct radeon_ps *rps)
 823 {
 824         struct trinity_ps *ps = trinity_get_ps(rps);
 825         u32 uvdstates = (ps->vclk_low_divider |
 826                          ps->vclk_high_divider << 8 |
 827                          ps->dclk_low_divider << 16 |
 828                          ps->dclk_high_divider << 24);
 829
 830         WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
 831 }
 832
 833 static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
 834                                            u32 interval)
 835 {
 836         u32 p, u;
 837         u32 tp = RREG32_SMC(PM_TP);
 838         u32 val;
 839         u32 xclk = radeon_get_xclk(rdev);
 840
 841         r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
 842
 843         val = (p + tp - 1) / tp;
 844
 845         WREG32_SMC(SMU_UVD_DPM_CNTL, val);
 846 }
 847
 848 static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
 849 {
 850         if ((rps->vclk == 0) && (rps->dclk == 0))
 851                 return true;
 852         else
 853                 return false;
 854 }
 855
 856 static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
 857                                      struct radeon_ps *rps2)
 858 {
 859         struct trinity_ps *ps1 = trinity_get_ps(rps1);
 860         struct trinity_ps *ps2 = trinity_get_ps(rps2);
 861
 862         if ((rps1->vclk == rps2->vclk) &&
 863             (rps1->dclk == rps2->dclk) &&
 864             (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
 865             (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
 866             (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
 867             (ps1->dclk_high_divider == ps2->dclk_high_divider))
 868                 return true;
 869         else
 870                 return false;
 871 }
 872
 873 static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
 874                                      struct radeon_ps *new_rps,
 875                                      struct radeon_ps *old_rps)
 876 {
 877         struct trinity_power_info *pi = trinity_get_pi(rdev);
 878
 879         if (pi->enable_gfx_power_gating) {
 880                 trinity_gfx_powergating_enable(rdev, false);
 881         }
 882
 883         if (pi->uvd_dpm) {
 884                 if (trinity_uvd_clocks_zero(new_rps) &&
 885                     !trinity_uvd_clocks_zero(old_rps)) {
 886                         trinity_setup_uvd_dpm_interval(rdev, 0);
 887                 } else if (!trinity_uvd_clocks_zero(new_rps)) {
 888                         trinity_setup_uvd_clock_table(rdev, new_rps);
 889
 890                         if (trinity_uvd_clocks_zero(old_rps)) {
 891                                 u32 tmp = RREG32(CG_MISC_REG);
 892                                 tmp &= 0xfffffffd;
 893                                 WREG32(CG_MISC_REG, tmp);
 894
 895                                 radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 896
 897                                 trinity_setup_uvd_dpm_interval(rdev, 3000);
 898                         }
 899                 }
 900                 trinity_uvd_dpm_config(rdev);
 901         } else {
 902                 if (trinity_uvd_clocks_zero(new_rps) ||
 903                     trinity_uvd_clocks_equal(new_rps, old_rps))
 904                         return;
 905
 906                 radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 907         }
 908
 909         if (pi->enable_gfx_power_gating) {
 910                 trinity_gfx_powergating_enable(rdev, true);
 911         }
 912 }
 913
 914 static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
 915                                                        struct radeon_ps *new_rps,
 916                                                        struct radeon_ps *old_rps)
 917 {
 918         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 919         struct trinity_ps *current_ps = trinity_get_ps(new_rps);
 920
 921         if (new_ps->levels[new_ps->num_levels - 1].sclk >=
 922             current_ps->levels[current_ps->num_levels - 1].sclk)
 923                 return;
 924
 925         trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 926 }
 927
 928 static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
 929                                                       struct radeon_ps *new_rps,
 930                                                       struct radeon_ps *old_rps)
 931 {
 932         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 933         struct trinity_ps *current_ps = trinity_get_ps(old_rps);
 934
 935         if (new_ps->levels[new_ps->num_levels - 1].sclk <
 936             current_ps->levels[current_ps->num_levels - 1].sclk)
 937                 return;
 938
 939         trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 940 }
 941
 942 static void trinity_set_vce_clock(struct radeon_device *rdev,
 943                                   struct radeon_ps *new_rps,
 944                                   struct radeon_ps *old_rps)
 945 {
 946         if ((old_rps->evclk != new_rps->evclk) ||
 947             (old_rps->ecclk != new_rps->ecclk)) {
 948                 /* turn the clocks on when encoding, off otherwise */
 949                 if (new_rps->evclk || new_rps->ecclk)
 950                         vce_v1_0_enable_mgcg(rdev, false);
 951                 else
 952                         vce_v1_0_enable_mgcg(rdev, true);
 953                 radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
 954         }
 955 }
 956
 957 static void trinity_program_ttt(struct radeon_device *rdev)
 958 {
 959         struct trinity_power_info *pi = trinity_get_pi(rdev);
 960         u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
 961
 962         value &= ~(HT_MASK | LT_MASK);
 963         value |= HT((pi->thermal_auto_throttling + 49) * 8);
 964         value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
 965         WREG32_SMC(SMU_SCLK_DPM_TTT, value);
 966 }
 967
 968 static void trinity_enable_att(struct radeon_device *rdev)
 969 {
 970         u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
 971
 972         value &= ~SCLK_TT_EN_MASK;
 973         value |= SCLK_TT_EN(1);
 974         WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
 975 }
 976
 977 static void trinity_program_sclk_dpm(struct radeon_device *rdev)
 978 {
 979         u32 p, u;
 980         u32 tp = RREG32_SMC(PM_TP);
 981         u32 ni;
 982         u32 xclk = radeon_get_xclk(rdev);
 983         u32 value;
 984
 985         r600_calculate_u_and_p(400, xclk, 16, &p, &u);
 986
 987         ni = (p + tp - 1) / tp;
 988
 989         value = RREG32_SMC(PM_I_CNTL_1);
 990         value &= ~SCLK_DPM_MASK;
 991         value |= SCLK_DPM(ni);
 992         WREG32_SMC(PM_I_CNTL_1, value);
 993 }
 994
 995 static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
 996                                                  int min_temp, int max_temp)
 997 {
 998         int low_temp = 0 * 1000;
 999         int high_temp = 255 * 1000;
1000
1001         if (low_temp < min_temp)
1002                 low_temp = min_temp;
1003         if (high_temp > max_temp)
1004                 high_temp = max_temp;
1005         if (high_temp < low_temp) {
1006                 DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1007                 return -EINVAL;
1008         }
1009
1010         WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1011         WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1012
1013         rdev->pm.dpm.thermal.min_temp = low_temp;
1014         rdev->pm.dpm.thermal.max_temp = high_temp;
1015
1016         return 0;
1017 }
1018
1019 static void trinity_update_current_ps(struct radeon_device *rdev,
1020                                       struct radeon_ps *rps)
1021 {
1022         struct trinity_ps *new_ps = trinity_get_ps(rps);
1023         struct trinity_power_info *pi = trinity_get_pi(rdev);
1024
1025         pi->current_rps = *rps;
1026         pi->current_ps = *new_ps;
1027         pi->current_rps.ps_priv = &pi->current_ps;
1028 }
1029
1030 static void trinity_update_requested_ps(struct radeon_device *rdev,
1031                                         struct radeon_ps *rps)
1032 {
1033         struct trinity_ps *new_ps = trinity_get_ps(rps);
1034         struct trinity_power_info *pi = trinity_get_pi(rdev);
1035
1036         pi->requested_rps = *rps;
1037         pi->requested_ps = *new_ps;
1038         pi->requested_rps.ps_priv = &pi->requested_ps;
1039 }
1040
1041 void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1042 {
1043         struct trinity_power_info *pi = trinity_get_pi(rdev);
1044
1045         if (pi->enable_bapm) {
1046                 trinity_acquire_mutex(rdev);
1047                 trinity_dpm_bapm_enable(rdev, enable);
1048                 trinity_release_mutex(rdev);
1049         }
1050 }
1051
1052 int trinity_dpm_enable(struct radeon_device *rdev)
1053 {
1054         struct trinity_power_info *pi = trinity_get_pi(rdev);
1055
1056         trinity_acquire_mutex(rdev);
1057
1058         if (trinity_dpm_enabled(rdev)) {
1059                 trinity_release_mutex(rdev);
1060                 return -EINVAL;
1061         }
1062
1063         trinity_program_bootup_state(rdev);
1064         sumo_program_vc(rdev, 0x00C00033);
1065         trinity_start_am(rdev);
1066         if (pi->enable_auto_thermal_throttling) {
1067                 trinity_program_ttt(rdev);
1068                 trinity_enable_att(rdev);
1069         }
1070         trinity_program_sclk_dpm(rdev);
1071         trinity_start_dpm(rdev);
1072         trinity_wait_for_dpm_enabled(rdev);
1073         trinity_dpm_bapm_enable(rdev, false);
1074         trinity_release_mutex(rdev);
1075
1076         trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1077
1078         return 0;
1079 }
1080
1081 int trinity_dpm_late_enable(struct radeon_device *rdev)
1082 {
1083         int ret;
1084
1085         trinity_acquire_mutex(rdev);
1086         trinity_enable_clock_power_gating(rdev);
1087
1088         if (rdev->irq.installed &&
1089             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1090                 ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1091                 if (ret) {
1092                         trinity_release_mutex(rdev);
1093                         return ret;
1094                 }
1095                 rdev->irq.dpm_thermal = true;
1096                 radeon_irq_set(rdev);
1097         }
1098         trinity_release_mutex(rdev);
1099
1100         return 0;
1101 }
1102
1103 void trinity_dpm_disable(struct radeon_device *rdev)
1104 {
1105         trinity_acquire_mutex(rdev);
1106         if (!trinity_dpm_enabled(rdev)) {
1107                 trinity_release_mutex(rdev);
1108                 return;
1109         }
1110         trinity_dpm_bapm_enable(rdev, false);
1111         trinity_disable_clock_power_gating(rdev);
1112         sumo_clear_vc(rdev);
1113         trinity_wait_for_level_0(rdev);
1114         trinity_stop_dpm(rdev);
1115         trinity_reset_am(rdev);
1116         trinity_release_mutex(rdev);
1117
1118         if (rdev->irq.installed &&
1119             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1120                 rdev->irq.dpm_thermal = false;
1121                 radeon_irq_set(rdev);
1122         }
1123
1124         trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1125 }
1126
1127 static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1128 {
1129         struct trinity_power_info *pi = trinity_get_pi(rdev);
1130
1131         pi->min_sclk_did =
1132                 (RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1133 }
1134
1135 static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1136                                   struct radeon_ps *rps)
1137 {
1138         struct trinity_power_info *pi = trinity_get_pi(rdev);
1139         struct trinity_ps *new_ps = trinity_get_ps(rps);
1140         u32 nbpsconfig;
1141
1142         if (pi->sys_info.nb_dpm_enable) {
1143                 nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1144                 nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1145                 nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1146                                Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1147                                DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1148                                DpmXNbPsHi(new_ps->DpmXNbPsHi));
1149                 WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1150         }
1151 }
1152
1153 int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1154                                         enum radeon_dpm_forced_level level)
1155 {
1156         struct trinity_power_info *pi = trinity_get_pi(rdev);
1157         struct radeon_ps *rps = &pi->current_rps;
1158         struct trinity_ps *ps = trinity_get_ps(rps);
1159         int i, ret;
1160
1161         if (ps->num_levels <= 1)
1162                 return 0;
1163
1164         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1165                 /* not supported by the hw */
1166                 return -EINVAL;
1167         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1168                 ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1169                 if (ret)
1170                         return ret;
1171         } else {
1172                 for (i = 0; i < ps->num_levels; i++) {
1173                         ret = trinity_dpm_n_levels_disabled(rdev, 0);
1174                         if (ret)
1175                                 return ret;
1176                 }
1177         }
1178
1179         rdev->pm.dpm.forced_level = level;
1180
1181         return 0;
1182 }
1183
1184 int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1185 {
1186         struct trinity_power_info *pi = trinity_get_pi(rdev);
1187         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1188         struct radeon_ps *new_ps = &requested_ps;
1189
1190         trinity_update_requested_ps(rdev, new_ps);
1191
1192         trinity_apply_state_adjust_rules(rdev,
1193                                          &pi->requested_rps,
1194                                          &pi->current_rps);
1195
1196         return 0;
1197 }
1198
1199 int trinity_dpm_set_power_state(struct radeon_device *rdev)
1200 {
1201         struct trinity_power_info *pi = trinity_get_pi(rdev);
1202         struct radeon_ps *new_ps = &pi->requested_rps;
1203         struct radeon_ps *old_ps = &pi->current_rps;
1204
1205         trinity_acquire_mutex(rdev);
1206         if (pi->enable_dpm) {
1207                 if (pi->enable_bapm)
1208                         trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1209                 trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1210                 trinity_enable_power_level_0(rdev);
1211                 trinity_force_level_0(rdev);
1212                 trinity_wait_for_level_0(rdev);
1213                 trinity_setup_nbp_sim(rdev, new_ps);
1214                 trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1215                 trinity_force_level_0(rdev);
1216                 trinity_unforce_levels(rdev);
1217                 trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1218                 trinity_set_vce_clock(rdev, new_ps, old_ps);
1219         }
1220         trinity_release_mutex(rdev);
1221
1222         return 0;
1223 }
1224
1225 void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1226 {
1227         struct trinity_power_info *pi = trinity_get_pi(rdev);
1228         struct radeon_ps *new_ps = &pi->requested_rps;
1229
1230         trinity_update_current_ps(rdev, new_ps);
1231 }
1232
1233 void trinity_dpm_setup_asic(struct radeon_device *rdev)
1234 {
1235         trinity_acquire_mutex(rdev);
1236         sumo_program_sstp(rdev);
1237         sumo_take_smu_control(rdev, true);
1238         trinity_get_min_sclk_divider(rdev);
1239         trinity_release_mutex(rdev);
1240 }
1241
1242 #if 0
1243 void trinity_dpm_reset_asic(struct radeon_device *rdev)
1244 {
1245         struct trinity_power_info *pi = trinity_get_pi(rdev);
1246
1247         trinity_acquire_mutex(rdev);
1248         if (pi->enable_dpm) {
1249                 trinity_enable_power_level_0(rdev);
1250                 trinity_force_level_0(rdev);
1251                 trinity_wait_for_level_0(rdev);
1252                 trinity_program_bootup_state(rdev);
1253                 trinity_force_level_0(rdev);
1254                 trinity_unforce_levels(rdev);
1255         }
1256         trinity_release_mutex(rdev);
1257 }
1258 #endif
1259
1260 static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1261                                                   u32 vid_2bit)
1262 {
1263         struct trinity_power_info *pi = trinity_get_pi(rdev);
1264         u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1265         u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1266         u32 step = (svi_mode == 0) ? 1250 : 625;
1267         u32 delta = vid_7bit * step + 50;
1268
1269         if (delta > 155000)
1270                 return 0;
1271
1272         return (155000 - delta) / 100;
1273 }
1274
1275 static void trinity_patch_boot_state(struct radeon_device *rdev,
1276                                      struct trinity_ps *ps)
1277 {
1278         struct trinity_power_info *pi = trinity_get_pi(rdev);
1279
1280         ps->num_levels = 1;
1281         ps->nbps_flags = 0;
1282         ps->bapm_flags = 0;
1283         ps->levels[0] = pi->boot_pl;
1284 }
1285
1286 static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1287 {
1288         if (sclk < 20000)
1289                 return 1;
1290         return 0;
1291 }
1292
1293 static void trinity_construct_boot_state(struct radeon_device *rdev)
1294 {
1295         struct trinity_power_info *pi = trinity_get_pi(rdev);
1296
1297         pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1298         pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1299         pi->boot_pl.ds_divider_index = 0;
1300         pi->boot_pl.ss_divider_index = 0;
1301         pi->boot_pl.allow_gnb_slow = 1;
1302         pi->boot_pl.force_nbp_state = 0;
1303         pi->boot_pl.display_wm = 0;
1304         pi->boot_pl.vce_wm = 0;
1305         pi->current_ps.num_levels = 1;
1306         pi->current_ps.levels[0] = pi->boot_pl;
1307 }
1308
1309 static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1310                                                   u32 sclk, u32 min_sclk_in_sr)
1311 {
1312         struct trinity_power_info *pi = trinity_get_pi(rdev);
1313         u32 i;
1314         u32 temp;
1315         u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1316                 min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1317
1318         if (sclk < min)
1319                 return 0;
1320
1321         if (!pi->enable_sclk_ds)
1322                 return 0;
1323
1324         for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1325                 temp = sclk / sumo_get_sleep_divider_from_id(i);
1326                 if (temp >= min || i == 0)
1327                         break;
1328         }
1329
1330         return (u8)i;
1331 }
1332
1333 static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1334                                           u32 lower_limit)
1335 {
1336         struct trinity_power_info *pi = trinity_get_pi(rdev);
1337         u32 i;
1338
1339         for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1340                 if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1341                         return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1342         }
1343
1344         if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1345                 DRM_ERROR("engine clock out of range!");
1346
1347         return 0;
1348 }
1349
1350 static void trinity_patch_thermal_state(struct radeon_device *rdev,
1351                                         struct trinity_ps *ps,
1352                                         struct trinity_ps *current_ps)
1353 {
1354         struct trinity_power_info *pi = trinity_get_pi(rdev);
1355         u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1356         u32 current_vddc;
1357         u32 current_sclk;
1358         u32 current_index = 0;
1359
1360         if (current_ps) {
1361                 current_vddc = current_ps->levels[current_index].vddc_index;
1362                 current_sclk = current_ps->levels[current_index].sclk;
1363         } else {
1364                 current_vddc = pi->boot_pl.vddc_index;
1365                 current_sclk = pi->boot_pl.sclk;
1366         }
1367
1368         ps->levels[0].vddc_index = current_vddc;
1369
1370         if (ps->levels[0].sclk > current_sclk)
1371                 ps->levels[0].sclk = current_sclk;
1372
1373         ps->levels[0].ds_divider_index =
1374                 trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1375         ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1376         ps->levels[0].allow_gnb_slow = 1;
1377         ps->levels[0].force_nbp_state = 0;
1378         ps->levels[0].display_wm = 0;
1379         ps->levels[0].vce_wm =
1380                 trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1381 }
1382
1383 static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1384                                        struct trinity_ps *ps, u32 index)
1385 {
1386         if (ps == NULL || ps->num_levels <= 1)
1387                 return 0;
1388         else if (ps->num_levels == 2) {
1389                 if (index == 0)
1390                         return 0;
1391                 else
1392                         return 1;
1393         } else {
1394                 if (index == 0)
1395                         return 0;
1396                 else if (ps->levels[index].sclk < 30000)
1397                         return 0;
1398                 else
1399                         return 1;
1400         }
1401 }
1402
1403 static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1404                                        struct radeon_ps *rps)
1405 {
1406         struct trinity_power_info *pi = trinity_get_pi(rdev);
1407         u32 i = 0;
1408
1409         for (i = 0; i < 4; i++) {
1410                 if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1411                     (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1412                     break;
1413         }
1414
1415         if (i >= 4) {
1416                 DRM_ERROR("UVD clock index not found!\n");
1417                 i = 3;
1418         }
1419         return i;
1420 }
1421
1422 static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1423                                      struct radeon_ps *rps)
1424 {
1425         struct trinity_ps *ps = trinity_get_ps(rps);
1426         struct trinity_power_info *pi = trinity_get_pi(rdev);
1427         u32 high_index = 0;
1428         u32 low_index = 0;
1429
1430         if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1431                 high_index = trinity_get_uvd_clock_index(rdev, rps);
1432
1433                 switch (high_index) {
1434                 case 3:
1435                 case 2:
1436                         low_index = 1;
1437                         break;
1438                 case 1:
1439                 case 0:
1440                 default:
1441                         low_index = 0;
1442                         break;
1443                 }
1444
1445                 ps->vclk_low_divider =
1446                         pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1447                 ps->dclk_low_divider =
1448                         pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1449                 ps->vclk_high_divider =
1450                         pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1451                 ps->dclk_high_divider =
1452                         pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1453         }
1454 }
1455
1456 static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1457                                          u32 evclk, u32 ecclk, u16 *voltage)
1458 {
1459         u32 i;
1460         int ret = -EINVAL;
1461         struct radeon_vce_clock_voltage_dependency_table *table =
1462                 &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1463
1464         if (((evclk == 0) && (ecclk == 0)) ||
1465             (table && (table->count == 0))) {
1466                 *voltage = 0;
1467                 return 0;
1468         }
1469
1470         for (i = 0; i < table->count; i++) {
1471                 if ((evclk <= table->entries[i].evclk) &&
1472                     (ecclk <= table->entries[i].ecclk)) {
1473                         *voltage = table->entries[i].v;
1474                         ret = 0;
1475                         break;
1476                 }
1477         }
1478
1479         /* if no match return the highest voltage */
1480         if (ret)
1481                 *voltage = table->entries[table->count - 1].v;
1482
1483         return ret;
1484 }
1485
1486 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1487                                              struct radeon_ps *new_rps,
1488                                              struct radeon_ps *old_rps)
1489 {
1490         struct trinity_ps *ps = trinity_get_ps(new_rps);
1491         struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1492         struct trinity_power_info *pi = trinity_get_pi(rdev);
1493         u32 min_voltage = 0; /* ??? */
1494         u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1495         u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1496         u32 i;
1497         u16 min_vce_voltage;
1498         bool force_high;
1499         u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1500
1501         if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1502                 return trinity_patch_thermal_state(rdev, ps, current_ps);
1503
1504         trinity_adjust_uvd_state(rdev, new_rps);
1505
1506         if (new_rps->vce_active) {
1507                 new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1508                 new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1509         } else {
1510                 new_rps->evclk = 0;
1511                 new_rps->ecclk = 0;
1512         }
1513
1514         for (i = 0; i < ps->num_levels; i++) {
1515                 if (ps->levels[i].vddc_index < min_voltage)
1516                         ps->levels[i].vddc_index = min_voltage;
1517
1518                 if (ps->levels[i].sclk < min_sclk)
1519                         ps->levels[i].sclk =
1520                                 trinity_get_valid_engine_clock(rdev, min_sclk);
1521
1522                 /* patch in vce limits */
1523                 if (new_rps->vce_active) {
1524                         /* sclk */
1525                         if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1526                                 ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1527                         /* vddc */
1528                         trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1529                         if (ps->levels[i].vddc_index < min_vce_voltage)
1530                                 ps->levels[i].vddc_index = min_vce_voltage;
1531                 }
1532
1533                 ps->levels[i].ds_divider_index =
1534                         sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1535
1536                 ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1537
1538                 ps->levels[i].allow_gnb_slow = 1;
1539                 ps->levels[i].force_nbp_state = 0;
1540                 ps->levels[i].display_wm =
1541                         trinity_calculate_display_wm(rdev, ps, i);
1542                 ps->levels[i].vce_wm =
1543                         trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1544         }
1545
1546         if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1547             ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1548                 ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1549
1550         if (pi->sys_info.nb_dpm_enable) {
1551                 ps->Dpm0PgNbPsLo = 0x1;
1552                 ps->Dpm0PgNbPsHi = 0x0;
1553                 ps->DpmXNbPsLo = 0x2;
1554                 ps->DpmXNbPsHi = 0x1;
1555
1556                 if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1557                     ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1558                         force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1559                                       ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1560                                        (pi->sys_info.uma_channel_number == 1)));
1561                         force_high = (num_active_displays >= 3) || force_high;
1562                         ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1563                         ps->Dpm0PgNbPsHi = 0x1;
1564                         ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1565                         ps->DpmXNbPsHi = 0x2;
1566                         ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1567                 }
1568         }
1569 }
1570
1571 static void trinity_cleanup_asic(struct radeon_device *rdev)
1572 {
1573         sumo_take_smu_control(rdev, false);
1574 }
1575
1576 #if 0
1577 static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1578 {
1579         struct trinity_power_info *pi = trinity_get_pi(rdev);
1580
1581         if (pi->voltage_drop_in_dce)
1582                 trinity_dce_enable_voltage_adjustment(rdev, false);
1583 }
1584 #endif
1585
1586 static void trinity_add_dccac_value(struct radeon_device *rdev)
1587 {
1588         u32 gpu_cac_avrg_cntl_window_size;
1589         u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1590         u64 disp_clk = rdev->clock.default_dispclk / 100;
1591         u32 dc_cac_value;
1592
1593         gpu_cac_avrg_cntl_window_size =
1594                 (RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1595
1596         dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1597                              (32 - gpu_cac_avrg_cntl_window_size));
1598
1599         WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1600 }
1601
1602 void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1603 {
1604         struct trinity_power_info *pi = trinity_get_pi(rdev);
1605
1606         if (pi->voltage_drop_in_dce)
1607                 trinity_dce_enable_voltage_adjustment(rdev, true);
1608         trinity_add_dccac_value(rdev);
1609 }
1610
1611 union power_info {
1612         struct _ATOM_POWERPLAY_INFO info;
1613         struct _ATOM_POWERPLAY_INFO_V2 info_2;
1614         struct _ATOM_POWERPLAY_INFO_V3 info_3;
1615         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1616         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1617         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1618 };
1619
1620 union pplib_clock_info {
1621         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1622         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1623         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1624         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1625 };
1626
1627 union pplib_power_state {
1628         struct _ATOM_PPLIB_STATE v1;
1629         struct _ATOM_PPLIB_STATE_V2 v2;
1630 };
1631
1632 static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1633                                                struct radeon_ps *rps,
1634                                                struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1635                                                u8 table_rev)
1636 {
1637         struct trinity_ps *ps = trinity_get_ps(rps);
1638
1639         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1640         rps->class = le16_to_cpu(non_clock_info->usClassification);
1641         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1642
1643         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1644                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1645                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1646         } else {
1647                 rps->vclk = 0;
1648                 rps->dclk = 0;
1649         }
1650
1651         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1652                 rdev->pm.dpm.boot_ps = rps;
1653                 trinity_patch_boot_state(rdev, ps);
1654         }
1655         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1656                 rdev->pm.dpm.uvd_ps = rps;
1657 }
1658
1659 static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1660                                            struct radeon_ps *rps, int index,
1661                                            union pplib_clock_info *clock_info)
1662 {
1663         struct trinity_power_info *pi = trinity_get_pi(rdev);
1664         struct trinity_ps *ps = trinity_get_ps(rps);
1665         struct trinity_pl *pl = &ps->levels[index];
1666         u32 sclk;
1667
1668         sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1669         sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1670         pl->sclk = sclk;
1671         pl->vddc_index = clock_info->sumo.vddcIndex;
1672
1673         ps->num_levels = index + 1;
1674
1675         if (pi->enable_sclk_ds) {
1676                 pl->ds_divider_index = 5;
1677                 pl->ss_divider_index = 5;
1678         }
1679 }
1680
1681 static int trinity_parse_power_table(struct radeon_device *rdev)
1682 {
1683         struct radeon_mode_info *mode_info = &rdev->mode_info;
1684         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1685         union pplib_power_state *power_state;
1686         int i, j, k, non_clock_array_index, clock_array_index;
1687         union pplib_clock_info *clock_info;
1688         struct _StateArray *state_array;
1689         struct _ClockInfoArray *clock_info_array;
1690         struct _NonClockInfoArray *non_clock_info_array;
1691         union power_info *power_info;
1692         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1693         u16 data_offset;
1694         u8 frev, crev;
1695         u8 *power_state_offset;
1696         struct sumo_ps *ps;
1697
1698         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1699                                    &frev, &crev, &data_offset))
1700                 return -EINVAL;
1701         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1702
1703         state_array = (struct _StateArray *)
1704                 (mode_info->atom_context->bios + data_offset +
1705                  le16_to_cpu(power_info->pplib.usStateArrayOffset));
1706         clock_info_array = (struct _ClockInfoArray *)
1707                 (mode_info->atom_context->bios + data_offset +
1708                  le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1709         non_clock_info_array = (struct _NonClockInfoArray *)
1710                 (mode_info->atom_context->bios + data_offset +
1711                  le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1712
1713         rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1714                                   sizeof(struct radeon_ps),
1715                                   GFP_KERNEL);
1716         if (!rdev->pm.dpm.ps)
1717                 return -ENOMEM;
1718         power_state_offset = (u8 *)state_array->states;
1719         for (i = 0; i < state_array->ucNumEntries; i++) {
1720                 u8 *idx;
1721                 power_state = (union pplib_power_state *)power_state_offset;
1722                 non_clock_array_index = power_state->v2.nonClockInfoIndex;
1723                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1724                         &non_clock_info_array->nonClockInfo[non_clock_array_index];
1725                 if (!rdev->pm.power_state[i].clock_info) {
1726                         kfree(rdev->pm.dpm.ps);
1727                         return -EINVAL;
1728                 }
1729                 ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1730                 if (ps == NULL) {
1731                         kfree(rdev->pm.dpm.ps);
1732                         return -ENOMEM;
1733                 }
1734                 rdev->pm.dpm.ps[i].ps_priv = ps;
1735                 k = 0;
1736                 idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1737                 for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1738                         clock_array_index = idx[j];
1739                         if (clock_array_index >= clock_info_array->ucNumEntries)
1740                                 continue;
1741                         if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1742                                 break;
1743                         clock_info = (union pplib_clock_info *)
1744                                 ((u8 *)&clock_info_array->clockInfo[0] +
1745                                  (clock_array_index * clock_info_array->ucEntrySize));
1746                         trinity_parse_pplib_clock_info(rdev,
1747                                                        &rdev->pm.dpm.ps[i], k,
1748                                                        clock_info);
1749                         k++;
1750                 }
1751                 trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1752                                                    non_clock_info,
1753                                                    non_clock_info_array->ucEntrySize);
1754                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1755         }
1756         rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1757
1758         /* fill in the vce power states */
1759         for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1760                 u32 sclk;
1761                 clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1762                 clock_info = (union pplib_clock_info *)
1763                         &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1764                 sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1765                 sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1766                 rdev->pm.dpm.vce_states[i].sclk = sclk;
1767                 rdev->pm.dpm.vce_states[i].mclk = 0;
1768         }
1769
1770         return 0;
1771 }
1772
1773 union igp_info {
1774         struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1775         struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1776         struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1777         struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1778         struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1779 };
1780
1781 static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1782 {
1783         struct trinity_power_info *pi = trinity_get_pi(rdev);
1784         u32 divider;
1785
1786         if (did >= 8 && did <= 0x3f)
1787                 divider = did * 25;
1788         else if (did > 0x3f && did <= 0x5f)
1789                 divider = (did - 64) * 50 + 1600;
1790         else if (did > 0x5f && did <= 0x7e)
1791                 divider = (did - 96) * 100 + 3200;
1792         else if (did == 0x7f)
1793                 divider = 128 * 100;
1794         else
1795                 return 10000;
1796
1797         return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1798 }
1799
1800 static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1801 {
1802         struct trinity_power_info *pi = trinity_get_pi(rdev);
1803         struct radeon_mode_info *mode_info = &rdev->mode_info;
1804         int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1805         union igp_info *igp_info;
1806         u8 frev, crev;
1807         u16 data_offset;
1808         int i;
1809
1810         if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1811                                    &frev, &crev, &data_offset)) {
1812                 igp_info = (union igp_info *)(mode_info->atom_context->bios +
1813                                               data_offset);
1814
1815                 if (crev != 7) {
1816                         DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1817                         return -EINVAL;
1818                 }
1819                 pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1820                 pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1821                 pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1822                 pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1823                 pi->sys_info.bootup_nb_voltage_index =
1824                         le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1825                 if (igp_info->info_7.ucHtcTmpLmt == 0)
1826                         pi->sys_info.htc_tmp_lmt = 203;
1827                 else
1828                         pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1829                 if (igp_info->info_7.ucHtcHystLmt == 0)
1830                         pi->sys_info.htc_hyst_lmt = 5;
1831                 else
1832                         pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1833                 if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1834                         DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1835                 }
1836
1837                 if (pi->enable_nbps_policy)
1838                         pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1839                 else
1840                         pi->sys_info.nb_dpm_enable = 0;
1841
1842                 for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1843                         pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1844                         pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1845                 }
1846
1847                 pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1848                 pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1849                 pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1850                 pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1851
1852                 if (!pi->sys_info.nb_dpm_enable) {
1853                         for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1854                                 pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1855                                 pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1856                                 pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1857                         }
1858                 }
1859
1860                 pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1861
1862                 sumo_construct_sclk_voltage_mapping_table(rdev,
1863                                                           &pi->sys_info.sclk_voltage_mapping_table,
1864                                                           igp_info->info_7.sAvail_SCLK);
1865                 sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1866                                                  igp_info->info_7.sAvail_SCLK);
1867
1868                 pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1869                         igp_info->info_7.ucDPMState0VclkFid;
1870                 pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1871                         igp_info->info_7.ucDPMState1VclkFid;
1872                 pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1873                         igp_info->info_7.ucDPMState2VclkFid;
1874                 pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1875                         igp_info->info_7.ucDPMState3VclkFid;
1876
1877                 pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1878                         igp_info->info_7.ucDPMState0DclkFid;
1879                 pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1880                         igp_info->info_7.ucDPMState1DclkFid;
1881                 pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1882                         igp_info->info_7.ucDPMState2DclkFid;
1883                 pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1884                         igp_info->info_7.ucDPMState3DclkFid;
1885
1886                 for (i = 0; i < 4; i++) {
1887                         pi->sys_info.uvd_clock_table_entries[i].vclk =
1888                                 trinity_convert_did_to_freq(rdev,
1889                                                             pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1890                         pi->sys_info.uvd_clock_table_entries[i].dclk =
1891                                 trinity_convert_did_to_freq(rdev,
1892                                                             pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1893                 }
1894
1895
1896
1897         }
1898         return 0;
1899 }
1900
1901 int trinity_dpm_init(struct radeon_device *rdev)
1902 {
1903         struct trinity_power_info *pi;
1904         int ret, i;
1905
1906         pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1907         if (pi == NULL)
1908                 return -ENOMEM;
1909         rdev->pm.dpm.priv = pi;
1910
1911         for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1912                 pi->at[i] = TRINITY_AT_DFLT;
1913
1914         if (radeon_bapm == -1) {
1915                 /* There are stability issues reported on with
1916                  * bapm enabled when switching between AC and battery
1917                  * power.  At the same time, some MSI boards hang
1918                  * if it's not enabled and dpm is enabled.  Just enable
1919                  * it for MSI boards right now.
1920                  */
1921                 if (rdev->pdev->subsystem_vendor == 0x1462)
1922                         pi->enable_bapm = true;
1923                 else
1924                         pi->enable_bapm = false;
1925         } else if (radeon_bapm == 0) {
1926                 pi->enable_bapm = false;
1927         } else {
1928                 pi->enable_bapm = true;
1929         }
1930         pi->enable_nbps_policy = true;
1931         pi->enable_sclk_ds = true;
1932         pi->enable_gfx_power_gating = true;
1933         pi->enable_gfx_clock_gating = true;
1934         pi->enable_mg_clock_gating = false;
1935         pi->enable_gfx_dynamic_mgpg = false;
1936         pi->override_dynamic_mgpg = false;
1937         pi->enable_auto_thermal_throttling = true;
1938         pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1939         pi->uvd_dpm = true; /* ??? */
1940
1941         ret = trinity_parse_sys_info_table(rdev);
1942         if (ret)
1943                 return ret;
1944
1945         trinity_construct_boot_state(rdev);
1946
1947         ret = r600_get_platform_caps(rdev);
1948         if (ret)
1949                 return ret;
1950
1951         ret = r600_parse_extended_power_table(rdev);
1952         if (ret)
1953                 return ret;
1954
1955         ret = trinity_parse_power_table(rdev);
1956         if (ret)
1957                 return ret;
1958
1959         pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
1960         pi->enable_dpm = true;
1961
1962         return 0;
1963 }
1964
1965 void trinity_dpm_print_power_state(struct radeon_device *rdev,
1966                                    struct radeon_ps *rps)
1967 {
1968         int i;
1969         struct trinity_ps *ps = trinity_get_ps(rps);
1970
1971         r600_dpm_print_class_info(rps->class, rps->class2);
1972         r600_dpm_print_cap_info(rps->caps);
1973         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1974         for (i = 0; i < ps->num_levels; i++) {
1975                 struct trinity_pl *pl = &ps->levels[i];
1976                 printk("\t\tpower level %d    sclk: %u vddc: %u\n",
1977                        i, pl->sclk,
1978                        trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
1979         }
1980         r600_dpm_print_ps_status(rdev, rps);
1981 }
1982
1983 void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
1984                                                          struct seq_file *m)
1985 {
1986         struct trinity_power_info *pi = trinity_get_pi(rdev);
1987         struct radeon_ps *rps = &pi->current_rps;
1988         struct trinity_ps *ps = trinity_get_ps(rps);
1989         struct trinity_pl *pl;
1990         u32 current_index =
1991                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
1992                 CURRENT_STATE_SHIFT;
1993
1994         if (current_index >= ps->num_levels) {
1995                 seq_printf(m, "invalid dpm profile %d\n", current_index);
1996         } else {
1997                 pl = &ps->levels[current_index];
1998                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1999                 seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2000                            current_index, pl->sclk,
2001                            trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2002         }
2003 }
2004
2005 u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2006 {
2007         struct trinity_power_info *pi = trinity_get_pi(rdev);
2008         struct radeon_ps *rps = &pi->current_rps;
2009         struct trinity_ps *ps = trinity_get_ps(rps);
2010         struct trinity_pl *pl;
2011         u32 current_index =
2012                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2013                 CURRENT_STATE_SHIFT;
2014
2015         if (current_index >= ps->num_levels) {
2016                 return 0;
2017         } else {
2018                 pl = &ps->levels[current_index];
2019                 return pl->sclk;
2020         }
2021 }
2022
2023 u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2024 {
2025         struct trinity_power_info *pi = trinity_get_pi(rdev);
2026
2027         return pi->sys_info.bootup_uma_clk;
2028 }
2029
2030 void trinity_dpm_fini(struct radeon_device *rdev)
2031 {
2032         int i;
2033
2034         trinity_cleanup_asic(rdev); /* ??? */
2035
2036         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2037                 kfree(rdev->pm.dpm.ps[i].ps_priv);
2038         }
2039         kfree(rdev->pm.dpm.ps);
2040         kfree(rdev->pm.dpm.priv);
2041         r600_free_extended_power_table(rdev);
2042 }
2043
2044 u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2045 {
2046         struct trinity_power_info *pi = trinity_get_pi(rdev);
2047         struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2048
2049         if (low)
2050                 return requested_state->levels[0].sclk;
2051         else
2052                 return requested_state->levels[requested_state->num_levels - 1].sclk;
2053 }
2054
2055 u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2056 {
2057         struct trinity_power_info *pi = trinity_get_pi(rdev);
2058
2059         return pi->sys_info.bootup_uma_clk;
2060 }