drivers/gpu/drm/radeon/trinity_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/pci.h>
  25 #include <linux/seq_file.h>
  26
  27 #include "r600_dpm.h"
  28 #include "radeon.h"
  29 #include "radeon_asic.h"
  30 #include "trinity_dpm.h"
  31 #include "trinityd.h"
  32 #include "vce.h"
  33
  34 #define TRINITY_MAX_DEEPSLEEP_DIVIDER_ID 5
  35 #define TRINITY_MINIMUM_ENGINE_CLOCK 800
  36 #define SCLK_MIN_DIV_INTV_SHIFT     12
  37 #define TRINITY_DISPCLK_BYPASS_THRESHOLD 10000
  38
  39 #ifndef TRINITY_MGCG_SEQUENCE
  40 #define TRINITY_MGCG_SEQUENCE  100
  41
  42 static const u32 trinity_mgcg_shls_default[] =
  43 {
  44         /* Register, Value, Mask */
  45         0x0000802c, 0xc0000000, 0xffffffff,
  46         0x00003fc4, 0xc0000000, 0xffffffff,
  47         0x00005448, 0x00000100, 0xffffffff,
  48         0x000055e4, 0x00000100, 0xffffffff,
  49         0x0000160c, 0x00000100, 0xffffffff,
  50         0x00008984, 0x06000100, 0xffffffff,
  51         0x0000c164, 0x00000100, 0xffffffff,
  52         0x00008a18, 0x00000100, 0xffffffff,
  53         0x0000897c, 0x06000100, 0xffffffff,
  54         0x00008b28, 0x00000100, 0xffffffff,
  55         0x00009144, 0x00800200, 0xffffffff,
  56         0x00009a60, 0x00000100, 0xffffffff,
  57         0x00009868, 0x00000100, 0xffffffff,
  58         0x00008d58, 0x00000100, 0xffffffff,
  59         0x00009510, 0x00000100, 0xffffffff,
  60         0x0000949c, 0x00000100, 0xffffffff,
  61         0x00009654, 0x00000100, 0xffffffff,
  62         0x00009030, 0x00000100, 0xffffffff,
  63         0x00009034, 0x00000100, 0xffffffff,
  64         0x00009038, 0x00000100, 0xffffffff,
  65         0x0000903c, 0x00000100, 0xffffffff,
  66         0x00009040, 0x00000100, 0xffffffff,
  67         0x0000a200, 0x00000100, 0xffffffff,
  68         0x0000a204, 0x00000100, 0xffffffff,
  69         0x0000a208, 0x00000100, 0xffffffff,
  70         0x0000a20c, 0x00000100, 0xffffffff,
  71         0x00009744, 0x00000100, 0xffffffff,
  72         0x00003f80, 0x00000100, 0xffffffff,
  73         0x0000a210, 0x00000100, 0xffffffff,
  74         0x0000a214, 0x00000100, 0xffffffff,
  75         0x000004d8, 0x00000100, 0xffffffff,
  76         0x00009664, 0x00000100, 0xffffffff,
  77         0x00009698, 0x00000100, 0xffffffff,
  78         0x000004d4, 0x00000200, 0xffffffff,
  79         0x000004d0, 0x00000000, 0xffffffff,
  80         0x000030cc, 0x00000104, 0xffffffff,
  81         0x0000d0c0, 0x00000100, 0xffffffff,
  82         0x0000d8c0, 0x00000100, 0xffffffff,
  83         0x0000951c, 0x00010000, 0xffffffff,
  84         0x00009160, 0x00030002, 0xffffffff,
  85         0x00009164, 0x00050004, 0xffffffff,
  86         0x00009168, 0x00070006, 0xffffffff,
  87         0x00009178, 0x00070000, 0xffffffff,
  88         0x0000917c, 0x00030002, 0xffffffff,
  89         0x00009180, 0x00050004, 0xffffffff,
  90         0x0000918c, 0x00010006, 0xffffffff,
  91         0x00009190, 0x00090008, 0xffffffff,
  92         0x00009194, 0x00070000, 0xffffffff,
  93         0x00009198, 0x00030002, 0xffffffff,
  94         0x0000919c, 0x00050004, 0xffffffff,
  95         0x000091a8, 0x00010006, 0xffffffff,
  96         0x000091ac, 0x00090008, 0xffffffff,
  97         0x000091b0, 0x00070000, 0xffffffff,
  98         0x000091b4, 0x00030002, 0xffffffff,
  99         0x000091b8, 0x00050004, 0xffffffff,
 100         0x000091c4, 0x00010006, 0xffffffff,
 101         0x000091c8, 0x00090008, 0xffffffff,
 102         0x000091cc, 0x00070000, 0xffffffff,
 103         0x000091d0, 0x00030002, 0xffffffff,
 104         0x000091d4, 0x00050004, 0xffffffff,
 105         0x000091e0, 0x00010006, 0xffffffff,
 106         0x000091e4, 0x00090008, 0xffffffff,
 107         0x000091e8, 0x00000000, 0xffffffff,
 108         0x000091ec, 0x00070000, 0xffffffff,
 109         0x000091f0, 0x00030002, 0xffffffff,
 110         0x000091f4, 0x00050004, 0xffffffff,
 111         0x00009200, 0x00010006, 0xffffffff,
 112         0x00009204, 0x00090008, 0xffffffff,
 113         0x00009208, 0x00070000, 0xffffffff,
 114         0x0000920c, 0x00030002, 0xffffffff,
 115         0x00009210, 0x00050004, 0xffffffff,
 116         0x0000921c, 0x00010006, 0xffffffff,
 117         0x00009220, 0x00090008, 0xffffffff,
 118         0x00009294, 0x00000000, 0xffffffff
 119 };
 120 #endif
 121
 122 #ifndef TRINITY_SYSLS_SEQUENCE
 123 #define TRINITY_SYSLS_SEQUENCE  100
 124
 125 static const u32 trinity_sysls_disable[] =
 126 {
 127         /* Register, Value, Mask */
 128         0x0000d0c0, 0x00000000, 0xffffffff,
 129         0x0000d8c0, 0x00000000, 0xffffffff,
 130         0x000055e8, 0x00000000, 0xffffffff,
 131         0x0000d0bc, 0x00000000, 0xffffffff,
 132         0x0000d8bc, 0x00000000, 0xffffffff,
 133         0x000015c0, 0x00041401, 0xffffffff,
 134         0x0000264c, 0x00040400, 0xffffffff,
 135         0x00002648, 0x00040400, 0xffffffff,
 136         0x00002650, 0x00040400, 0xffffffff,
 137         0x000020b8, 0x00040400, 0xffffffff,
 138         0x000020bc, 0x00040400, 0xffffffff,
 139         0x000020c0, 0x00040c80, 0xffffffff,
 140         0x0000f4a0, 0x000000c0, 0xffffffff,
 141         0x0000f4a4, 0x00680000, 0xffffffff,
 142         0x00002f50, 0x00000404, 0xffffffff,
 143         0x000004c8, 0x00000001, 0xffffffff,
 144         0x0000641c, 0x00007ffd, 0xffffffff,
 145         0x00000c7c, 0x0000ff00, 0xffffffff,
 146         0x00006dfc, 0x0000007f, 0xffffffff
 147 };
 148
 149 static const u32 trinity_sysls_enable[] =
 150 {
 151         /* Register, Value, Mask */
 152         0x000055e8, 0x00000001, 0xffffffff,
 153         0x0000d0bc, 0x00000100, 0xffffffff,
 154         0x0000d8bc, 0x00000100, 0xffffffff,
 155         0x000015c0, 0x000c1401, 0xffffffff,
 156         0x0000264c, 0x000c0400, 0xffffffff,
 157         0x00002648, 0x000c0400, 0xffffffff,
 158         0x00002650, 0x000c0400, 0xffffffff,
 159         0x000020b8, 0x000c0400, 0xffffffff,
 160         0x000020bc, 0x000c0400, 0xffffffff,
 161         0x000020c0, 0x000c0c80, 0xffffffff,
 162         0x0000f4a0, 0x000000c0, 0xffffffff,
 163         0x0000f4a4, 0x00680fff, 0xffffffff,
 164         0x00002f50, 0x00000903, 0xffffffff,
 165         0x000004c8, 0x00000000, 0xffffffff,
 166         0x0000641c, 0x00000000, 0xffffffff,
 167         0x00000c7c, 0x00000000, 0xffffffff,
 168         0x00006dfc, 0x00000000, 0xffffffff
 169 };
 170 #endif
 171
 172 static const u32 trinity_override_mgpg_sequences[] =
 173 {
 174         /* Register, Value */
 175         0x00000200, 0xE030032C,
 176         0x00000204, 0x00000FFF,
 177         0x00000200, 0xE0300058,
 178         0x00000204, 0x00030301,
 179         0x00000200, 0xE0300054,
 180         0x00000204, 0x500010FF,
 181         0x00000200, 0xE0300074,
 182         0x00000204, 0x00030301,
 183         0x00000200, 0xE0300070,
 184         0x00000204, 0x500010FF,
 185         0x00000200, 0xE0300090,
 186         0x00000204, 0x00030301,
 187         0x00000200, 0xE030008C,
 188         0x00000204, 0x500010FF,
 189         0x00000200, 0xE03000AC,
 190         0x00000204, 0x00030301,
 191         0x00000200, 0xE03000A8,
 192         0x00000204, 0x500010FF,
 193         0x00000200, 0xE03000C8,
 194         0x00000204, 0x00030301,
 195         0x00000200, 0xE03000C4,
 196         0x00000204, 0x500010FF,
 197         0x00000200, 0xE03000E4,
 198         0x00000204, 0x00030301,
 199         0x00000200, 0xE03000E0,
 200         0x00000204, 0x500010FF,
 201         0x00000200, 0xE0300100,
 202         0x00000204, 0x00030301,
 203         0x00000200, 0xE03000FC,
 204         0x00000204, 0x500010FF,
 205         0x00000200, 0xE0300058,
 206         0x00000204, 0x00030303,
 207         0x00000200, 0xE0300054,
 208         0x00000204, 0x600010FF,
 209         0x00000200, 0xE0300074,
 210         0x00000204, 0x00030303,
 211         0x00000200, 0xE0300070,
 212         0x00000204, 0x600010FF,
 213         0x00000200, 0xE0300090,
 214         0x00000204, 0x00030303,
 215         0x00000200, 0xE030008C,
 216         0x00000204, 0x600010FF,
 217         0x00000200, 0xE03000AC,
 218         0x00000204, 0x00030303,
 219         0x00000200, 0xE03000A8,
 220         0x00000204, 0x600010FF,
 221         0x00000200, 0xE03000C8,
 222         0x00000204, 0x00030303,
 223         0x00000200, 0xE03000C4,
 224         0x00000204, 0x600010FF,
 225         0x00000200, 0xE03000E4,
 226         0x00000204, 0x00030303,
 227         0x00000200, 0xE03000E0,
 228         0x00000204, 0x600010FF,
 229         0x00000200, 0xE0300100,
 230         0x00000204, 0x00030303,
 231         0x00000200, 0xE03000FC,
 232         0x00000204, 0x600010FF,
 233         0x00000200, 0xE0300058,
 234         0x00000204, 0x00030303,
 235         0x00000200, 0xE0300054,
 236         0x00000204, 0x700010FF,
 237         0x00000200, 0xE0300074,
 238         0x00000204, 0x00030303,
 239         0x00000200, 0xE0300070,
 240         0x00000204, 0x700010FF,
 241         0x00000200, 0xE0300090,
 242         0x00000204, 0x00030303,
 243         0x00000200, 0xE030008C,
 244         0x00000204, 0x700010FF,
 245         0x00000200, 0xE03000AC,
 246         0x00000204, 0x00030303,
 247         0x00000200, 0xE03000A8,
 248         0x00000204, 0x700010FF,
 249         0x00000200, 0xE03000C8,
 250         0x00000204, 0x00030303,
 251         0x00000200, 0xE03000C4,
 252         0x00000204, 0x700010FF,
 253         0x00000200, 0xE03000E4,
 254         0x00000204, 0x00030303,
 255         0x00000200, 0xE03000E0,
 256         0x00000204, 0x700010FF,
 257         0x00000200, 0xE0300100,
 258         0x00000204, 0x00030303,
 259         0x00000200, 0xE03000FC,
 260         0x00000204, 0x700010FF,
 261         0x00000200, 0xE0300058,
 262         0x00000204, 0x00010303,
 263         0x00000200, 0xE0300054,
 264         0x00000204, 0x800010FF,
 265         0x00000200, 0xE0300074,
 266         0x00000204, 0x00010303,
 267         0x00000200, 0xE0300070,
 268         0x00000204, 0x800010FF,
 269         0x00000200, 0xE0300090,
 270         0x00000204, 0x00010303,
 271         0x00000200, 0xE030008C,
 272         0x00000204, 0x800010FF,
 273         0x00000200, 0xE03000AC,
 274         0x00000204, 0x00010303,
 275         0x00000200, 0xE03000A8,
 276         0x00000204, 0x800010FF,
 277         0x00000200, 0xE03000C4,
 278         0x00000204, 0x800010FF,
 279         0x00000200, 0xE03000C8,
 280         0x00000204, 0x00010303,
 281         0x00000200, 0xE03000E4,
 282         0x00000204, 0x00010303,
 283         0x00000200, 0xE03000E0,
 284         0x00000204, 0x800010FF,
 285         0x00000200, 0xE0300100,
 286         0x00000204, 0x00010303,
 287         0x00000200, 0xE03000FC,
 288         0x00000204, 0x800010FF,
 289         0x00000200, 0x0001f198,
 290         0x00000204, 0x0003ffff,
 291         0x00000200, 0x0001f19C,
 292         0x00000204, 0x3fffffff,
 293         0x00000200, 0xE030032C,
 294         0x00000204, 0x00000000,
 295 };
 296
 297 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 298                                                    const u32 *seq, u32 count);
 299 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev);
 300 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
 301                                              struct radeon_ps *new_rps,
 302                                              struct radeon_ps *old_rps);
 303
 304 static struct trinity_ps *trinity_get_ps(struct radeon_ps *rps)
 305 {
 306         struct trinity_ps *ps = rps->ps_priv;
 307
 308         return ps;
 309 }
 310
 311 static struct trinity_power_info *trinity_get_pi(struct radeon_device *rdev)
 312 {
 313         struct trinity_power_info *pi = rdev->pm.dpm.priv;
 314
 315         return pi;
 316 }
 317
 318 static void trinity_gfx_powergating_initialize(struct radeon_device *rdev)
 319 {
 320         struct trinity_power_info *pi = trinity_get_pi(rdev);
 321         u32 p, u;
 322         u32 value;
 323         struct atom_clock_dividers dividers;
 324         u32 xclk = radeon_get_xclk(rdev);
 325         u32 sssd = 1;
 326         int ret;
 327         u32 hw_rev = (RREG32(HW_REV) & ATI_REV_ID_MASK) >> ATI_REV_ID_SHIFT;
 328
 329         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 330                                              25000, false, &dividers);
 331         if (ret)
 332                 return;
 333
 334         value = RREG32_SMC(GFX_POWER_GATING_CNTL);
 335         value &= ~(SSSD_MASK | PDS_DIV_MASK);
 336         if (sssd)
 337                 value |= SSSD(1);
 338         value |= PDS_DIV(dividers.post_div);
 339         WREG32_SMC(GFX_POWER_GATING_CNTL, value);
 340
 341         r600_calculate_u_and_p(500, xclk, 16, &p, &u);
 342
 343         WREG32(CG_PG_CTRL, SP(p) | SU(u));
 344
 345         WREG32_P(CG_GIPOTS, CG_GIPOT(p), ~CG_GIPOT_MASK);
 346
 347         /* XXX double check hw_rev */
 348         if (pi->override_dynamic_mgpg && (hw_rev == 0))
 349                 trinity_override_dynamic_mg_powergating(rdev);
 350
 351 }
 352
 353 #define CGCG_CGTT_LOCAL0_MASK       0xFFFF33FF
 354 #define CGCG_CGTT_LOCAL1_MASK       0xFFFB0FFE
 355 #define CGTS_SM_CTRL_REG_DISABLE    0x00600000
 356 #define CGTS_SM_CTRL_REG_ENABLE     0x96944200
 357
 358 static void trinity_mg_clockgating_enable(struct radeon_device *rdev,
 359                                           bool enable)
 360 {
 361         u32 local0;
 362         u32 local1;
 363
 364         if (enable) {
 365                 local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 366                 local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 367
 368                 WREG32_CG(CG_CGTT_LOCAL_0,
 369                           (0x00380000 & CGCG_CGTT_LOCAL0_MASK) | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 370                 WREG32_CG(CG_CGTT_LOCAL_1,
 371                           (0x0E000000 & CGCG_CGTT_LOCAL1_MASK) | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 372
 373                 WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_ENABLE);
 374         } else {
 375                 WREG32(CGTS_SM_CTRL_REG, CGTS_SM_CTRL_REG_DISABLE);
 376
 377                 local0 = RREG32_CG(CG_CGTT_LOCAL_0);
 378                 local1 = RREG32_CG(CG_CGTT_LOCAL_1);
 379
 380                 WREG32_CG(CG_CGTT_LOCAL_0,
 381                           CGCG_CGTT_LOCAL0_MASK | (local0 & ~CGCG_CGTT_LOCAL0_MASK) );
 382                 WREG32_CG(CG_CGTT_LOCAL_1,
 383                           CGCG_CGTT_LOCAL1_MASK | (local1 & ~CGCG_CGTT_LOCAL1_MASK) );
 384         }
 385 }
 386
 387 static void trinity_mg_clockgating_initialize(struct radeon_device *rdev)
 388 {
 389         u32 count;
 390         const u32 *seq = NULL;
 391
 392         seq = &trinity_mgcg_shls_default[0];
 393         count = sizeof(trinity_mgcg_shls_default) / (3 * sizeof(u32));
 394
 395         trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 396 }
 397
 398 static void trinity_gfx_clockgating_enable(struct radeon_device *rdev,
 399                                            bool enable)
 400 {
 401         if (enable) {
 402                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_GFX_CLK_OFF_EN, ~DYN_GFX_CLK_OFF_EN);
 403         } else {
 404                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
 405                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
 406                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
 407                 RREG32(GB_ADDR_CONFIG);
 408         }
 409 }
 410
 411 static void trinity_program_clk_gating_hw_sequence(struct radeon_device *rdev,
 412                                                    const u32 *seq, u32 count)
 413 {
 414         u32 i, length = count * 3;
 415
 416         for (i = 0; i < length; i += 3)
 417                 WREG32_P(seq[i], seq[i+1], ~seq[i+2]);
 418 }
 419
 420 static void trinity_program_override_mgpg_sequences(struct radeon_device *rdev,
 421                                                     const u32 *seq, u32 count)
 422 {
 423         u32  i, length = count * 2;
 424
 425         for (i = 0; i < length; i += 2)
 426                 WREG32(seq[i], seq[i+1]);
 427
 428 }
 429
 430 static void trinity_override_dynamic_mg_powergating(struct radeon_device *rdev)
 431 {
 432         u32 count;
 433         const u32 *seq = NULL;
 434
 435         seq = &trinity_override_mgpg_sequences[0];
 436         count = sizeof(trinity_override_mgpg_sequences) / (2 * sizeof(u32));
 437
 438         trinity_program_override_mgpg_sequences(rdev, seq, count);
 439 }
 440
 441 static void trinity_ls_clockgating_enable(struct radeon_device *rdev,
 442                                           bool enable)
 443 {
 444         u32 count;
 445         const u32 *seq = NULL;
 446
 447         if (enable) {
 448                 seq = &trinity_sysls_enable[0];
 449                 count = sizeof(trinity_sysls_enable) / (3 * sizeof(u32));
 450         } else {
 451                 seq = &trinity_sysls_disable[0];
 452                 count = sizeof(trinity_sysls_disable) / (3 * sizeof(u32));
 453         }
 454
 455         trinity_program_clk_gating_hw_sequence(rdev, seq, count);
 456 }
 457
 458 static void trinity_gfx_powergating_enable(struct radeon_device *rdev,
 459                                            bool enable)
 460 {
 461         if (enable) {
 462                 if (RREG32_SMC(CC_SMU_TST_EFUSE1_MISC) & RB_BACKEND_DISABLE_MASK)
 463                         WREG32_SMC(SMU_SCRATCH_A, (RREG32_SMC(SMU_SCRATCH_A) | 0x01));
 464
 465                 WREG32_P(SCLK_PWRMGT_CNTL, DYN_PWR_DOWN_EN, ~DYN_PWR_DOWN_EN);
 466         } else {
 467                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_PWR_DOWN_EN);
 468                 RREG32(GB_ADDR_CONFIG);
 469         }
 470 }
 471
 472 static void trinity_gfx_dynamic_mgpg_enable(struct radeon_device *rdev,
 473                                             bool enable)
 474 {
 475         u32 value;
 476
 477         if (enable) {
 478                 value = RREG32_SMC(PM_I_CNTL_1);
 479                 value &= ~DS_PG_CNTL_MASK;
 480                 value |= DS_PG_CNTL(1);
 481                 WREG32_SMC(PM_I_CNTL_1, value);
 482
 483                 value = RREG32_SMC(SMU_S_PG_CNTL);
 484                 value &= ~DS_PG_EN_MASK;
 485                 value |= DS_PG_EN(1);
 486                 WREG32_SMC(SMU_S_PG_CNTL, value);
 487         } else {
 488                 value = RREG32_SMC(SMU_S_PG_CNTL);
 489                 value &= ~DS_PG_EN_MASK;
 490                 WREG32_SMC(SMU_S_PG_CNTL, value);
 491
 492                 value = RREG32_SMC(PM_I_CNTL_1);
 493                 value &= ~DS_PG_CNTL_MASK;
 494                 WREG32_SMC(PM_I_CNTL_1, value);
 495         }
 496
 497         trinity_gfx_dynamic_mgpg_config(rdev);
 498
 499 }
 500
 501 static void trinity_enable_clock_power_gating(struct radeon_device *rdev)
 502 {
 503         struct trinity_power_info *pi = trinity_get_pi(rdev);
 504
 505         if (pi->enable_gfx_clock_gating)
 506                 sumo_gfx_clockgating_initialize(rdev);
 507         if (pi->enable_mg_clock_gating)
 508                 trinity_mg_clockgating_initialize(rdev);
 509         if (pi->enable_gfx_power_gating)
 510                 trinity_gfx_powergating_initialize(rdev);
 511         if (pi->enable_mg_clock_gating) {
 512                 trinity_ls_clockgating_enable(rdev, true);
 513                 trinity_mg_clockgating_enable(rdev, true);
 514         }
 515         if (pi->enable_gfx_clock_gating)
 516                 trinity_gfx_clockgating_enable(rdev, true);
 517         if (pi->enable_gfx_dynamic_mgpg)
 518                 trinity_gfx_dynamic_mgpg_enable(rdev, true);
 519         if (pi->enable_gfx_power_gating)
 520                 trinity_gfx_powergating_enable(rdev, true);
 521 }
 522
 523 static void trinity_disable_clock_power_gating(struct radeon_device *rdev)
 524 {
 525         struct trinity_power_info *pi = trinity_get_pi(rdev);
 526
 527         if (pi->enable_gfx_power_gating)
 528                 trinity_gfx_powergating_enable(rdev, false);
 529         if (pi->enable_gfx_dynamic_mgpg)
 530                 trinity_gfx_dynamic_mgpg_enable(rdev, false);
 531         if (pi->enable_gfx_clock_gating)
 532                 trinity_gfx_clockgating_enable(rdev, false);
 533         if (pi->enable_mg_clock_gating) {
 534                 trinity_mg_clockgating_enable(rdev, false);
 535                 trinity_ls_clockgating_enable(rdev, false);
 536         }
 537 }
 538
 539 static void trinity_set_divider_value(struct radeon_device *rdev,
 540                                       u32 index, u32 sclk)
 541 {
 542         struct atom_clock_dividers  dividers;
 543         int ret;
 544         u32 value;
 545         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 546
 547         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 548                                              sclk, false, &dividers);
 549         if (ret)
 550                 return;
 551
 552         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 553         value &= ~CLK_DIVIDER_MASK;
 554         value |= CLK_DIVIDER(dividers.post_div);
 555         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 556
 557         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
 558                                              sclk/2, false, &dividers);
 559         if (ret)
 560                 return;
 561
 562         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix);
 563         value &= ~PD_SCLK_DIVIDER_MASK;
 564         value |= PD_SCLK_DIVIDER(dividers.post_div);
 565         WREG32_SMC(SMU_SCLK_DPM_STATE_0_PG_CNTL + ix, value);
 566 }
 567
 568 static void trinity_set_ds_dividers(struct radeon_device *rdev,
 569                                     u32 index, u32 divider)
 570 {
 571         u32 value;
 572         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 573
 574         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 575         value &= ~DS_DIV_MASK;
 576         value |= DS_DIV(divider);
 577         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 578 }
 579
 580 static void trinity_set_ss_dividers(struct radeon_device *rdev,
 581                                     u32 index, u32 divider)
 582 {
 583         u32 value;
 584         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 585
 586         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 587         value &= ~DS_SH_DIV_MASK;
 588         value |= DS_SH_DIV(divider);
 589         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 590 }
 591
 592 static void trinity_set_vid(struct radeon_device *rdev, u32 index, u32 vid)
 593 {
 594         struct trinity_power_info *pi = trinity_get_pi(rdev);
 595         u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid);
 596         u32 value;
 597         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 598
 599         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 600         value &= ~VID_MASK;
 601         value |= VID(vid_7bit);
 602         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 603
 604         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 605         value &= ~LVRT_MASK;
 606         value |= LVRT(0);
 607         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 608 }
 609
 610 static void trinity_set_allos_gnb_slow(struct radeon_device *rdev,
 611                                        u32 index, u32 gnb_slow)
 612 {
 613         u32 value;
 614         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 615
 616         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 617         value &= ~GNB_SLOW_MASK;
 618         value |= GNB_SLOW(gnb_slow);
 619         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 620 }
 621
 622 static void trinity_set_force_nbp_state(struct radeon_device *rdev,
 623                                         u32 index, u32 force_nbp_state)
 624 {
 625         u32 value;
 626         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 627
 628         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix);
 629         value &= ~FORCE_NBPS1_MASK;
 630         value |= FORCE_NBPS1(force_nbp_state);
 631         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_3 + ix, value);
 632 }
 633
 634 static void trinity_set_display_wm(struct radeon_device *rdev,
 635                                    u32 index, u32 wm)
 636 {
 637         u32 value;
 638         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 639
 640         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 641         value &= ~DISPLAY_WM_MASK;
 642         value |= DISPLAY_WM(wm);
 643         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 644 }
 645
 646 static void trinity_set_vce_wm(struct radeon_device *rdev,
 647                                u32 index, u32 wm)
 648 {
 649         u32 value;
 650         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 651
 652         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix);
 653         value &= ~VCE_WM_MASK;
 654         value |= VCE_WM(wm);
 655         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_1 + ix, value);
 656 }
 657
 658 static void trinity_set_at(struct radeon_device *rdev,
 659                            u32 index, u32 at)
 660 {
 661         u32 value;
 662         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 663
 664         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix);
 665         value &= ~AT_MASK;
 666         value |= AT(at);
 667         WREG32_SMC(SMU_SCLK_DPM_STATE_0_AT + ix, value);
 668 }
 669
 670 static void trinity_program_power_level(struct radeon_device *rdev,
 671                                         struct trinity_pl *pl, u32 index)
 672 {
 673         struct trinity_power_info *pi = trinity_get_pi(rdev);
 674
 675         if (index >= SUMO_MAX_HARDWARE_POWERLEVELS)
 676                 return;
 677
 678         trinity_set_divider_value(rdev, index, pl->sclk);
 679         trinity_set_vid(rdev, index, pl->vddc_index);
 680         trinity_set_ss_dividers(rdev, index, pl->ss_divider_index);
 681         trinity_set_ds_dividers(rdev, index, pl->ds_divider_index);
 682         trinity_set_allos_gnb_slow(rdev, index, pl->allow_gnb_slow);
 683         trinity_set_force_nbp_state(rdev, index, pl->force_nbp_state);
 684         trinity_set_display_wm(rdev, index, pl->display_wm);
 685         trinity_set_vce_wm(rdev, index, pl->vce_wm);
 686         trinity_set_at(rdev, index, pi->at[index]);
 687 }
 688
 689 static void trinity_power_level_enable_disable(struct radeon_device *rdev,
 690                                                u32 index, bool enable)
 691 {
 692         u32 value;
 693         u32 ix = index * TRINITY_SIZEOF_DPM_STATE_TABLE;
 694
 695         value = RREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix);
 696         value &= ~STATE_VALID_MASK;
 697         if (enable)
 698                 value |= STATE_VALID(1);
 699         WREG32_SMC(SMU_SCLK_DPM_STATE_0_CNTL_0 + ix, value);
 700 }
 701
 702 static bool trinity_dpm_enabled(struct radeon_device *rdev)
 703 {
 704         if (RREG32_SMC(SMU_SCLK_DPM_CNTL) & SCLK_DPM_EN(1))
 705                 return true;
 706         else
 707                 return false;
 708 }
 709
 710 static void trinity_start_dpm(struct radeon_device *rdev)
 711 {
 712         u32 value = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 713
 714         value &= ~(SCLK_DPM_EN_MASK | SCLK_DPM_BOOT_STATE_MASK | VOLTAGE_CHG_EN_MASK);
 715         value |= SCLK_DPM_EN(1) | SCLK_DPM_BOOT_STATE(0) | VOLTAGE_CHG_EN(1);
 716         WREG32_SMC(SMU_SCLK_DPM_CNTL, value);
 717
 718         WREG32_P(GENERAL_PWRMGT, GLOBAL_PWRMGT_EN, ~GLOBAL_PWRMGT_EN);
 719         WREG32_P(CG_CG_VOLTAGE_CNTL, 0, ~EN);
 720
 721         trinity_dpm_config(rdev, true);
 722 }
 723
 724 static void trinity_wait_for_dpm_enabled(struct radeon_device *rdev)
 725 {
 726         int i;
 727
 728         for (i = 0; i < rdev->usec_timeout; i++) {
 729                 if (RREG32(SCLK_PWRMGT_CNTL) & DYNAMIC_PM_EN)
 730                         break;
 731                 udelay(1);
 732         }
 733         for (i = 0; i < rdev->usec_timeout; i++) {
 734                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & TARGET_STATE_MASK) == 0)
 735                         break;
 736                 udelay(1);
 737         }
 738         for (i = 0; i < rdev->usec_timeout; i++) {
 739                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 740                         break;
 741                 udelay(1);
 742         }
 743 }
 744
 745 static void trinity_stop_dpm(struct radeon_device *rdev)
 746 {
 747         u32 sclk_dpm_cntl;
 748
 749         WREG32_P(CG_CG_VOLTAGE_CNTL, EN, ~EN);
 750
 751         sclk_dpm_cntl = RREG32_SMC(SMU_SCLK_DPM_CNTL);
 752         sclk_dpm_cntl &= ~(SCLK_DPM_EN_MASK | VOLTAGE_CHG_EN_MASK);
 753         WREG32_SMC(SMU_SCLK_DPM_CNTL, sclk_dpm_cntl);
 754
 755         trinity_dpm_config(rdev, false);
 756 }
 757
 758 static void trinity_start_am(struct radeon_device *rdev)
 759 {
 760         WREG32_P(SCLK_PWRMGT_CNTL, 0, ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 761 }
 762
 763 static void trinity_reset_am(struct radeon_device *rdev)
 764 {
 765         WREG32_P(SCLK_PWRMGT_CNTL, RESET_SCLK_CNT | RESET_BUSY_CNT,
 766                  ~(RESET_SCLK_CNT | RESET_BUSY_CNT));
 767 }
 768
 769 static void trinity_wait_for_level_0(struct radeon_device *rdev)
 770 {
 771         int i;
 772
 773         for (i = 0; i < rdev->usec_timeout; i++) {
 774                 if ((RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) == 0)
 775                         break;
 776                 udelay(1);
 777         }
 778 }
 779
 780 static void trinity_enable_power_level_0(struct radeon_device *rdev)
 781 {
 782         trinity_power_level_enable_disable(rdev, 0, true);
 783 }
 784
 785 static void trinity_force_level_0(struct radeon_device *rdev)
 786 {
 787         trinity_dpm_force_state(rdev, 0);
 788 }
 789
 790 static void trinity_unforce_levels(struct radeon_device *rdev)
 791 {
 792         trinity_dpm_no_forced_level(rdev);
 793 }
 794
 795 static void trinity_program_power_levels_0_to_n(struct radeon_device *rdev,
 796                                                 struct radeon_ps *new_rps,
 797                                                 struct radeon_ps *old_rps)
 798 {
 799         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 800         struct trinity_ps *old_ps = trinity_get_ps(old_rps);
 801         u32 i;
 802         u32 n_current_state_levels = (old_ps == NULL) ? 1 : old_ps->num_levels;
 803
 804         for (i = 0; i < new_ps->num_levels; i++) {
 805                 trinity_program_power_level(rdev, &new_ps->levels[i], i);
 806                 trinity_power_level_enable_disable(rdev, i, true);
 807         }
 808
 809         for (i = new_ps->num_levels; i < n_current_state_levels; i++)
 810                 trinity_power_level_enable_disable(rdev, i, false);
 811 }
 812
 813 static void trinity_program_bootup_state(struct radeon_device *rdev)
 814 {
 815         struct trinity_power_info *pi = trinity_get_pi(rdev);
 816         u32 i;
 817
 818         trinity_program_power_level(rdev, &pi->boot_pl, 0);
 819         trinity_power_level_enable_disable(rdev, 0, true);
 820
 821         for (i = 1; i < 8; i++)
 822                 trinity_power_level_enable_disable(rdev, i, false);
 823 }
 824
 825 static void trinity_setup_uvd_clock_table(struct radeon_device *rdev,
 826                                           struct radeon_ps *rps)
 827 {
 828         struct trinity_ps *ps = trinity_get_ps(rps);
 829         u32 uvdstates = (ps->vclk_low_divider |
 830                          ps->vclk_high_divider << 8 |
 831                          ps->dclk_low_divider << 16 |
 832                          ps->dclk_high_divider << 24);
 833
 834         WREG32_SMC(SMU_UVD_DPM_STATES, uvdstates);
 835 }
 836
 837 static void trinity_setup_uvd_dpm_interval(struct radeon_device *rdev,
 838                                            u32 interval)
 839 {
 840         u32 p, u;
 841         u32 tp = RREG32_SMC(PM_TP);
 842         u32 val;
 843         u32 xclk = radeon_get_xclk(rdev);
 844
 845         r600_calculate_u_and_p(interval, xclk, 16, &p, &u);
 846
 847         val = (p + tp - 1) / tp;
 848
 849         WREG32_SMC(SMU_UVD_DPM_CNTL, val);
 850 }
 851
 852 static bool trinity_uvd_clocks_zero(struct radeon_ps *rps)
 853 {
 854         if ((rps->vclk == 0) && (rps->dclk == 0))
 855                 return true;
 856         else
 857                 return false;
 858 }
 859
 860 static bool trinity_uvd_clocks_equal(struct radeon_ps *rps1,
 861                                      struct radeon_ps *rps2)
 862 {
 863         struct trinity_ps *ps1 = trinity_get_ps(rps1);
 864         struct trinity_ps *ps2 = trinity_get_ps(rps2);
 865
 866         if ((rps1->vclk == rps2->vclk) &&
 867             (rps1->dclk == rps2->dclk) &&
 868             (ps1->vclk_low_divider == ps2->vclk_low_divider) &&
 869             (ps1->vclk_high_divider == ps2->vclk_high_divider) &&
 870             (ps1->dclk_low_divider == ps2->dclk_low_divider) &&
 871             (ps1->dclk_high_divider == ps2->dclk_high_divider))
 872                 return true;
 873         else
 874                 return false;
 875 }
 876
 877 static void trinity_setup_uvd_clocks(struct radeon_device *rdev,
 878                                      struct radeon_ps *new_rps,
 879                                      struct radeon_ps *old_rps)
 880 {
 881         struct trinity_power_info *pi = trinity_get_pi(rdev);
 882
 883         if (pi->enable_gfx_power_gating) {
 884                 trinity_gfx_powergating_enable(rdev, false);
 885         }
 886
 887         if (pi->uvd_dpm) {
 888                 if (trinity_uvd_clocks_zero(new_rps) &&
 889                     !trinity_uvd_clocks_zero(old_rps)) {
 890                         trinity_setup_uvd_dpm_interval(rdev, 0);
 891                 } else if (!trinity_uvd_clocks_zero(new_rps)) {
 892                         trinity_setup_uvd_clock_table(rdev, new_rps);
 893
 894                         if (trinity_uvd_clocks_zero(old_rps)) {
 895                                 u32 tmp = RREG32(CG_MISC_REG);
 896                                 tmp &= 0xfffffffd;
 897                                 WREG32(CG_MISC_REG, tmp);
 898
 899                                 radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 900
 901                                 trinity_setup_uvd_dpm_interval(rdev, 3000);
 902                         }
 903                 }
 904                 trinity_uvd_dpm_config(rdev);
 905         } else {
 906                 if (trinity_uvd_clocks_zero(new_rps) ||
 907                     trinity_uvd_clocks_equal(new_rps, old_rps))
 908                         return;
 909
 910                 radeon_set_uvd_clocks(rdev, new_rps->vclk, new_rps->dclk);
 911         }
 912
 913         if (pi->enable_gfx_power_gating) {
 914                 trinity_gfx_powergating_enable(rdev, true);
 915         }
 916 }
 917
 918 static void trinity_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
 919                                                        struct radeon_ps *new_rps,
 920                                                        struct radeon_ps *old_rps)
 921 {
 922         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 923         struct trinity_ps *current_ps = trinity_get_ps(new_rps);
 924
 925         if (new_ps->levels[new_ps->num_levels - 1].sclk >=
 926             current_ps->levels[current_ps->num_levels - 1].sclk)
 927                 return;
 928
 929         trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 930 }
 931
 932 static void trinity_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
 933                                                       struct radeon_ps *new_rps,
 934                                                       struct radeon_ps *old_rps)
 935 {
 936         struct trinity_ps *new_ps = trinity_get_ps(new_rps);
 937         struct trinity_ps *current_ps = trinity_get_ps(old_rps);
 938
 939         if (new_ps->levels[new_ps->num_levels - 1].sclk <
 940             current_ps->levels[current_ps->num_levels - 1].sclk)
 941                 return;
 942
 943         trinity_setup_uvd_clocks(rdev, new_rps, old_rps);
 944 }
 945
 946 static void trinity_set_vce_clock(struct radeon_device *rdev,
 947                                   struct radeon_ps *new_rps,
 948                                   struct radeon_ps *old_rps)
 949 {
 950         if ((old_rps->evclk != new_rps->evclk) ||
 951             (old_rps->ecclk != new_rps->ecclk)) {
 952                 /* turn the clocks on when encoding, off otherwise */
 953                 if (new_rps->evclk || new_rps->ecclk)
 954                         vce_v1_0_enable_mgcg(rdev, false);
 955                 else
 956                         vce_v1_0_enable_mgcg(rdev, true);
 957                 radeon_set_vce_clocks(rdev, new_rps->evclk, new_rps->ecclk);
 958         }
 959 }
 960
 961 static void trinity_program_ttt(struct radeon_device *rdev)
 962 {
 963         struct trinity_power_info *pi = trinity_get_pi(rdev);
 964         u32 value = RREG32_SMC(SMU_SCLK_DPM_TTT);
 965
 966         value &= ~(HT_MASK | LT_MASK);
 967         value |= HT((pi->thermal_auto_throttling + 49) * 8);
 968         value |= LT((pi->thermal_auto_throttling + 49 - pi->sys_info.htc_hyst_lmt) * 8);
 969         WREG32_SMC(SMU_SCLK_DPM_TTT, value);
 970 }
 971
 972 static void trinity_enable_att(struct radeon_device *rdev)
 973 {
 974         u32 value = RREG32_SMC(SMU_SCLK_DPM_TT_CNTL);
 975
 976         value &= ~SCLK_TT_EN_MASK;
 977         value |= SCLK_TT_EN(1);
 978         WREG32_SMC(SMU_SCLK_DPM_TT_CNTL, value);
 979 }
 980
 981 static void trinity_program_sclk_dpm(struct radeon_device *rdev)
 982 {
 983         u32 p, u;
 984         u32 tp = RREG32_SMC(PM_TP);
 985         u32 ni;
 986         u32 xclk = radeon_get_xclk(rdev);
 987         u32 value;
 988
 989         r600_calculate_u_and_p(400, xclk, 16, &p, &u);
 990
 991         ni = (p + tp - 1) / tp;
 992
 993         value = RREG32_SMC(PM_I_CNTL_1);
 994         value &= ~SCLK_DPM_MASK;
 995         value |= SCLK_DPM(ni);
 996         WREG32_SMC(PM_I_CNTL_1, value);
 997 }
 998
 999 static int trinity_set_thermal_temperature_range(struct radeon_device *rdev,
1000                                                  int min_temp, int max_temp)
1001 {
1002         int low_temp = 0 * 1000;
1003         int high_temp = 255 * 1000;
1004
1005         if (low_temp < min_temp)
1006                 low_temp = min_temp;
1007         if (high_temp > max_temp)
1008                 high_temp = max_temp;
1009         if (high_temp < low_temp) {
1010                 DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
1011                 return -EINVAL;
1012         }
1013
1014         WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTH(49 + (high_temp / 1000)), ~DIG_THERM_INTH_MASK);
1015         WREG32_P(CG_THERMAL_INT_CTRL, DIG_THERM_INTL(49 + (low_temp / 1000)), ~DIG_THERM_INTL_MASK);
1016
1017         rdev->pm.dpm.thermal.min_temp = low_temp;
1018         rdev->pm.dpm.thermal.max_temp = high_temp;
1019
1020         return 0;
1021 }
1022
1023 static void trinity_update_current_ps(struct radeon_device *rdev,
1024                                       struct radeon_ps *rps)
1025 {
1026         struct trinity_ps *new_ps = trinity_get_ps(rps);
1027         struct trinity_power_info *pi = trinity_get_pi(rdev);
1028
1029         pi->current_rps = *rps;
1030         pi->current_ps = *new_ps;
1031         pi->current_rps.ps_priv = &pi->current_ps;
1032 }
1033
1034 static void trinity_update_requested_ps(struct radeon_device *rdev,
1035                                         struct radeon_ps *rps)
1036 {
1037         struct trinity_ps *new_ps = trinity_get_ps(rps);
1038         struct trinity_power_info *pi = trinity_get_pi(rdev);
1039
1040         pi->requested_rps = *rps;
1041         pi->requested_ps = *new_ps;
1042         pi->requested_rps.ps_priv = &pi->requested_ps;
1043 }
1044
1045 void trinity_dpm_enable_bapm(struct radeon_device *rdev, bool enable)
1046 {
1047         struct trinity_power_info *pi = trinity_get_pi(rdev);
1048
1049         if (pi->enable_bapm) {
1050                 trinity_acquire_mutex(rdev);
1051                 trinity_dpm_bapm_enable(rdev, enable);
1052                 trinity_release_mutex(rdev);
1053         }
1054 }
1055
1056 int trinity_dpm_enable(struct radeon_device *rdev)
1057 {
1058         struct trinity_power_info *pi = trinity_get_pi(rdev);
1059
1060         trinity_acquire_mutex(rdev);
1061
1062         if (trinity_dpm_enabled(rdev)) {
1063                 trinity_release_mutex(rdev);
1064                 return -EINVAL;
1065         }
1066
1067         trinity_program_bootup_state(rdev);
1068         sumo_program_vc(rdev, 0x00C00033);
1069         trinity_start_am(rdev);
1070         if (pi->enable_auto_thermal_throttling) {
1071                 trinity_program_ttt(rdev);
1072                 trinity_enable_att(rdev);
1073         }
1074         trinity_program_sclk_dpm(rdev);
1075         trinity_start_dpm(rdev);
1076         trinity_wait_for_dpm_enabled(rdev);
1077         trinity_dpm_bapm_enable(rdev, false);
1078         trinity_release_mutex(rdev);
1079
1080         trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1081
1082         return 0;
1083 }
1084
1085 int trinity_dpm_late_enable(struct radeon_device *rdev)
1086 {
1087         int ret;
1088
1089         trinity_acquire_mutex(rdev);
1090         trinity_enable_clock_power_gating(rdev);
1091
1092         if (rdev->irq.installed &&
1093             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1094                 ret = trinity_set_thermal_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1095                 if (ret) {
1096                         trinity_release_mutex(rdev);
1097                         return ret;
1098                 }
1099                 rdev->irq.dpm_thermal = true;
1100                 radeon_irq_set(rdev);
1101         }
1102         trinity_release_mutex(rdev);
1103
1104         return 0;
1105 }
1106
1107 void trinity_dpm_disable(struct radeon_device *rdev)
1108 {
1109         trinity_acquire_mutex(rdev);
1110         if (!trinity_dpm_enabled(rdev)) {
1111                 trinity_release_mutex(rdev);
1112                 return;
1113         }
1114         trinity_dpm_bapm_enable(rdev, false);
1115         trinity_disable_clock_power_gating(rdev);
1116         sumo_clear_vc(rdev);
1117         trinity_wait_for_level_0(rdev);
1118         trinity_stop_dpm(rdev);
1119         trinity_reset_am(rdev);
1120         trinity_release_mutex(rdev);
1121
1122         if (rdev->irq.installed &&
1123             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
1124                 rdev->irq.dpm_thermal = false;
1125                 radeon_irq_set(rdev);
1126         }
1127
1128         trinity_update_current_ps(rdev, rdev->pm.dpm.boot_ps);
1129 }
1130
1131 static void trinity_get_min_sclk_divider(struct radeon_device *rdev)
1132 {
1133         struct trinity_power_info *pi = trinity_get_pi(rdev);
1134
1135         pi->min_sclk_did =
1136                 (RREG32_SMC(CC_SMU_MISC_FUSES) & MinSClkDid_MASK) >> MinSClkDid_SHIFT;
1137 }
1138
1139 static void trinity_setup_nbp_sim(struct radeon_device *rdev,
1140                                   struct radeon_ps *rps)
1141 {
1142         struct trinity_power_info *pi = trinity_get_pi(rdev);
1143         struct trinity_ps *new_ps = trinity_get_ps(rps);
1144         u32 nbpsconfig;
1145
1146         if (pi->sys_info.nb_dpm_enable) {
1147                 nbpsconfig = RREG32_SMC(NB_PSTATE_CONFIG);
1148                 nbpsconfig &= ~(Dpm0PgNbPsLo_MASK | Dpm0PgNbPsHi_MASK | DpmXNbPsLo_MASK | DpmXNbPsHi_MASK);
1149                 nbpsconfig |= (Dpm0PgNbPsLo(new_ps->Dpm0PgNbPsLo) |
1150                                Dpm0PgNbPsHi(new_ps->Dpm0PgNbPsHi) |
1151                                DpmXNbPsLo(new_ps->DpmXNbPsLo) |
1152                                DpmXNbPsHi(new_ps->DpmXNbPsHi));
1153                 WREG32_SMC(NB_PSTATE_CONFIG, nbpsconfig);
1154         }
1155 }
1156
1157 int trinity_dpm_force_performance_level(struct radeon_device *rdev,
1158                                         enum radeon_dpm_forced_level level)
1159 {
1160         struct trinity_power_info *pi = trinity_get_pi(rdev);
1161         struct radeon_ps *rps = &pi->current_rps;
1162         struct trinity_ps *ps = trinity_get_ps(rps);
1163         int i, ret;
1164
1165         if (ps->num_levels <= 1)
1166                 return 0;
1167
1168         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1169                 /* not supported by the hw */
1170                 return -EINVAL;
1171         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1172                 ret = trinity_dpm_n_levels_disabled(rdev, ps->num_levels - 1);
1173                 if (ret)
1174                         return ret;
1175         } else {
1176                 for (i = 0; i < ps->num_levels; i++) {
1177                         ret = trinity_dpm_n_levels_disabled(rdev, 0);
1178                         if (ret)
1179                                 return ret;
1180                 }
1181         }
1182
1183         rdev->pm.dpm.forced_level = level;
1184
1185         return 0;
1186 }
1187
1188 int trinity_dpm_pre_set_power_state(struct radeon_device *rdev)
1189 {
1190         struct trinity_power_info *pi = trinity_get_pi(rdev);
1191         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
1192         struct radeon_ps *new_ps = &requested_ps;
1193
1194         trinity_update_requested_ps(rdev, new_ps);
1195
1196         trinity_apply_state_adjust_rules(rdev,
1197                                          &pi->requested_rps,
1198                                          &pi->current_rps);
1199
1200         return 0;
1201 }
1202
1203 int trinity_dpm_set_power_state(struct radeon_device *rdev)
1204 {
1205         struct trinity_power_info *pi = trinity_get_pi(rdev);
1206         struct radeon_ps *new_ps = &pi->requested_rps;
1207         struct radeon_ps *old_ps = &pi->current_rps;
1208
1209         trinity_acquire_mutex(rdev);
1210         if (pi->enable_dpm) {
1211                 if (pi->enable_bapm)
1212                         trinity_dpm_bapm_enable(rdev, rdev->pm.dpm.ac_power);
1213                 trinity_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
1214                 trinity_enable_power_level_0(rdev);
1215                 trinity_force_level_0(rdev);
1216                 trinity_wait_for_level_0(rdev);
1217                 trinity_setup_nbp_sim(rdev, new_ps);
1218                 trinity_program_power_levels_0_to_n(rdev, new_ps, old_ps);
1219                 trinity_force_level_0(rdev);
1220                 trinity_unforce_levels(rdev);
1221                 trinity_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
1222                 trinity_set_vce_clock(rdev, new_ps, old_ps);
1223         }
1224         trinity_release_mutex(rdev);
1225
1226         return 0;
1227 }
1228
1229 void trinity_dpm_post_set_power_state(struct radeon_device *rdev)
1230 {
1231         struct trinity_power_info *pi = trinity_get_pi(rdev);
1232         struct radeon_ps *new_ps = &pi->requested_rps;
1233
1234         trinity_update_current_ps(rdev, new_ps);
1235 }
1236
1237 void trinity_dpm_setup_asic(struct radeon_device *rdev)
1238 {
1239         trinity_acquire_mutex(rdev);
1240         sumo_program_sstp(rdev);
1241         sumo_take_smu_control(rdev, true);
1242         trinity_get_min_sclk_divider(rdev);
1243         trinity_release_mutex(rdev);
1244 }
1245
1246 #if 0
1247 void trinity_dpm_reset_asic(struct radeon_device *rdev)
1248 {
1249         struct trinity_power_info *pi = trinity_get_pi(rdev);
1250
1251         trinity_acquire_mutex(rdev);
1252         if (pi->enable_dpm) {
1253                 trinity_enable_power_level_0(rdev);
1254                 trinity_force_level_0(rdev);
1255                 trinity_wait_for_level_0(rdev);
1256                 trinity_program_bootup_state(rdev);
1257                 trinity_force_level_0(rdev);
1258                 trinity_unforce_levels(rdev);
1259         }
1260         trinity_release_mutex(rdev);
1261 }
1262 #endif
1263
1264 static u16 trinity_convert_voltage_index_to_value(struct radeon_device *rdev,
1265                                                   u32 vid_2bit)
1266 {
1267         struct trinity_power_info *pi = trinity_get_pi(rdev);
1268         u32 vid_7bit = sumo_convert_vid2_to_vid7(rdev, &pi->sys_info.vid_mapping_table, vid_2bit);
1269         u32 svi_mode = (RREG32_SMC(PM_CONFIG) & SVI_Mode) ? 1 : 0;
1270         u32 step = (svi_mode == 0) ? 1250 : 625;
1271         u32 delta = vid_7bit * step + 50;
1272
1273         if (delta > 155000)
1274                 return 0;
1275
1276         return (155000 - delta) / 100;
1277 }
1278
1279 static void trinity_patch_boot_state(struct radeon_device *rdev,
1280                                      struct trinity_ps *ps)
1281 {
1282         struct trinity_power_info *pi = trinity_get_pi(rdev);
1283
1284         ps->num_levels = 1;
1285         ps->nbps_flags = 0;
1286         ps->bapm_flags = 0;
1287         ps->levels[0] = pi->boot_pl;
1288 }
1289
1290 static u8 trinity_calculate_vce_wm(struct radeon_device *rdev, u32 sclk)
1291 {
1292         if (sclk < 20000)
1293                 return 1;
1294         return 0;
1295 }
1296
1297 static void trinity_construct_boot_state(struct radeon_device *rdev)
1298 {
1299         struct trinity_power_info *pi = trinity_get_pi(rdev);
1300
1301         pi->boot_pl.sclk = pi->sys_info.bootup_sclk;
1302         pi->boot_pl.vddc_index = pi->sys_info.bootup_nb_voltage_index;
1303         pi->boot_pl.ds_divider_index = 0;
1304         pi->boot_pl.ss_divider_index = 0;
1305         pi->boot_pl.allow_gnb_slow = 1;
1306         pi->boot_pl.force_nbp_state = 0;
1307         pi->boot_pl.display_wm = 0;
1308         pi->boot_pl.vce_wm = 0;
1309         pi->current_ps.num_levels = 1;
1310         pi->current_ps.levels[0] = pi->boot_pl;
1311 }
1312
1313 static u8 trinity_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
1314                                                   u32 sclk, u32 min_sclk_in_sr)
1315 {
1316         struct trinity_power_info *pi = trinity_get_pi(rdev);
1317         u32 i;
1318         u32 temp;
1319         u32 min = (min_sclk_in_sr > TRINITY_MINIMUM_ENGINE_CLOCK) ?
1320                 min_sclk_in_sr : TRINITY_MINIMUM_ENGINE_CLOCK;
1321
1322         if (sclk < min)
1323                 return 0;
1324
1325         if (!pi->enable_sclk_ds)
1326                 return 0;
1327
1328         for (i = TRINITY_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
1329                 temp = sclk / sumo_get_sleep_divider_from_id(i);
1330                 if (temp >= min || i == 0)
1331                         break;
1332         }
1333
1334         return (u8)i;
1335 }
1336
1337 static u32 trinity_get_valid_engine_clock(struct radeon_device *rdev,
1338                                           u32 lower_limit)
1339 {
1340         struct trinity_power_info *pi = trinity_get_pi(rdev);
1341         u32 i;
1342
1343         for (i = 0; i < pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries; i++) {
1344                 if (pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency >= lower_limit)
1345                         return pi->sys_info.sclk_voltage_mapping_table.entries[i].sclk_frequency;
1346         }
1347
1348         if (i == pi->sys_info.sclk_voltage_mapping_table.num_max_dpm_entries)
1349                 DRM_ERROR("engine clock out of range!");
1350
1351         return 0;
1352 }
1353
1354 static void trinity_patch_thermal_state(struct radeon_device *rdev,
1355                                         struct trinity_ps *ps,
1356                                         struct trinity_ps *current_ps)
1357 {
1358         struct trinity_power_info *pi = trinity_get_pi(rdev);
1359         u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1360         u32 current_vddc;
1361         u32 current_sclk;
1362         u32 current_index = 0;
1363
1364         if (current_ps) {
1365                 current_vddc = current_ps->levels[current_index].vddc_index;
1366                 current_sclk = current_ps->levels[current_index].sclk;
1367         } else {
1368                 current_vddc = pi->boot_pl.vddc_index;
1369                 current_sclk = pi->boot_pl.sclk;
1370         }
1371
1372         ps->levels[0].vddc_index = current_vddc;
1373
1374         if (ps->levels[0].sclk > current_sclk)
1375                 ps->levels[0].sclk = current_sclk;
1376
1377         ps->levels[0].ds_divider_index =
1378                 trinity_get_sleep_divider_id_from_clock(rdev, ps->levels[0].sclk, sclk_in_sr);
1379         ps->levels[0].ss_divider_index = ps->levels[0].ds_divider_index;
1380         ps->levels[0].allow_gnb_slow = 1;
1381         ps->levels[0].force_nbp_state = 0;
1382         ps->levels[0].display_wm = 0;
1383         ps->levels[0].vce_wm =
1384                 trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1385 }
1386
1387 static u8 trinity_calculate_display_wm(struct radeon_device *rdev,
1388                                        struct trinity_ps *ps, u32 index)
1389 {
1390         if (ps == NULL || ps->num_levels <= 1)
1391                 return 0;
1392         else if (ps->num_levels == 2) {
1393                 if (index == 0)
1394                         return 0;
1395                 else
1396                         return 1;
1397         } else {
1398                 if (index == 0)
1399                         return 0;
1400                 else if (ps->levels[index].sclk < 30000)
1401                         return 0;
1402                 else
1403                         return 1;
1404         }
1405 }
1406
1407 static u32 trinity_get_uvd_clock_index(struct radeon_device *rdev,
1408                                        struct radeon_ps *rps)
1409 {
1410         struct trinity_power_info *pi = trinity_get_pi(rdev);
1411         u32 i = 0;
1412
1413         for (i = 0; i < 4; i++) {
1414                 if ((rps->vclk == pi->sys_info.uvd_clock_table_entries[i].vclk) &&
1415                     (rps->dclk == pi->sys_info.uvd_clock_table_entries[i].dclk))
1416                     break;
1417         }
1418
1419         if (i >= 4) {
1420                 DRM_ERROR("UVD clock index not found!\n");
1421                 i = 3;
1422         }
1423         return i;
1424 }
1425
1426 static void trinity_adjust_uvd_state(struct radeon_device *rdev,
1427                                      struct radeon_ps *rps)
1428 {
1429         struct trinity_ps *ps = trinity_get_ps(rps);
1430         struct trinity_power_info *pi = trinity_get_pi(rdev);
1431         u32 high_index = 0;
1432         u32 low_index = 0;
1433
1434         if (pi->uvd_dpm && r600_is_uvd_state(rps->class, rps->class2)) {
1435                 high_index = trinity_get_uvd_clock_index(rdev, rps);
1436
1437                 switch(high_index) {
1438                 case 3:
1439                 case 2:
1440                         low_index = 1;
1441                         break;
1442                 case 1:
1443                 case 0:
1444                 default:
1445                         low_index = 0;
1446                         break;
1447                 }
1448
1449                 ps->vclk_low_divider =
1450                         pi->sys_info.uvd_clock_table_entries[high_index].vclk_did;
1451                 ps->dclk_low_divider =
1452                         pi->sys_info.uvd_clock_table_entries[high_index].dclk_did;
1453                 ps->vclk_high_divider =
1454                         pi->sys_info.uvd_clock_table_entries[low_index].vclk_did;
1455                 ps->dclk_high_divider =
1456                         pi->sys_info.uvd_clock_table_entries[low_index].dclk_did;
1457         }
1458 }
1459
1460 static int trinity_get_vce_clock_voltage(struct radeon_device *rdev,
1461                                          u32 evclk, u32 ecclk, u16 *voltage)
1462 {
1463         u32 i;
1464         int ret = -EINVAL;
1465         struct radeon_vce_clock_voltage_dependency_table *table =
1466                 &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
1467
1468         if (((evclk == 0) && (ecclk == 0)) ||
1469             (table && (table->count == 0))) {
1470                 *voltage = 0;
1471                 return 0;
1472         }
1473
1474         for (i = 0; i < table->count; i++) {
1475                 if ((evclk <= table->entries[i].evclk) &&
1476                     (ecclk <= table->entries[i].ecclk)) {
1477                         *voltage = table->entries[i].v;
1478                         ret = 0;
1479                         break;
1480                 }
1481         }
1482
1483         /* if no match return the highest voltage */
1484         if (ret)
1485                 *voltage = table->entries[table->count - 1].v;
1486
1487         return ret;
1488 }
1489
1490 static void trinity_apply_state_adjust_rules(struct radeon_device *rdev,
1491                                              struct radeon_ps *new_rps,
1492                                              struct radeon_ps *old_rps)
1493 {
1494         struct trinity_ps *ps = trinity_get_ps(new_rps);
1495         struct trinity_ps *current_ps = trinity_get_ps(old_rps);
1496         struct trinity_power_info *pi = trinity_get_pi(rdev);
1497         u32 min_voltage = 0; /* ??? */
1498         u32 min_sclk = pi->sys_info.min_sclk; /* XXX check against disp reqs */
1499         u32 sclk_in_sr = pi->sys_info.min_sclk; /* ??? */
1500         u32 i;
1501         u16 min_vce_voltage;
1502         bool force_high;
1503         u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1504
1505         if (new_rps->class & ATOM_PPLIB_CLASSIFICATION_THERMAL)
1506                 return trinity_patch_thermal_state(rdev, ps, current_ps);
1507
1508         trinity_adjust_uvd_state(rdev, new_rps);
1509
1510         if (new_rps->vce_active) {
1511                 new_rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
1512                 new_rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
1513         } else {
1514                 new_rps->evclk = 0;
1515                 new_rps->ecclk = 0;
1516         }
1517
1518         for (i = 0; i < ps->num_levels; i++) {
1519                 if (ps->levels[i].vddc_index < min_voltage)
1520                         ps->levels[i].vddc_index = min_voltage;
1521
1522                 if (ps->levels[i].sclk < min_sclk)
1523                         ps->levels[i].sclk =
1524                                 trinity_get_valid_engine_clock(rdev, min_sclk);
1525
1526                 /* patch in vce limits */
1527                 if (new_rps->vce_active) {
1528                         /* sclk */
1529                         if (ps->levels[i].sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
1530                                 ps->levels[i].sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
1531                         /* vddc */
1532                         trinity_get_vce_clock_voltage(rdev, new_rps->evclk, new_rps->ecclk, &min_vce_voltage);
1533                         if (ps->levels[i].vddc_index < min_vce_voltage)
1534                                 ps->levels[i].vddc_index = min_vce_voltage;
1535                 }
1536
1537                 ps->levels[i].ds_divider_index =
1538                         sumo_get_sleep_divider_id_from_clock(rdev, ps->levels[i].sclk, sclk_in_sr);
1539
1540                 ps->levels[i].ss_divider_index = ps->levels[i].ds_divider_index;
1541
1542                 ps->levels[i].allow_gnb_slow = 1;
1543                 ps->levels[i].force_nbp_state = 0;
1544                 ps->levels[i].display_wm =
1545                         trinity_calculate_display_wm(rdev, ps, i);
1546                 ps->levels[i].vce_wm =
1547                         trinity_calculate_vce_wm(rdev, ps->levels[0].sclk);
1548         }
1549
1550         if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1551             ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY))
1552                 ps->bapm_flags |= TRINITY_POWERSTATE_FLAGS_BAPM_DISABLE;
1553
1554         if (pi->sys_info.nb_dpm_enable) {
1555                 ps->Dpm0PgNbPsLo = 0x1;
1556                 ps->Dpm0PgNbPsHi = 0x0;
1557                 ps->DpmXNbPsLo = 0x2;
1558                 ps->DpmXNbPsHi = 0x1;
1559
1560                 if ((new_rps->class & (ATOM_PPLIB_CLASSIFICATION_HDSTATE | ATOM_PPLIB_CLASSIFICATION_SDSTATE)) ||
1561                     ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)) {
1562                         force_high = ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_HDSTATE) ||
1563                                       ((new_rps->class & ATOM_PPLIB_CLASSIFICATION_SDSTATE) &&
1564                                        (pi->sys_info.uma_channel_number == 1)));
1565                         force_high = (num_active_displays >= 3) || force_high;
1566                         ps->Dpm0PgNbPsLo = force_high ? 0x2 : 0x3;
1567                         ps->Dpm0PgNbPsHi = 0x1;
1568                         ps->DpmXNbPsLo = force_high ? 0x2 : 0x3;
1569                         ps->DpmXNbPsHi = 0x2;
1570                         ps->levels[ps->num_levels - 1].allow_gnb_slow = 0;
1571                 }
1572         }
1573 }
1574
1575 static void trinity_cleanup_asic(struct radeon_device *rdev)
1576 {
1577         sumo_take_smu_control(rdev, false);
1578 }
1579
1580 #if 0
1581 static void trinity_pre_display_configuration_change(struct radeon_device *rdev)
1582 {
1583         struct trinity_power_info *pi = trinity_get_pi(rdev);
1584
1585         if (pi->voltage_drop_in_dce)
1586                 trinity_dce_enable_voltage_adjustment(rdev, false);
1587 }
1588 #endif
1589
1590 static void trinity_add_dccac_value(struct radeon_device *rdev)
1591 {
1592         u32 gpu_cac_avrg_cntl_window_size;
1593         u32 num_active_displays = rdev->pm.dpm.new_active_crtc_count;
1594         u64 disp_clk = rdev->clock.default_dispclk / 100;
1595         u32 dc_cac_value;
1596
1597         gpu_cac_avrg_cntl_window_size =
1598                 (RREG32_SMC(GPU_CAC_AVRG_CNTL) & WINDOW_SIZE_MASK) >> WINDOW_SIZE_SHIFT;
1599
1600         dc_cac_value = (u32)((14213 * disp_clk * disp_clk * (u64)num_active_displays) >>
1601                              (32 - gpu_cac_avrg_cntl_window_size));
1602
1603         WREG32_SMC(DC_CAC_VALUE, dc_cac_value);
1604 }
1605
1606 void trinity_dpm_display_configuration_changed(struct radeon_device *rdev)
1607 {
1608         struct trinity_power_info *pi = trinity_get_pi(rdev);
1609
1610         if (pi->voltage_drop_in_dce)
1611                 trinity_dce_enable_voltage_adjustment(rdev, true);
1612         trinity_add_dccac_value(rdev);
1613 }
1614
1615 union power_info {
1616         struct _ATOM_POWERPLAY_INFO info;
1617         struct _ATOM_POWERPLAY_INFO_V2 info_2;
1618         struct _ATOM_POWERPLAY_INFO_V3 info_3;
1619         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
1620         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
1621         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
1622 };
1623
1624 union pplib_clock_info {
1625         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
1626         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
1627         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
1628         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
1629 };
1630
1631 union pplib_power_state {
1632         struct _ATOM_PPLIB_STATE v1;
1633         struct _ATOM_PPLIB_STATE_V2 v2;
1634 };
1635
1636 static void trinity_parse_pplib_non_clock_info(struct radeon_device *rdev,
1637                                                struct radeon_ps *rps,
1638                                                struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
1639                                                u8 table_rev)
1640 {
1641         struct trinity_ps *ps = trinity_get_ps(rps);
1642
1643         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
1644         rps->class = le16_to_cpu(non_clock_info->usClassification);
1645         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
1646
1647         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
1648                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
1649                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
1650         } else {
1651                 rps->vclk = 0;
1652                 rps->dclk = 0;
1653         }
1654
1655         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
1656                 rdev->pm.dpm.boot_ps = rps;
1657                 trinity_patch_boot_state(rdev, ps);
1658         }
1659         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
1660                 rdev->pm.dpm.uvd_ps = rps;
1661 }
1662
1663 static void trinity_parse_pplib_clock_info(struct radeon_device *rdev,
1664                                            struct radeon_ps *rps, int index,
1665                                            union pplib_clock_info *clock_info)
1666 {
1667         struct trinity_power_info *pi = trinity_get_pi(rdev);
1668         struct trinity_ps *ps = trinity_get_ps(rps);
1669         struct trinity_pl *pl = &ps->levels[index];
1670         u32 sclk;
1671
1672         sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1673         sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1674         pl->sclk = sclk;
1675         pl->vddc_index = clock_info->sumo.vddcIndex;
1676
1677         ps->num_levels = index + 1;
1678
1679         if (pi->enable_sclk_ds) {
1680                 pl->ds_divider_index = 5;
1681                 pl->ss_divider_index = 5;
1682         }
1683 }
1684
1685 static int trinity_parse_power_table(struct radeon_device *rdev)
1686 {
1687         struct radeon_mode_info *mode_info = &rdev->mode_info;
1688         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
1689         union pplib_power_state *power_state;
1690         int i, j, k, non_clock_array_index, clock_array_index;
1691         union pplib_clock_info *clock_info;
1692         struct _StateArray *state_array;
1693         struct _ClockInfoArray *clock_info_array;
1694         struct _NonClockInfoArray *non_clock_info_array;
1695         union power_info *power_info;
1696         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
1697         u16 data_offset;
1698         u8 frev, crev;
1699         u8 *power_state_offset;
1700         struct sumo_ps *ps;
1701
1702         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
1703                                    &frev, &crev, &data_offset))
1704                 return -EINVAL;
1705         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
1706
1707         state_array = (struct _StateArray *)
1708                 (mode_info->atom_context->bios + data_offset +
1709                  le16_to_cpu(power_info->pplib.usStateArrayOffset));
1710         clock_info_array = (struct _ClockInfoArray *)
1711                 (mode_info->atom_context->bios + data_offset +
1712                  le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
1713         non_clock_info_array = (struct _NonClockInfoArray *)
1714                 (mode_info->atom_context->bios + data_offset +
1715                  le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
1716
1717         rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
1718                                   sizeof(struct radeon_ps),
1719                                   GFP_KERNEL);
1720         if (!rdev->pm.dpm.ps)
1721                 return -ENOMEM;
1722         power_state_offset = (u8 *)state_array->states;
1723         for (i = 0; i < state_array->ucNumEntries; i++) {
1724                 u8 *idx;
1725                 power_state = (union pplib_power_state *)power_state_offset;
1726                 non_clock_array_index = power_state->v2.nonClockInfoIndex;
1727                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
1728                         &non_clock_info_array->nonClockInfo[non_clock_array_index];
1729                 if (!rdev->pm.power_state[i].clock_info)
1730                         return -EINVAL;
1731                 ps = kzalloc(sizeof(struct sumo_ps), GFP_KERNEL);
1732                 if (ps == NULL) {
1733                         kfree(rdev->pm.dpm.ps);
1734                         return -ENOMEM;
1735                 }
1736                 rdev->pm.dpm.ps[i].ps_priv = ps;
1737                 k = 0;
1738                 idx = (u8 *)&power_state->v2.clockInfoIndex[0];
1739                 for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
1740                         clock_array_index = idx[j];
1741                         if (clock_array_index >= clock_info_array->ucNumEntries)
1742                                 continue;
1743                         if (k >= SUMO_MAX_HARDWARE_POWERLEVELS)
1744                                 break;
1745                         clock_info = (union pplib_clock_info *)
1746                                 ((u8 *)&clock_info_array->clockInfo[0] +
1747                                  (clock_array_index * clock_info_array->ucEntrySize));
1748                         trinity_parse_pplib_clock_info(rdev,
1749                                                        &rdev->pm.dpm.ps[i], k,
1750                                                        clock_info);
1751                         k++;
1752                 }
1753                 trinity_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
1754                                                    non_clock_info,
1755                                                    non_clock_info_array->ucEntrySize);
1756                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
1757         }
1758         rdev->pm.dpm.num_ps = state_array->ucNumEntries;
1759
1760         /* fill in the vce power states */
1761         for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
1762                 u32 sclk;
1763                 clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
1764                 clock_info = (union pplib_clock_info *)
1765                         &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
1766                 sclk = le16_to_cpu(clock_info->sumo.usEngineClockLow);
1767                 sclk |= clock_info->sumo.ucEngineClockHigh << 16;
1768                 rdev->pm.dpm.vce_states[i].sclk = sclk;
1769                 rdev->pm.dpm.vce_states[i].mclk = 0;
1770         }
1771
1772         return 0;
1773 }
1774
1775 union igp_info {
1776         struct _ATOM_INTEGRATED_SYSTEM_INFO info;
1777         struct _ATOM_INTEGRATED_SYSTEM_INFO_V2 info_2;
1778         struct _ATOM_INTEGRATED_SYSTEM_INFO_V5 info_5;
1779         struct _ATOM_INTEGRATED_SYSTEM_INFO_V6 info_6;
1780         struct _ATOM_INTEGRATED_SYSTEM_INFO_V1_7 info_7;
1781 };
1782
1783 static u32 trinity_convert_did_to_freq(struct radeon_device *rdev, u8 did)
1784 {
1785         struct trinity_power_info *pi = trinity_get_pi(rdev);
1786         u32 divider;
1787
1788         if (did >= 8 && did <= 0x3f)
1789                 divider = did * 25;
1790         else if (did > 0x3f && did <= 0x5f)
1791                 divider = (did - 64) * 50 + 1600;
1792         else if (did > 0x5f && did <= 0x7e)
1793                 divider = (did - 96) * 100 + 3200;
1794         else if (did == 0x7f)
1795                 divider = 128 * 100;
1796         else
1797                 return 10000;
1798
1799         return ((pi->sys_info.dentist_vco_freq * 100) + (divider - 1)) / divider;
1800 }
1801
1802 static int trinity_parse_sys_info_table(struct radeon_device *rdev)
1803 {
1804         struct trinity_power_info *pi = trinity_get_pi(rdev);
1805         struct radeon_mode_info *mode_info = &rdev->mode_info;
1806         int index = GetIndexIntoMasterTable(DATA, IntegratedSystemInfo);
1807         union igp_info *igp_info;
1808         u8 frev, crev;
1809         u16 data_offset;
1810         int i;
1811
1812         if (atom_parse_data_header(mode_info->atom_context, index, NULL,
1813                                    &frev, &crev, &data_offset)) {
1814                 igp_info = (union igp_info *)(mode_info->atom_context->bios +
1815                                               data_offset);
1816
1817                 if (crev != 7) {
1818                         DRM_ERROR("Unsupported IGP table: %d %d\n", frev, crev);
1819                         return -EINVAL;
1820                 }
1821                 pi->sys_info.bootup_sclk = le32_to_cpu(igp_info->info_7.ulBootUpEngineClock);
1822                 pi->sys_info.min_sclk = le32_to_cpu(igp_info->info_7.ulMinEngineClock);
1823                 pi->sys_info.bootup_uma_clk = le32_to_cpu(igp_info->info_7.ulBootUpUMAClock);
1824                 pi->sys_info.dentist_vco_freq = le32_to_cpu(igp_info->info_7.ulDentistVCOFreq);
1825                 pi->sys_info.bootup_nb_voltage_index =
1826                         le16_to_cpu(igp_info->info_7.usBootUpNBVoltage);
1827                 if (igp_info->info_7.ucHtcTmpLmt == 0)
1828                         pi->sys_info.htc_tmp_lmt = 203;
1829                 else
1830                         pi->sys_info.htc_tmp_lmt = igp_info->info_7.ucHtcTmpLmt;
1831                 if (igp_info->info_7.ucHtcHystLmt == 0)
1832                         pi->sys_info.htc_hyst_lmt = 5;
1833                 else
1834                         pi->sys_info.htc_hyst_lmt = igp_info->info_7.ucHtcHystLmt;
1835                 if (pi->sys_info.htc_tmp_lmt <= pi->sys_info.htc_hyst_lmt) {
1836                         DRM_ERROR("The htcTmpLmt should be larger than htcHystLmt.\n");
1837                 }
1838
1839                 if (pi->enable_nbps_policy)
1840                         pi->sys_info.nb_dpm_enable = igp_info->info_7.ucNBDPMEnable;
1841                 else
1842                         pi->sys_info.nb_dpm_enable = 0;
1843
1844                 for (i = 0; i < TRINITY_NUM_NBPSTATES; i++) {
1845                         pi->sys_info.nbp_mclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateMemclkFreq[i]);
1846                         pi->sys_info.nbp_nclk[i] = le32_to_cpu(igp_info->info_7.ulNbpStateNClkFreq[i]);
1847                 }
1848
1849                 pi->sys_info.nbp_voltage_index[0] = le16_to_cpu(igp_info->info_7.usNBP0Voltage);
1850                 pi->sys_info.nbp_voltage_index[1] = le16_to_cpu(igp_info->info_7.usNBP1Voltage);
1851                 pi->sys_info.nbp_voltage_index[2] = le16_to_cpu(igp_info->info_7.usNBP2Voltage);
1852                 pi->sys_info.nbp_voltage_index[3] = le16_to_cpu(igp_info->info_7.usNBP3Voltage);
1853
1854                 if (!pi->sys_info.nb_dpm_enable) {
1855                         for (i = 1; i < TRINITY_NUM_NBPSTATES; i++) {
1856                                 pi->sys_info.nbp_mclk[i] = pi->sys_info.nbp_mclk[0];
1857                                 pi->sys_info.nbp_nclk[i] = pi->sys_info.nbp_nclk[0];
1858                                 pi->sys_info.nbp_voltage_index[i] = pi->sys_info.nbp_voltage_index[0];
1859                         }
1860                 }
1861
1862                 pi->sys_info.uma_channel_number = igp_info->info_7.ucUMAChannelNumber;
1863
1864                 sumo_construct_sclk_voltage_mapping_table(rdev,
1865                                                           &pi->sys_info.sclk_voltage_mapping_table,
1866                                                           igp_info->info_7.sAvail_SCLK);
1867                 sumo_construct_vid_mapping_table(rdev, &pi->sys_info.vid_mapping_table,
1868                                                  igp_info->info_7.sAvail_SCLK);
1869
1870                 pi->sys_info.uvd_clock_table_entries[0].vclk_did =
1871                         igp_info->info_7.ucDPMState0VclkFid;
1872                 pi->sys_info.uvd_clock_table_entries[1].vclk_did =
1873                         igp_info->info_7.ucDPMState1VclkFid;
1874                 pi->sys_info.uvd_clock_table_entries[2].vclk_did =
1875                         igp_info->info_7.ucDPMState2VclkFid;
1876                 pi->sys_info.uvd_clock_table_entries[3].vclk_did =
1877                         igp_info->info_7.ucDPMState3VclkFid;
1878
1879                 pi->sys_info.uvd_clock_table_entries[0].dclk_did =
1880                         igp_info->info_7.ucDPMState0DclkFid;
1881                 pi->sys_info.uvd_clock_table_entries[1].dclk_did =
1882                         igp_info->info_7.ucDPMState1DclkFid;
1883                 pi->sys_info.uvd_clock_table_entries[2].dclk_did =
1884                         igp_info->info_7.ucDPMState2DclkFid;
1885                 pi->sys_info.uvd_clock_table_entries[3].dclk_did =
1886                         igp_info->info_7.ucDPMState3DclkFid;
1887
1888                 for (i = 0; i < 4; i++) {
1889                         pi->sys_info.uvd_clock_table_entries[i].vclk =
1890                                 trinity_convert_did_to_freq(rdev,
1891                                                             pi->sys_info.uvd_clock_table_entries[i].vclk_did);
1892                         pi->sys_info.uvd_clock_table_entries[i].dclk =
1893                                 trinity_convert_did_to_freq(rdev,
1894                                                             pi->sys_info.uvd_clock_table_entries[i].dclk_did);
1895                 }
1896
1897
1898
1899         }
1900         return 0;
1901 }
1902
1903 int trinity_dpm_init(struct radeon_device *rdev)
1904 {
1905         struct trinity_power_info *pi;
1906         int ret, i;
1907
1908         pi = kzalloc(sizeof(struct trinity_power_info), GFP_KERNEL);
1909         if (pi == NULL)
1910                 return -ENOMEM;
1911         rdev->pm.dpm.priv = pi;
1912
1913         for (i = 0; i < SUMO_MAX_HARDWARE_POWERLEVELS; i++)
1914                 pi->at[i] = TRINITY_AT_DFLT;
1915
1916         if (radeon_bapm == -1) {
1917                 /* There are stability issues reported on with
1918                  * bapm enabled when switching between AC and battery
1919                  * power.  At the same time, some MSI boards hang
1920                  * if it's not enabled and dpm is enabled.  Just enable
1921                  * it for MSI boards right now.
1922                  */
1923                 if (rdev->pdev->subsystem_vendor == 0x1462)
1924                         pi->enable_bapm = true;
1925                 else
1926                         pi->enable_bapm = false;
1927         } else if (radeon_bapm == 0) {
1928                 pi->enable_bapm = false;
1929         } else {
1930                 pi->enable_bapm = true;
1931         }
1932         pi->enable_nbps_policy = true;
1933         pi->enable_sclk_ds = true;
1934         pi->enable_gfx_power_gating = true;
1935         pi->enable_gfx_clock_gating = true;
1936         pi->enable_mg_clock_gating = false;
1937         pi->enable_gfx_dynamic_mgpg = false;
1938         pi->override_dynamic_mgpg = false;
1939         pi->enable_auto_thermal_throttling = true;
1940         pi->voltage_drop_in_dce = false; /* need to restructure dpm/modeset interaction */
1941         pi->uvd_dpm = true; /* ??? */
1942
1943         ret = trinity_parse_sys_info_table(rdev);
1944         if (ret)
1945                 return ret;
1946
1947         trinity_construct_boot_state(rdev);
1948
1949         ret = r600_get_platform_caps(rdev);
1950         if (ret)
1951                 return ret;
1952
1953         ret = r600_parse_extended_power_table(rdev);
1954         if (ret)
1955                 return ret;
1956
1957         ret = trinity_parse_power_table(rdev);
1958         if (ret)
1959                 return ret;
1960
1961         pi->thermal_auto_throttling = pi->sys_info.htc_tmp_lmt;
1962         pi->enable_dpm = true;
1963
1964         return 0;
1965 }
1966
1967 void trinity_dpm_print_power_state(struct radeon_device *rdev,
1968                                    struct radeon_ps *rps)
1969 {
1970         int i;
1971         struct trinity_ps *ps = trinity_get_ps(rps);
1972
1973         r600_dpm_print_class_info(rps->class, rps->class2);
1974         r600_dpm_print_cap_info(rps->caps);
1975         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
1976         for (i = 0; i < ps->num_levels; i++) {
1977                 struct trinity_pl *pl = &ps->levels[i];
1978                 printk("\t\tpower level %d    sclk: %u vddc: %u\n",
1979                        i, pl->sclk,
1980                        trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
1981         }
1982         r600_dpm_print_ps_status(rdev, rps);
1983 }
1984
1985 void trinity_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
1986                                                          struct seq_file *m)
1987 {
1988         struct trinity_power_info *pi = trinity_get_pi(rdev);
1989         struct radeon_ps *rps = &pi->current_rps;
1990         struct trinity_ps *ps = trinity_get_ps(rps);
1991         struct trinity_pl *pl;
1992         u32 current_index =
1993                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
1994                 CURRENT_STATE_SHIFT;
1995
1996         if (current_index >= ps->num_levels) {
1997                 seq_printf(m, "invalid dpm profile %d\n", current_index);
1998         } else {
1999                 pl = &ps->levels[current_index];
2000                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
2001                 seq_printf(m, "power level %d    sclk: %u vddc: %u\n",
2002                            current_index, pl->sclk,
2003                            trinity_convert_voltage_index_to_value(rdev, pl->vddc_index));
2004         }
2005 }
2006
2007 u32 trinity_dpm_get_current_sclk(struct radeon_device *rdev)
2008 {
2009         struct trinity_power_info *pi = trinity_get_pi(rdev);
2010         struct radeon_ps *rps = &pi->current_rps;
2011         struct trinity_ps *ps = trinity_get_ps(rps);
2012         struct trinity_pl *pl;
2013         u32 current_index =
2014                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_MASK) >>
2015                 CURRENT_STATE_SHIFT;
2016
2017         if (current_index >= ps->num_levels) {
2018                 return 0;
2019         } else {
2020                 pl = &ps->levels[current_index];
2021                 return pl->sclk;
2022         }
2023 }
2024
2025 u32 trinity_dpm_get_current_mclk(struct radeon_device *rdev)
2026 {
2027         struct trinity_power_info *pi = trinity_get_pi(rdev);
2028
2029         return pi->sys_info.bootup_uma_clk;
2030 }
2031
2032 void trinity_dpm_fini(struct radeon_device *rdev)
2033 {
2034         int i;
2035
2036         trinity_cleanup_asic(rdev); /* ??? */
2037
2038         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
2039                 kfree(rdev->pm.dpm.ps[i].ps_priv);
2040         }
2041         kfree(rdev->pm.dpm.ps);
2042         kfree(rdev->pm.dpm.priv);
2043         r600_free_extended_power_table(rdev);
2044 }
2045
2046 u32 trinity_dpm_get_sclk(struct radeon_device *rdev, bool low)
2047 {
2048         struct trinity_power_info *pi = trinity_get_pi(rdev);
2049         struct trinity_ps *requested_state = trinity_get_ps(&pi->requested_rps);
2050
2051         if (low)
2052                 return requested_state->levels[0].sclk;
2053         else
2054                 return requested_state->levels[requested_state->num_levels - 1].sclk;
2055 }
2056
2057 u32 trinity_dpm_get_mclk(struct radeon_device *rdev, bool low)
2058 {
2059         struct trinity_power_info *pi = trinity_get_pi(rdev);
2060
2061         return pi->sys_info.bootup_uma_clk;
2062 }