drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/math64.h>
  25 #include <linux/seq_file.h>
  26
  27 #include <drm/drm_pci.h>
  28
  29 #include "atom.h"
  30 #include "ni_dpm.h"
  31 #include "nid.h"
  32 #include "r600_dpm.h"
  33 #include "radeon.h"
  34 #include "radeon_asic.h"
  35
  36 #define MC_CG_ARB_FREQ_F0           0x0a
  37 #define MC_CG_ARB_FREQ_F1           0x0b
  38 #define MC_CG_ARB_FREQ_F2           0x0c
  39 #define MC_CG_ARB_FREQ_F3           0x0d
  40
  41 #define SMC_RAM_END 0xC000
  42
  43 static const struct ni_cac_weights cac_weights_cayman_xt =
  44 {
  45         0x15,
  46         0x2,
  47         0x19,
  48         0x2,
  49         0x8,
  50         0x14,
  51         0x2,
  52         0x16,
  53         0xE,
  54         0x17,
  55         0x13,
  56         0x2B,
  57         0x10,
  58         0x7,
  59         0x5,
  60         0x5,
  61         0x5,
  62         0x2,
  63         0x3,
  64         0x9,
  65         0x10,
  66         0x10,
  67         0x2B,
  68         0xA,
  69         0x9,
  70         0x4,
  71         0xD,
  72         0xD,
  73         0x3E,
  74         0x18,
  75         0x14,
  76         0,
  77         0x3,
  78         0x3,
  79         0x5,
  80         0,
  81         0x2,
  82         0,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0,
  89         0,
  90         0,
  91         0x1CC,
  92         0,
  93         0x164,
  94         1,
  95         1,
  96         1,
  97         1,
  98         12,
  99         12,
 100         12,
 101         0x12,
 102         0x1F,
 103         132,
 104         5,
 105         7,
 106         0,
 107         { 0, 0, 0, 0, 0, 0, 0, 0 },
 108         { 0, 0, 0, 0 },
 109         true
 110 };
 111
 112 static const struct ni_cac_weights cac_weights_cayman_pro =
 113 {
 114         0x16,
 115         0x4,
 116         0x10,
 117         0x2,
 118         0xA,
 119         0x16,
 120         0x2,
 121         0x18,
 122         0x10,
 123         0x1A,
 124         0x16,
 125         0x2D,
 126         0x12,
 127         0xA,
 128         0x6,
 129         0x6,
 130         0x6,
 131         0x2,
 132         0x4,
 133         0xB,
 134         0x11,
 135         0x11,
 136         0x2D,
 137         0xC,
 138         0xC,
 139         0x7,
 140         0x10,
 141         0x10,
 142         0x3F,
 143         0x1A,
 144         0x16,
 145         0,
 146         0x7,
 147         0x4,
 148         0x6,
 149         1,
 150         0x2,
 151         0x1,
 152         0,
 153         0,
 154         0,
 155         0,
 156         0,
 157         0,
 158         0x30,
 159         0,
 160         0x1CF,
 161         0,
 162         0x166,
 163         1,
 164         1,
 165         1,
 166         1,
 167         12,
 168         12,
 169         12,
 170         0x15,
 171         0x1F,
 172         132,
 173         6,
 174         6,
 175         0,
 176         { 0, 0, 0, 0, 0, 0, 0, 0 },
 177         { 0, 0, 0, 0 },
 178         true
 179 };
 180
 181 static const struct ni_cac_weights cac_weights_cayman_le =
 182 {
 183         0x7,
 184         0xE,
 185         0x1,
 186         0xA,
 187         0x1,
 188         0x3F,
 189         0x2,
 190         0x18,
 191         0x10,
 192         0x1A,
 193         0x1,
 194         0x3F,
 195         0x1,
 196         0xE,
 197         0x6,
 198         0x6,
 199         0x6,
 200         0x2,
 201         0x4,
 202         0x9,
 203         0x1A,
 204         0x1A,
 205         0x2C,
 206         0xA,
 207         0x11,
 208         0x8,
 209         0x19,
 210         0x19,
 211         0x1,
 212         0x1,
 213         0x1A,
 214         0,
 215         0x8,
 216         0x5,
 217         0x8,
 218         0x1,
 219         0x3,
 220         0x1,
 221         0,
 222         0,
 223         0,
 224         0,
 225         0,
 226         0,
 227         0x38,
 228         0x38,
 229         0x239,
 230         0x3,
 231         0x18A,
 232         1,
 233         1,
 234         1,
 235         1,
 236         12,
 237         12,
 238         12,
 239         0x15,
 240         0x22,
 241         132,
 242         6,
 243         6,
 244         0,
 245         { 0, 0, 0, 0, 0, 0, 0, 0 },
 246         { 0, 0, 0, 0 },
 247         true
 248 };
 249
 250 #define NISLANDS_MGCG_SEQUENCE  300
 251
 252 static const u32 cayman_cgcg_cgls_default[] =
 253 {
 254         0x000008f8, 0x00000010, 0xffffffff,
 255         0x000008fc, 0x00000000, 0xffffffff,
 256         0x000008f8, 0x00000011, 0xffffffff,
 257         0x000008fc, 0x00000000, 0xffffffff,
 258         0x000008f8, 0x00000012, 0xffffffff,
 259         0x000008fc, 0x00000000, 0xffffffff,
 260         0x000008f8, 0x00000013, 0xffffffff,
 261         0x000008fc, 0x00000000, 0xffffffff,
 262         0x000008f8, 0x00000014, 0xffffffff,
 263         0x000008fc, 0x00000000, 0xffffffff,
 264         0x000008f8, 0x00000015, 0xffffffff,
 265         0x000008fc, 0x00000000, 0xffffffff,
 266         0x000008f8, 0x00000016, 0xffffffff,
 267         0x000008fc, 0x00000000, 0xffffffff,
 268         0x000008f8, 0x00000017, 0xffffffff,
 269         0x000008fc, 0x00000000, 0xffffffff,
 270         0x000008f8, 0x00000018, 0xffffffff,
 271         0x000008fc, 0x00000000, 0xffffffff,
 272         0x000008f8, 0x00000019, 0xffffffff,
 273         0x000008fc, 0x00000000, 0xffffffff,
 274         0x000008f8, 0x0000001a, 0xffffffff,
 275         0x000008fc, 0x00000000, 0xffffffff,
 276         0x000008f8, 0x0000001b, 0xffffffff,
 277         0x000008fc, 0x00000000, 0xffffffff,
 278         0x000008f8, 0x00000020, 0xffffffff,
 279         0x000008fc, 0x00000000, 0xffffffff,
 280         0x000008f8, 0x00000021, 0xffffffff,
 281         0x000008fc, 0x00000000, 0xffffffff,
 282         0x000008f8, 0x00000022, 0xffffffff,
 283         0x000008fc, 0x00000000, 0xffffffff,
 284         0x000008f8, 0x00000023, 0xffffffff,
 285         0x000008fc, 0x00000000, 0xffffffff,
 286         0x000008f8, 0x00000024, 0xffffffff,
 287         0x000008fc, 0x00000000, 0xffffffff,
 288         0x000008f8, 0x00000025, 0xffffffff,
 289         0x000008fc, 0x00000000, 0xffffffff,
 290         0x000008f8, 0x00000026, 0xffffffff,
 291         0x000008fc, 0x00000000, 0xffffffff,
 292         0x000008f8, 0x00000027, 0xffffffff,
 293         0x000008fc, 0x00000000, 0xffffffff,
 294         0x000008f8, 0x00000028, 0xffffffff,
 295         0x000008fc, 0x00000000, 0xffffffff,
 296         0x000008f8, 0x00000029, 0xffffffff,
 297         0x000008fc, 0x00000000, 0xffffffff,
 298         0x000008f8, 0x0000002a, 0xffffffff,
 299         0x000008fc, 0x00000000, 0xffffffff,
 300         0x000008f8, 0x0000002b, 0xffffffff,
 301         0x000008fc, 0x00000000, 0xffffffff
 302 };
 303 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 304
 305 static const u32 cayman_cgcg_cgls_disable[] =
 306 {
 307         0x000008f8, 0x00000010, 0xffffffff,
 308         0x000008fc, 0xffffffff, 0xffffffff,
 309         0x000008f8, 0x00000011, 0xffffffff,
 310         0x000008fc, 0xffffffff, 0xffffffff,
 311         0x000008f8, 0x00000012, 0xffffffff,
 312         0x000008fc, 0xffffffff, 0xffffffff,
 313         0x000008f8, 0x00000013, 0xffffffff,
 314         0x000008fc, 0xffffffff, 0xffffffff,
 315         0x000008f8, 0x00000014, 0xffffffff,
 316         0x000008fc, 0xffffffff, 0xffffffff,
 317         0x000008f8, 0x00000015, 0xffffffff,
 318         0x000008fc, 0xffffffff, 0xffffffff,
 319         0x000008f8, 0x00000016, 0xffffffff,
 320         0x000008fc, 0xffffffff, 0xffffffff,
 321         0x000008f8, 0x00000017, 0xffffffff,
 322         0x000008fc, 0xffffffff, 0xffffffff,
 323         0x000008f8, 0x00000018, 0xffffffff,
 324         0x000008fc, 0xffffffff, 0xffffffff,
 325         0x000008f8, 0x00000019, 0xffffffff,
 326         0x000008fc, 0xffffffff, 0xffffffff,
 327         0x000008f8, 0x0000001a, 0xffffffff,
 328         0x000008fc, 0xffffffff, 0xffffffff,
 329         0x000008f8, 0x0000001b, 0xffffffff,
 330         0x000008fc, 0xffffffff, 0xffffffff,
 331         0x000008f8, 0x00000020, 0xffffffff,
 332         0x000008fc, 0x00000000, 0xffffffff,
 333         0x000008f8, 0x00000021, 0xffffffff,
 334         0x000008fc, 0x00000000, 0xffffffff,
 335         0x000008f8, 0x00000022, 0xffffffff,
 336         0x000008fc, 0x00000000, 0xffffffff,
 337         0x000008f8, 0x00000023, 0xffffffff,
 338         0x000008fc, 0x00000000, 0xffffffff,
 339         0x000008f8, 0x00000024, 0xffffffff,
 340         0x000008fc, 0x00000000, 0xffffffff,
 341         0x000008f8, 0x00000025, 0xffffffff,
 342         0x000008fc, 0x00000000, 0xffffffff,
 343         0x000008f8, 0x00000026, 0xffffffff,
 344         0x000008fc, 0x00000000, 0xffffffff,
 345         0x000008f8, 0x00000027, 0xffffffff,
 346         0x000008fc, 0x00000000, 0xffffffff,
 347         0x000008f8, 0x00000028, 0xffffffff,
 348         0x000008fc, 0x00000000, 0xffffffff,
 349         0x000008f8, 0x00000029, 0xffffffff,
 350         0x000008fc, 0x00000000, 0xffffffff,
 351         0x000008f8, 0x0000002a, 0xffffffff,
 352         0x000008fc, 0x00000000, 0xffffffff,
 353         0x000008f8, 0x0000002b, 0xffffffff,
 354         0x000008fc, 0x00000000, 0xffffffff,
 355         0x00000644, 0x000f7902, 0x001f4180,
 356         0x00000644, 0x000f3802, 0x001f4180
 357 };
 358 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 359
 360 static const u32 cayman_cgcg_cgls_enable[] =
 361 {
 362         0x00000644, 0x000f7882, 0x001f4080,
 363         0x000008f8, 0x00000010, 0xffffffff,
 364         0x000008fc, 0x00000000, 0xffffffff,
 365         0x000008f8, 0x00000011, 0xffffffff,
 366         0x000008fc, 0x00000000, 0xffffffff,
 367         0x000008f8, 0x00000012, 0xffffffff,
 368         0x000008fc, 0x00000000, 0xffffffff,
 369         0x000008f8, 0x00000013, 0xffffffff,
 370         0x000008fc, 0x00000000, 0xffffffff,
 371         0x000008f8, 0x00000014, 0xffffffff,
 372         0x000008fc, 0x00000000, 0xffffffff,
 373         0x000008f8, 0x00000015, 0xffffffff,
 374         0x000008fc, 0x00000000, 0xffffffff,
 375         0x000008f8, 0x00000016, 0xffffffff,
 376         0x000008fc, 0x00000000, 0xffffffff,
 377         0x000008f8, 0x00000017, 0xffffffff,
 378         0x000008fc, 0x00000000, 0xffffffff,
 379         0x000008f8, 0x00000018, 0xffffffff,
 380         0x000008fc, 0x00000000, 0xffffffff,
 381         0x000008f8, 0x00000019, 0xffffffff,
 382         0x000008fc, 0x00000000, 0xffffffff,
 383         0x000008f8, 0x0000001a, 0xffffffff,
 384         0x000008fc, 0x00000000, 0xffffffff,
 385         0x000008f8, 0x0000001b, 0xffffffff,
 386         0x000008fc, 0x00000000, 0xffffffff,
 387         0x000008f8, 0x00000020, 0xffffffff,
 388         0x000008fc, 0xffffffff, 0xffffffff,
 389         0x000008f8, 0x00000021, 0xffffffff,
 390         0x000008fc, 0xffffffff, 0xffffffff,
 391         0x000008f8, 0x00000022, 0xffffffff,
 392         0x000008fc, 0xffffffff, 0xffffffff,
 393         0x000008f8, 0x00000023, 0xffffffff,
 394         0x000008fc, 0xffffffff, 0xffffffff,
 395         0x000008f8, 0x00000024, 0xffffffff,
 396         0x000008fc, 0xffffffff, 0xffffffff,
 397         0x000008f8, 0x00000025, 0xffffffff,
 398         0x000008fc, 0xffffffff, 0xffffffff,
 399         0x000008f8, 0x00000026, 0xffffffff,
 400         0x000008fc, 0xffffffff, 0xffffffff,
 401         0x000008f8, 0x00000027, 0xffffffff,
 402         0x000008fc, 0xffffffff, 0xffffffff,
 403         0x000008f8, 0x00000028, 0xffffffff,
 404         0x000008fc, 0xffffffff, 0xffffffff,
 405         0x000008f8, 0x00000029, 0xffffffff,
 406         0x000008fc, 0xffffffff, 0xffffffff,
 407         0x000008f8, 0x0000002a, 0xffffffff,
 408         0x000008fc, 0xffffffff, 0xffffffff,
 409         0x000008f8, 0x0000002b, 0xffffffff,
 410         0x000008fc, 0xffffffff, 0xffffffff
 411 };
 412 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 413
 414 static const u32 cayman_mgcg_default[] =
 415 {
 416         0x0000802c, 0xc0000000, 0xffffffff,
 417         0x00003fc4, 0xc0000000, 0xffffffff,
 418         0x00005448, 0x00000100, 0xffffffff,
 419         0x000055e4, 0x00000100, 0xffffffff,
 420         0x0000160c, 0x00000100, 0xffffffff,
 421         0x00008984, 0x06000100, 0xffffffff,
 422         0x0000c164, 0x00000100, 0xffffffff,
 423         0x00008a18, 0x00000100, 0xffffffff,
 424         0x0000897c, 0x06000100, 0xffffffff,
 425         0x00008b28, 0x00000100, 0xffffffff,
 426         0x00009144, 0x00800200, 0xffffffff,
 427         0x00009a60, 0x00000100, 0xffffffff,
 428         0x00009868, 0x00000100, 0xffffffff,
 429         0x00008d58, 0x00000100, 0xffffffff,
 430         0x00009510, 0x00000100, 0xffffffff,
 431         0x0000949c, 0x00000100, 0xffffffff,
 432         0x00009654, 0x00000100, 0xffffffff,
 433         0x00009030, 0x00000100, 0xffffffff,
 434         0x00009034, 0x00000100, 0xffffffff,
 435         0x00009038, 0x00000100, 0xffffffff,
 436         0x0000903c, 0x00000100, 0xffffffff,
 437         0x00009040, 0x00000100, 0xffffffff,
 438         0x0000a200, 0x00000100, 0xffffffff,
 439         0x0000a204, 0x00000100, 0xffffffff,
 440         0x0000a208, 0x00000100, 0xffffffff,
 441         0x0000a20c, 0x00000100, 0xffffffff,
 442         0x00009744, 0x00000100, 0xffffffff,
 443         0x00003f80, 0x00000100, 0xffffffff,
 444         0x0000a210, 0x00000100, 0xffffffff,
 445         0x0000a214, 0x00000100, 0xffffffff,
 446         0x000004d8, 0x00000100, 0xffffffff,
 447         0x00009664, 0x00000100, 0xffffffff,
 448         0x00009698, 0x00000100, 0xffffffff,
 449         0x000004d4, 0x00000200, 0xffffffff,
 450         0x000004d0, 0x00000000, 0xffffffff,
 451         0x000030cc, 0x00000104, 0xffffffff,
 452         0x0000d0c0, 0x00000100, 0xffffffff,
 453         0x0000d8c0, 0x00000100, 0xffffffff,
 454         0x0000802c, 0x40000000, 0xffffffff,
 455         0x00003fc4, 0x40000000, 0xffffffff,
 456         0x0000915c, 0x00010000, 0xffffffff,
 457         0x00009160, 0x00030002, 0xffffffff,
 458         0x00009164, 0x00050004, 0xffffffff,
 459         0x00009168, 0x00070006, 0xffffffff,
 460         0x00009178, 0x00070000, 0xffffffff,
 461         0x0000917c, 0x00030002, 0xffffffff,
 462         0x00009180, 0x00050004, 0xffffffff,
 463         0x0000918c, 0x00010006, 0xffffffff,
 464         0x00009190, 0x00090008, 0xffffffff,
 465         0x00009194, 0x00070000, 0xffffffff,
 466         0x00009198, 0x00030002, 0xffffffff,
 467         0x0000919c, 0x00050004, 0xffffffff,
 468         0x000091a8, 0x00010006, 0xffffffff,
 469         0x000091ac, 0x00090008, 0xffffffff,
 470         0x000091b0, 0x00070000, 0xffffffff,
 471         0x000091b4, 0x00030002, 0xffffffff,
 472         0x000091b8, 0x00050004, 0xffffffff,
 473         0x000091c4, 0x00010006, 0xffffffff,
 474         0x000091c8, 0x00090008, 0xffffffff,
 475         0x000091cc, 0x00070000, 0xffffffff,
 476         0x000091d0, 0x00030002, 0xffffffff,
 477         0x000091d4, 0x00050004, 0xffffffff,
 478         0x000091e0, 0x00010006, 0xffffffff,
 479         0x000091e4, 0x00090008, 0xffffffff,
 480         0x000091e8, 0x00000000, 0xffffffff,
 481         0x000091ec, 0x00070000, 0xffffffff,
 482         0x000091f0, 0x00030002, 0xffffffff,
 483         0x000091f4, 0x00050004, 0xffffffff,
 484         0x00009200, 0x00010006, 0xffffffff,
 485         0x00009204, 0x00090008, 0xffffffff,
 486         0x00009208, 0x00070000, 0xffffffff,
 487         0x0000920c, 0x00030002, 0xffffffff,
 488         0x00009210, 0x00050004, 0xffffffff,
 489         0x0000921c, 0x00010006, 0xffffffff,
 490         0x00009220, 0x00090008, 0xffffffff,
 491         0x00009224, 0x00070000, 0xffffffff,
 492         0x00009228, 0x00030002, 0xffffffff,
 493         0x0000922c, 0x00050004, 0xffffffff,
 494         0x00009238, 0x00010006, 0xffffffff,
 495         0x0000923c, 0x00090008, 0xffffffff,
 496         0x00009240, 0x00070000, 0xffffffff,
 497         0x00009244, 0x00030002, 0xffffffff,
 498         0x00009248, 0x00050004, 0xffffffff,
 499         0x00009254, 0x00010006, 0xffffffff,
 500         0x00009258, 0x00090008, 0xffffffff,
 501         0x0000925c, 0x00070000, 0xffffffff,
 502         0x00009260, 0x00030002, 0xffffffff,
 503         0x00009264, 0x00050004, 0xffffffff,
 504         0x00009270, 0x00010006, 0xffffffff,
 505         0x00009274, 0x00090008, 0xffffffff,
 506         0x00009278, 0x00070000, 0xffffffff,
 507         0x0000927c, 0x00030002, 0xffffffff,
 508         0x00009280, 0x00050004, 0xffffffff,
 509         0x0000928c, 0x00010006, 0xffffffff,
 510         0x00009290, 0x00090008, 0xffffffff,
 511         0x000092a8, 0x00070000, 0xffffffff,
 512         0x000092ac, 0x00030002, 0xffffffff,
 513         0x000092b0, 0x00050004, 0xffffffff,
 514         0x000092bc, 0x00010006, 0xffffffff,
 515         0x000092c0, 0x00090008, 0xffffffff,
 516         0x000092c4, 0x00070000, 0xffffffff,
 517         0x000092c8, 0x00030002, 0xffffffff,
 518         0x000092cc, 0x00050004, 0xffffffff,
 519         0x000092d8, 0x00010006, 0xffffffff,
 520         0x000092dc, 0x00090008, 0xffffffff,
 521         0x00009294, 0x00000000, 0xffffffff,
 522         0x0000802c, 0x40010000, 0xffffffff,
 523         0x00003fc4, 0x40010000, 0xffffffff,
 524         0x0000915c, 0x00010000, 0xffffffff,
 525         0x00009160, 0x00030002, 0xffffffff,
 526         0x00009164, 0x00050004, 0xffffffff,
 527         0x00009168, 0x00070006, 0xffffffff,
 528         0x00009178, 0x00070000, 0xffffffff,
 529         0x0000917c, 0x00030002, 0xffffffff,
 530         0x00009180, 0x00050004, 0xffffffff,
 531         0x0000918c, 0x00010006, 0xffffffff,
 532         0x00009190, 0x00090008, 0xffffffff,
 533         0x00009194, 0x00070000, 0xffffffff,
 534         0x00009198, 0x00030002, 0xffffffff,
 535         0x0000919c, 0x00050004, 0xffffffff,
 536         0x000091a8, 0x00010006, 0xffffffff,
 537         0x000091ac, 0x00090008, 0xffffffff,
 538         0x000091b0, 0x00070000, 0xffffffff,
 539         0x000091b4, 0x00030002, 0xffffffff,
 540         0x000091b8, 0x00050004, 0xffffffff,
 541         0x000091c4, 0x00010006, 0xffffffff,
 542         0x000091c8, 0x00090008, 0xffffffff,
 543         0x000091cc, 0x00070000, 0xffffffff,
 544         0x000091d0, 0x00030002, 0xffffffff,
 545         0x000091d4, 0x00050004, 0xffffffff,
 546         0x000091e0, 0x00010006, 0xffffffff,
 547         0x000091e4, 0x00090008, 0xffffffff,
 548         0x000091e8, 0x00000000, 0xffffffff,
 549         0x000091ec, 0x00070000, 0xffffffff,
 550         0x000091f0, 0x00030002, 0xffffffff,
 551         0x000091f4, 0x00050004, 0xffffffff,
 552         0x00009200, 0x00010006, 0xffffffff,
 553         0x00009204, 0x00090008, 0xffffffff,
 554         0x00009208, 0x00070000, 0xffffffff,
 555         0x0000920c, 0x00030002, 0xffffffff,
 556         0x00009210, 0x00050004, 0xffffffff,
 557         0x0000921c, 0x00010006, 0xffffffff,
 558         0x00009220, 0x00090008, 0xffffffff,
 559         0x00009224, 0x00070000, 0xffffffff,
 560         0x00009228, 0x00030002, 0xffffffff,
 561         0x0000922c, 0x00050004, 0xffffffff,
 562         0x00009238, 0x00010006, 0xffffffff,
 563         0x0000923c, 0x00090008, 0xffffffff,
 564         0x00009240, 0x00070000, 0xffffffff,
 565         0x00009244, 0x00030002, 0xffffffff,
 566         0x00009248, 0x00050004, 0xffffffff,
 567         0x00009254, 0x00010006, 0xffffffff,
 568         0x00009258, 0x00090008, 0xffffffff,
 569         0x0000925c, 0x00070000, 0xffffffff,
 570         0x00009260, 0x00030002, 0xffffffff,
 571         0x00009264, 0x00050004, 0xffffffff,
 572         0x00009270, 0x00010006, 0xffffffff,
 573         0x00009274, 0x00090008, 0xffffffff,
 574         0x00009278, 0x00070000, 0xffffffff,
 575         0x0000927c, 0x00030002, 0xffffffff,
 576         0x00009280, 0x00050004, 0xffffffff,
 577         0x0000928c, 0x00010006, 0xffffffff,
 578         0x00009290, 0x00090008, 0xffffffff,
 579         0x000092a8, 0x00070000, 0xffffffff,
 580         0x000092ac, 0x00030002, 0xffffffff,
 581         0x000092b0, 0x00050004, 0xffffffff,
 582         0x000092bc, 0x00010006, 0xffffffff,
 583         0x000092c0, 0x00090008, 0xffffffff,
 584         0x000092c4, 0x00070000, 0xffffffff,
 585         0x000092c8, 0x00030002, 0xffffffff,
 586         0x000092cc, 0x00050004, 0xffffffff,
 587         0x000092d8, 0x00010006, 0xffffffff,
 588         0x000092dc, 0x00090008, 0xffffffff,
 589         0x00009294, 0x00000000, 0xffffffff,
 590         0x0000802c, 0xc0000000, 0xffffffff,
 591         0x00003fc4, 0xc0000000, 0xffffffff,
 592         0x000008f8, 0x00000010, 0xffffffff,
 593         0x000008fc, 0x00000000, 0xffffffff,
 594         0x000008f8, 0x00000011, 0xffffffff,
 595         0x000008fc, 0x00000000, 0xffffffff,
 596         0x000008f8, 0x00000012, 0xffffffff,
 597         0x000008fc, 0x00000000, 0xffffffff,
 598         0x000008f8, 0x00000013, 0xffffffff,
 599         0x000008fc, 0x00000000, 0xffffffff,
 600         0x000008f8, 0x00000014, 0xffffffff,
 601         0x000008fc, 0x00000000, 0xffffffff,
 602         0x000008f8, 0x00000015, 0xffffffff,
 603         0x000008fc, 0x00000000, 0xffffffff,
 604         0x000008f8, 0x00000016, 0xffffffff,
 605         0x000008fc, 0x00000000, 0xffffffff,
 606         0x000008f8, 0x00000017, 0xffffffff,
 607         0x000008fc, 0x00000000, 0xffffffff,
 608         0x000008f8, 0x00000018, 0xffffffff,
 609         0x000008fc, 0x00000000, 0xffffffff,
 610         0x000008f8, 0x00000019, 0xffffffff,
 611         0x000008fc, 0x00000000, 0xffffffff,
 612         0x000008f8, 0x0000001a, 0xffffffff,
 613         0x000008fc, 0x00000000, 0xffffffff,
 614         0x000008f8, 0x0000001b, 0xffffffff,
 615         0x000008fc, 0x00000000, 0xffffffff
 616 };
 617 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 618
 619 static const u32 cayman_mgcg_disable[] =
 620 {
 621         0x0000802c, 0xc0000000, 0xffffffff,
 622         0x000008f8, 0x00000000, 0xffffffff,
 623         0x000008fc, 0xffffffff, 0xffffffff,
 624         0x000008f8, 0x00000001, 0xffffffff,
 625         0x000008fc, 0xffffffff, 0xffffffff,
 626         0x000008f8, 0x00000002, 0xffffffff,
 627         0x000008fc, 0xffffffff, 0xffffffff,
 628         0x000008f8, 0x00000003, 0xffffffff,
 629         0x000008fc, 0xffffffff, 0xffffffff,
 630         0x00009150, 0x00600000, 0xffffffff
 631 };
 632 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 633
 634 static const u32 cayman_mgcg_enable[] =
 635 {
 636         0x0000802c, 0xc0000000, 0xffffffff,
 637         0x000008f8, 0x00000000, 0xffffffff,
 638         0x000008fc, 0x00000000, 0xffffffff,
 639         0x000008f8, 0x00000001, 0xffffffff,
 640         0x000008fc, 0x00000000, 0xffffffff,
 641         0x000008f8, 0x00000002, 0xffffffff,
 642         0x000008fc, 0x00600000, 0xffffffff,
 643         0x000008f8, 0x00000003, 0xffffffff,
 644         0x000008fc, 0x00000000, 0xffffffff,
 645         0x00009150, 0x96944200, 0xffffffff
 646 };
 647
 648 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 649
 650 #define NISLANDS_SYSLS_SEQUENCE  100
 651
 652 static const u32 cayman_sysls_default[] =
 653 {
 654         /* Register,   Value,     Mask bits */
 655         0x000055e8, 0x00000000, 0xffffffff,
 656         0x0000d0bc, 0x00000000, 0xffffffff,
 657         0x0000d8bc, 0x00000000, 0xffffffff,
 658         0x000015c0, 0x000c1401, 0xffffffff,
 659         0x0000264c, 0x000c0400, 0xffffffff,
 660         0x00002648, 0x000c0400, 0xffffffff,
 661         0x00002650, 0x000c0400, 0xffffffff,
 662         0x000020b8, 0x000c0400, 0xffffffff,
 663         0x000020bc, 0x000c0400, 0xffffffff,
 664         0x000020c0, 0x000c0c80, 0xffffffff,
 665         0x0000f4a0, 0x000000c0, 0xffffffff,
 666         0x0000f4a4, 0x00680fff, 0xffffffff,
 667         0x00002f50, 0x00000404, 0xffffffff,
 668         0x000004c8, 0x00000001, 0xffffffff,
 669         0x000064ec, 0x00000000, 0xffffffff,
 670         0x00000c7c, 0x00000000, 0xffffffff,
 671         0x00008dfc, 0x00000000, 0xffffffff
 672 };
 673 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 674
 675 static const u32 cayman_sysls_disable[] =
 676 {
 677         /* Register,   Value,     Mask bits */
 678         0x0000d0c0, 0x00000000, 0xffffffff,
 679         0x0000d8c0, 0x00000000, 0xffffffff,
 680         0x000055e8, 0x00000000, 0xffffffff,
 681         0x0000d0bc, 0x00000000, 0xffffffff,
 682         0x0000d8bc, 0x00000000, 0xffffffff,
 683         0x000015c0, 0x00041401, 0xffffffff,
 684         0x0000264c, 0x00040400, 0xffffffff,
 685         0x00002648, 0x00040400, 0xffffffff,
 686         0x00002650, 0x00040400, 0xffffffff,
 687         0x000020b8, 0x00040400, 0xffffffff,
 688         0x000020bc, 0x00040400, 0xffffffff,
 689         0x000020c0, 0x00040c80, 0xffffffff,
 690         0x0000f4a0, 0x000000c0, 0xffffffff,
 691         0x0000f4a4, 0x00680000, 0xffffffff,
 692         0x00002f50, 0x00000404, 0xffffffff,
 693         0x000004c8, 0x00000001, 0xffffffff,
 694         0x000064ec, 0x00007ffd, 0xffffffff,
 695         0x00000c7c, 0x0000ff00, 0xffffffff,
 696         0x00008dfc, 0x0000007f, 0xffffffff
 697 };
 698 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 699
 700 static const u32 cayman_sysls_enable[] =
 701 {
 702         /* Register,   Value,     Mask bits */
 703         0x000055e8, 0x00000001, 0xffffffff,
 704         0x0000d0bc, 0x00000100, 0xffffffff,
 705         0x0000d8bc, 0x00000100, 0xffffffff,
 706         0x000015c0, 0x000c1401, 0xffffffff,
 707         0x0000264c, 0x000c0400, 0xffffffff,
 708         0x00002648, 0x000c0400, 0xffffffff,
 709         0x00002650, 0x000c0400, 0xffffffff,
 710         0x000020b8, 0x000c0400, 0xffffffff,
 711         0x000020bc, 0x000c0400, 0xffffffff,
 712         0x000020c0, 0x000c0c80, 0xffffffff,
 713         0x0000f4a0, 0x000000c0, 0xffffffff,
 714         0x0000f4a4, 0x00680fff, 0xffffffff,
 715         0x00002f50, 0x00000903, 0xffffffff,
 716         0x000004c8, 0x00000000, 0xffffffff,
 717         0x000064ec, 0x00000000, 0xffffffff,
 718         0x00000c7c, 0x00000000, 0xffffffff,
 719         0x00008dfc, 0x00000000, 0xffffffff
 720 };
 721 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 722
 723 struct rv7xx_power_info *rv770_get_pi(struct radeon_device *rdev);
 724 struct evergreen_power_info *evergreen_get_pi(struct radeon_device *rdev);
 725
 726 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 727
 728 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 729 {
 730         struct ni_power_info *pi = rdev->pm.dpm.priv;
 731
 732         return pi;
 733 }
 734
 735 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 736 {
 737         struct ni_ps *ps = rps->ps_priv;
 738
 739         return ps;
 740 }
 741
 742 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 743                                                      u16 v, s32 t,
 744                                                      u32 ileakage,
 745                                                      u32 *leakage)
 746 {
 747         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 748
 749         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 750         vddc = div64_s64(drm_int2fixp(v), 1000);
 751         temperature = div64_s64(drm_int2fixp(t), 1000);
 752
 753         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 754                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 755         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 756                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 757
 758         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 759
 760         *leakage = drm_fixp2int(leakage_w * 1000);
 761 }
 762
 763 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 764                                              const struct ni_leakage_coeffients *coeff,
 765                                              u16 v,
 766                                              s32 t,
 767                                              u32 i_leakage,
 768                                              u32 *leakage)
 769 {
 770         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 771 }
 772
 773 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 774 {
 775         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 776         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 777         /* we never hit the non-gddr5 limit so disable it */
 778         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 779
 780         if (vblank_time < switch_limit)
 781                 return true;
 782         else
 783                 return false;
 784
 785 }
 786
 787 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 788                                         struct radeon_ps *rps)
 789 {
 790         struct ni_ps *ps = ni_get_ps(rps);
 791         struct radeon_clock_and_voltage_limits *max_limits;
 792         bool disable_mclk_switching;
 793         u32 mclk;
 794         u16 vddci;
 795         int i;
 796
 797         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 798             ni_dpm_vblank_too_short(rdev))
 799                 disable_mclk_switching = true;
 800         else
 801                 disable_mclk_switching = false;
 802
 803         if (rdev->pm.dpm.ac_power)
 804                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 805         else
 806                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 807
 808         if (rdev->pm.dpm.ac_power == false) {
 809                 for (i = 0; i < ps->performance_level_count; i++) {
 810                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 811                                 ps->performance_levels[i].mclk = max_limits->mclk;
 812                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 813                                 ps->performance_levels[i].sclk = max_limits->sclk;
 814                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 815                                 ps->performance_levels[i].vddc = max_limits->vddc;
 816                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 817                                 ps->performance_levels[i].vddci = max_limits->vddci;
 818                 }
 819         }
 820
 821         /* XXX validate the min clocks required for display */
 822
 823         /* adjust low state */
 824         if (disable_mclk_switching) {
 825                 ps->performance_levels[0].mclk =
 826                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 827                 ps->performance_levels[0].vddci =
 828                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 829         }
 830
 831         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 832                                   &ps->performance_levels[0].sclk,
 833                                   &ps->performance_levels[0].mclk);
 834
 835         for (i = 1; i < ps->performance_level_count; i++) {
 836                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 837                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 838                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 839                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 840         }
 841
 842         /* adjust remaining states */
 843         if (disable_mclk_switching) {
 844                 mclk = ps->performance_levels[0].mclk;
 845                 vddci = ps->performance_levels[0].vddci;
 846                 for (i = 1; i < ps->performance_level_count; i++) {
 847                         if (mclk < ps->performance_levels[i].mclk)
 848                                 mclk = ps->performance_levels[i].mclk;
 849                         if (vddci < ps->performance_levels[i].vddci)
 850                                 vddci = ps->performance_levels[i].vddci;
 851                 }
 852                 for (i = 0; i < ps->performance_level_count; i++) {
 853                         ps->performance_levels[i].mclk = mclk;
 854                         ps->performance_levels[i].vddci = vddci;
 855                 }
 856         } else {
 857                 for (i = 1; i < ps->performance_level_count; i++) {
 858                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 859                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 860                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 861                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 862                 }
 863         }
 864
 865         for (i = 1; i < ps->performance_level_count; i++)
 866                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 867                                           &ps->performance_levels[i].sclk,
 868                                           &ps->performance_levels[i].mclk);
 869
 870         for (i = 0; i < ps->performance_level_count; i++)
 871                 btc_adjust_clock_combinations(rdev, max_limits,
 872                                               &ps->performance_levels[i]);
 873
 874         for (i = 0; i < ps->performance_level_count; i++) {
 875                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 876                                                    ps->performance_levels[i].sclk,
 877                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 878                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 879                                                    ps->performance_levels[i].mclk,
 880                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 881                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 882                                                    ps->performance_levels[i].mclk,
 883                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 884                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 885                                                    rdev->clock.current_dispclk,
 886                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 887         }
 888
 889         for (i = 0; i < ps->performance_level_count; i++) {
 890                 btc_apply_voltage_delta_rules(rdev,
 891                                               max_limits->vddc, max_limits->vddci,
 892                                               &ps->performance_levels[i].vddc,
 893                                               &ps->performance_levels[i].vddci);
 894         }
 895
 896         ps->dc_compatible = true;
 897         for (i = 0; i < ps->performance_level_count; i++) {
 898                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 899                         ps->dc_compatible = false;
 900
 901                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 902                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 903         }
 904 }
 905
 906 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 907 {
 908         u32 count;
 909         const u32 *ps = NULL;
 910
 911         ps = (const u32 *)&cayman_cgcg_cgls_default;
 912         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 913
 914         btc_program_mgcg_hw_sequence(rdev, ps, count);
 915 }
 916
 917 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 918                                       bool enable)
 919 {
 920         u32 count;
 921         const u32 *ps = NULL;
 922
 923         if (enable) {
 924                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 925                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 926         } else {
 927                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 928                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 929         }
 930
 931         btc_program_mgcg_hw_sequence(rdev, ps, count);
 932 }
 933
 934 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 935 {
 936         u32 count;
 937         const u32 *ps = NULL;
 938
 939         ps = (const u32 *)&cayman_mgcg_default;
 940         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 941
 942         btc_program_mgcg_hw_sequence(rdev, ps, count);
 943 }
 944
 945 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 946                                      bool enable)
 947 {
 948         u32 count;
 949         const u32 *ps = NULL;
 950
 951         if (enable) {
 952                 ps = (const u32 *)&cayman_mgcg_enable;
 953                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 954         } else {
 955                 ps = (const u32 *)&cayman_mgcg_disable;
 956                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 957         }
 958
 959         btc_program_mgcg_hw_sequence(rdev, ps, count);
 960 }
 961
 962 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 963 {
 964         u32 count;
 965         const u32 *ps = NULL;
 966
 967         ps = (const u32 *)&cayman_sysls_default;
 968         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 969
 970         btc_program_mgcg_hw_sequence(rdev, ps, count);
 971 }
 972
 973 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 974                                      bool enable)
 975 {
 976         u32 count;
 977         const u32 *ps = NULL;
 978
 979         if (enable) {
 980                 ps = (const u32 *)&cayman_sysls_enable;
 981                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 982         } else {
 983                 ps = (const u32 *)&cayman_sysls_disable;
 984                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 985         }
 986
 987         btc_program_mgcg_hw_sequence(rdev, ps, count);
 988
 989 }
 990
 991 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 992                                                              struct radeon_clock_voltage_dependency_table *table)
 993 {
 994         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 995         u32 i;
 996
 997         if (table) {
 998                 for (i = 0; i < table->count; i++) {
 999                         if (0xff01 == table->entries[i].v) {
1000                                 if (pi->max_vddc == 0)
1001                                         return -EINVAL;
1002                                 table->entries[i].v = pi->max_vddc;
1003                         }
1004                 }
1005         }
1006         return 0;
1007 }
1008
1009 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1010 {
1011         int ret = 0;
1012
1013         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1014                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1015
1016         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1017                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1018         return ret;
1019 }
1020
1021 static void ni_stop_dpm(struct radeon_device *rdev)
1022 {
1023         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1024 }
1025
1026 #if 0
1027 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1028                                         bool ac_power)
1029 {
1030         if (ac_power)
1031                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1032                         0 : -EINVAL;
1033
1034         return 0;
1035 }
1036 #endif
1037
1038 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1039                                                       PPSMC_Msg msg, u32 parameter)
1040 {
1041         WREG32(SMC_SCRATCH0, parameter);
1042         return rv770_send_msg_to_smc(rdev, msg);
1043 }
1044
1045 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1046 {
1047         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1048                 return -EINVAL;
1049
1050         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1051                 0 : -EINVAL;
1052 }
1053
1054 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1055                                    enum radeon_dpm_forced_level level)
1056 {
1057         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1058                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1059                         return -EINVAL;
1060
1061                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1062                         return -EINVAL;
1063         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1064                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1065                         return -EINVAL;
1066
1067                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1068                         return -EINVAL;
1069         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1070                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1071                         return -EINVAL;
1072
1073                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1074                         return -EINVAL;
1075         }
1076
1077         rdev->pm.dpm.forced_level = level;
1078
1079         return 0;
1080 }
1081
1082 static void ni_stop_smc(struct radeon_device *rdev)
1083 {
1084         u32 tmp;
1085         int i;
1086
1087         for (i = 0; i < rdev->usec_timeout; i++) {
1088                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1089                 if (tmp != 1)
1090                         break;
1091                 udelay(1);
1092         }
1093
1094         udelay(100);
1095
1096         r7xx_stop_smc(rdev);
1097 }
1098
1099 static int ni_process_firmware_header(struct radeon_device *rdev)
1100 {
1101         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1102         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1103         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1104         u32 tmp;
1105         int ret;
1106
1107         ret = rv770_read_smc_sram_dword(rdev,
1108                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1109                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1110                                         &tmp, pi->sram_end);
1111
1112         if (ret)
1113                 return ret;
1114
1115         pi->state_table_start = (u16)tmp;
1116
1117         ret = rv770_read_smc_sram_dword(rdev,
1118                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1119                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1120                                         &tmp, pi->sram_end);
1121
1122         if (ret)
1123                 return ret;
1124
1125         pi->soft_regs_start = (u16)tmp;
1126
1127         ret = rv770_read_smc_sram_dword(rdev,
1128                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1129                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1130                                         &tmp, pi->sram_end);
1131
1132         if (ret)
1133                 return ret;
1134
1135         eg_pi->mc_reg_table_start = (u16)tmp;
1136
1137         ret = rv770_read_smc_sram_dword(rdev,
1138                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1139                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1140                                         &tmp, pi->sram_end);
1141
1142         if (ret)
1143                 return ret;
1144
1145         ni_pi->fan_table_start = (u16)tmp;
1146
1147         ret = rv770_read_smc_sram_dword(rdev,
1148                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1149                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1150                                         &tmp, pi->sram_end);
1151
1152         if (ret)
1153                 return ret;
1154
1155         ni_pi->arb_table_start = (u16)tmp;
1156
1157         ret = rv770_read_smc_sram_dword(rdev,
1158                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1159                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1160                                         &tmp, pi->sram_end);
1161
1162         if (ret)
1163                 return ret;
1164
1165         ni_pi->cac_table_start = (u16)tmp;
1166
1167         ret = rv770_read_smc_sram_dword(rdev,
1168                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1169                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1170                                         &tmp, pi->sram_end);
1171
1172         if (ret)
1173                 return ret;
1174
1175         ni_pi->spll_table_start = (u16)tmp;
1176
1177
1178         return ret;
1179 }
1180
1181 static void ni_read_clock_registers(struct radeon_device *rdev)
1182 {
1183         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1184
1185         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1186         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1187         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1188         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1189         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1190         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1191         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1192         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1193         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1194         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1195         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1196         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1197         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1198         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1199 }
1200
1201 #if 0
1202 static int ni_enter_ulp_state(struct radeon_device *rdev)
1203 {
1204         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1205
1206         if (pi->gfx_clock_gating) {
1207                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1208                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1209                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1210                 RREG32(GB_ADDR_CONFIG);
1211         }
1212
1213         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1214                  ~HOST_SMC_MSG_MASK);
1215
1216         udelay(25000);
1217
1218         return 0;
1219 }
1220 #endif
1221
1222 static void ni_program_response_times(struct radeon_device *rdev)
1223 {
1224         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1225         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1226         u32 reference_clock;
1227
1228         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1229
1230         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1231         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1232
1233         if (voltage_response_time == 0)
1234                 voltage_response_time = 1000;
1235
1236         if (backbias_response_time == 0)
1237                 backbias_response_time = 1000;
1238
1239         acpi_delay_time = 15000;
1240         vbi_time_out = 100000;
1241
1242         reference_clock = radeon_get_xclk(rdev);
1243
1244         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1245         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1246         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1247         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1248
1249         mclk_switch_limit = (460 * reference_clock) / 100;
1250
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1254         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1255         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1256         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1257 }
1258
1259 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1260                                           struct atom_voltage_table *voltage_table,
1261                                           NISLANDS_SMC_STATETABLE *table)
1262 {
1263         unsigned int i;
1264
1265         for (i = 0; i < voltage_table->count; i++) {
1266                 table->highSMIO[i] = 0;
1267                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1268         }
1269 }
1270
1271 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1272                                            NISLANDS_SMC_STATETABLE *table)
1273 {
1274         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1275         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1276         unsigned char i;
1277
1278         if (eg_pi->vddc_voltage_table.count) {
1279                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1280                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1281                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1282                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1283
1284                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1285                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1286                                 table->maxVDDCIndexInPPTable = i;
1287                                 break;
1288                         }
1289                 }
1290         }
1291
1292         if (eg_pi->vddci_voltage_table.count) {
1293                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1294
1295                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1296                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1297                         cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1298         }
1299 }
1300
1301 static int ni_populate_voltage_value(struct radeon_device *rdev,
1302                                      struct atom_voltage_table *table,
1303                                      u16 value,
1304                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1305 {
1306         unsigned int i;
1307
1308         for (i = 0; i < table->count; i++) {
1309                 if (value <= table->entries[i].value) {
1310                         voltage->index = (u8)i;
1311                         voltage->value = cpu_to_be16(table->entries[i].value);
1312                         break;
1313                 }
1314         }
1315
1316         if (i >= table->count)
1317                 return -EINVAL;
1318
1319         return 0;
1320 }
1321
1322 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1323                                    u32 mclk,
1324                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1325 {
1326         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1327         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1328
1329         if (!pi->mvdd_control) {
1330                 voltage->index = eg_pi->mvdd_high_index;
1331                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1332                 return;
1333         }
1334
1335         if (mclk <= pi->mvdd_split_frequency) {
1336                 voltage->index = eg_pi->mvdd_low_index;
1337                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1338         } else {
1339                 voltage->index = eg_pi->mvdd_high_index;
1340                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1341         }
1342 }
1343
1344 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1345                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1346                                     u16 *std_voltage)
1347 {
1348         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1349             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1350                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1351         else
1352                 *std_voltage = be16_to_cpu(voltage->value);
1353
1354         return 0;
1355 }
1356
1357 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1358                                           u16 value, u8 index,
1359                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1360 {
1361         voltage->index = index;
1362         voltage->value = cpu_to_be16(value);
1363 }
1364
1365 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1366 {
1367         u32 xclk_period;
1368         u32 xclk = radeon_get_xclk(rdev);
1369         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1370
1371         xclk_period = (1000000000UL / xclk);
1372         xclk_period /= 10000UL;
1373
1374         return tmp * xclk_period;
1375 }
1376
1377 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1378 {
1379         return (power_in_watts * scaling_factor) << 2;
1380 }
1381
1382 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1383                                           struct radeon_ps *radeon_state,
1384                                           u32 near_tdp_limit)
1385 {
1386         struct ni_ps *state = ni_get_ps(radeon_state);
1387         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1388         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1389         u32 power_boost_limit = 0;
1390         int ret;
1391
1392         if (ni_pi->enable_power_containment &&
1393             ni_pi->use_power_boost_limit) {
1394                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1395                 u16 std_vddc_med;
1396                 u16 std_vddc_high;
1397                 u64 tmp, n, d;
1398
1399                 if (state->performance_level_count < 3)
1400                         return 0;
1401
1402                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1403                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1404                                                 &vddc);
1405                 if (ret)
1406                         return 0;
1407
1408                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1409                 if (ret)
1410                         return 0;
1411
1412                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1413                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1414                                                 &vddc);
1415                 if (ret)
1416                         return 0;
1417
1418                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1419                 if (ret)
1420                         return 0;
1421
1422                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1423                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1424                 tmp = div64_u64(n, d);
1425
1426                 if (tmp >> 32)
1427                         return 0;
1428                 power_boost_limit = (u32)tmp;
1429         }
1430
1431         return power_boost_limit;
1432 }
1433
1434 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1435                                             bool adjust_polarity,
1436                                             u32 tdp_adjustment,
1437                                             u32 *tdp_limit,
1438                                             u32 *near_tdp_limit)
1439 {
1440         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1441                 return -EINVAL;
1442
1443         if (adjust_polarity) {
1444                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1445                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1446         } else {
1447                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1448                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1449         }
1450
1451         return 0;
1452 }
1453
1454 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1455                                       struct radeon_ps *radeon_state)
1456 {
1457         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1458         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1459
1460         if (ni_pi->enable_power_containment) {
1461                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1462                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1463                 u32 tdp_limit;
1464                 u32 near_tdp_limit;
1465                 u32 power_boost_limit;
1466                 int ret;
1467
1468                 if (scaling_factor == 0)
1469                         return -EINVAL;
1470
1471                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1472
1473                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1474                                                        false, /* ??? */
1475                                                        rdev->pm.dpm.tdp_adjustment,
1476                                                        &tdp_limit,
1477                                                        &near_tdp_limit);
1478                 if (ret)
1479                         return ret;
1480
1481                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1482                                                                    near_tdp_limit);
1483
1484                 smc_table->dpm2Params.TDPLimit =
1485                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1486                 smc_table->dpm2Params.NearTDPLimit =
1487                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1488                 smc_table->dpm2Params.SafePowerLimit =
1489                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1490                                                            scaling_factor));
1491                 smc_table->dpm2Params.PowerBoostLimit =
1492                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1493
1494                 ret = rv770_copy_bytes_to_smc(rdev,
1495                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1496                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1497                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1498                                               sizeof(u32) * 4, pi->sram_end);
1499                 if (ret)
1500                         return ret;
1501         }
1502
1503         return 0;
1504 }
1505
1506 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1507                                 u32 arb_freq_src, u32 arb_freq_dest)
1508 {
1509         u32 mc_arb_dram_timing;
1510         u32 mc_arb_dram_timing2;
1511         u32 burst_time;
1512         u32 mc_cg_config;
1513
1514         switch (arb_freq_src) {
1515         case MC_CG_ARB_FREQ_F0:
1516                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1517                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1518                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1519                 break;
1520         case MC_CG_ARB_FREQ_F1:
1521                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1522                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1523                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1524                 break;
1525         case MC_CG_ARB_FREQ_F2:
1526                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1527                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1528                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1529                 break;
1530         case MC_CG_ARB_FREQ_F3:
1531                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1532                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1533                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1534                 break;
1535         default:
1536                 return -EINVAL;
1537         }
1538
1539         switch (arb_freq_dest) {
1540         case MC_CG_ARB_FREQ_F0:
1541                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1542                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1543                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1544                 break;
1545         case MC_CG_ARB_FREQ_F1:
1546                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1547                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1548                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1549                 break;
1550         case MC_CG_ARB_FREQ_F2:
1551                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1552                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1553                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1554                 break;
1555         case MC_CG_ARB_FREQ_F3:
1556                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1557                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1558                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1559                 break;
1560         default:
1561                 return -EINVAL;
1562         }
1563
1564         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1565         WREG32(MC_CG_CONFIG, mc_cg_config);
1566         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1567
1568         return 0;
1569 }
1570
1571 static int ni_init_arb_table_index(struct radeon_device *rdev)
1572 {
1573         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1574         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1575         u32 tmp;
1576         int ret;
1577
1578         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1579                                         &tmp, pi->sram_end);
1580         if (ret)
1581                 return ret;
1582
1583         tmp &= 0x00FFFFFF;
1584         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1585
1586         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1587                                           tmp, pi->sram_end);
1588 }
1589
1590 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1591 {
1592         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1593 }
1594
1595 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1596 {
1597         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1598         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1599         u32 tmp;
1600         int ret;
1601
1602         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1603                                         &tmp, pi->sram_end);
1604         if (ret)
1605                 return ret;
1606
1607         tmp = (tmp >> 24) & 0xff;
1608
1609         if (tmp == MC_CG_ARB_FREQ_F0)
1610                 return 0;
1611
1612         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1613 }
1614
1615 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1616                                                 struct rv7xx_pl *pl,
1617                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1618 {
1619         u32 dram_timing;
1620         u32 dram_timing2;
1621
1622         arb_regs->mc_arb_rfsh_rate =
1623                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1624
1625
1626         radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1627
1628         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1629         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1630
1631         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1632         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1633
1634         return 0;
1635 }
1636
1637 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1638                                                   struct radeon_ps *radeon_state,
1639                                                   unsigned int first_arb_set)
1640 {
1641         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1642         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1643         struct ni_ps *state = ni_get_ps(radeon_state);
1644         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1645         int i, ret = 0;
1646
1647         for (i = 0; i < state->performance_level_count; i++) {
1648                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1649                 if (ret)
1650                         break;
1651
1652                 ret = rv770_copy_bytes_to_smc(rdev,
1653                                               (u16)(ni_pi->arb_table_start +
1654                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1655                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1656                                               (u8 *)&arb_regs,
1657                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1658                                               pi->sram_end);
1659                 if (ret)
1660                         break;
1661         }
1662         return ret;
1663 }
1664
1665 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1666                                                struct radeon_ps *radeon_new_state)
1667 {
1668         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1669                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1670 }
1671
1672 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1673                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1674 {
1675         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1676
1677         voltage->index = eg_pi->mvdd_high_index;
1678         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1679 }
1680
1681 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1682                                          struct radeon_ps *radeon_initial_state,
1683                                          NISLANDS_SMC_STATETABLE *table)
1684 {
1685         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1686         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1687         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1688         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1689         u32 reg;
1690         int ret;
1691
1692         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL =
1693                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1694         table->initialState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 =
1695                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1696         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL =
1697                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1698         table->initialState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 =
1699                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1700         table->initialState.levels[0].mclk.vMCLK_PWRMGT_CNTL =
1701                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1702         table->initialState.levels[0].mclk.vDLL_CNTL =
1703                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1704         table->initialState.levels[0].mclk.vMPLL_SS =
1705                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1706         table->initialState.levels[0].mclk.vMPLL_SS2 =
1707                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1708         table->initialState.levels[0].mclk.mclk_value =
1709                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1710
1711         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL =
1712                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1713         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 =
1714                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1715         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 =
1716                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1717         table->initialState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 =
1718                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1719         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM =
1720                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1721         table->initialState.levels[0].sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1722                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1723         table->initialState.levels[0].sclk.sclk_value =
1724                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1725         table->initialState.levels[0].arbRefreshState =
1726                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1727
1728         table->initialState.levels[0].ACIndex = 0;
1729
1730         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1731                                         initial_state->performance_levels[0].vddc,
1732                                         &table->initialState.levels[0].vddc);
1733         if (!ret) {
1734                 u16 std_vddc;
1735
1736                 ret = ni_get_std_voltage_value(rdev,
1737                                                &table->initialState.levels[0].vddc,
1738                                                &std_vddc);
1739                 if (!ret)
1740                         ni_populate_std_voltage_value(rdev, std_vddc,
1741                                                       table->initialState.levels[0].vddc.index,
1742                                                       &table->initialState.levels[0].std_vddc);
1743         }
1744
1745         if (eg_pi->vddci_control)
1746                 ni_populate_voltage_value(rdev,
1747                                           &eg_pi->vddci_voltage_table,
1748                                           initial_state->performance_levels[0].vddci,
1749                                           &table->initialState.levels[0].vddci);
1750
1751         ni_populate_initial_mvdd_value(rdev, &table->initialState.levels[0].mvdd);
1752
1753         reg = CG_R(0xffff) | CG_L(0);
1754         table->initialState.levels[0].aT = cpu_to_be32(reg);
1755
1756         table->initialState.levels[0].bSP = cpu_to_be32(pi->dsp);
1757
1758         if (pi->boot_in_gen2)
1759                 table->initialState.levels[0].gen2PCIE = 1;
1760         else
1761                 table->initialState.levels[0].gen2PCIE = 0;
1762
1763         if (pi->mem_gddr5) {
1764                 table->initialState.levels[0].strobeMode =
1765                         cypress_get_strobe_mode_settings(rdev,
1766                                                          initial_state->performance_levels[0].mclk);
1767
1768                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1769                         table->initialState.levels[0].mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1770                 else
1771                         table->initialState.levels[0].mcFlags =  0;
1772         }
1773
1774         table->initialState.levelCount = 1;
1775
1776         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1777
1778         table->initialState.levels[0].dpm2.MaxPS = 0;
1779         table->initialState.levels[0].dpm2.NearTDPDec = 0;
1780         table->initialState.levels[0].dpm2.AboveSafeInc = 0;
1781         table->initialState.levels[0].dpm2.BelowSafeInc = 0;
1782
1783         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1784         table->initialState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1785
1786         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1787         table->initialState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1788
1789         return 0;
1790 }
1791
1792 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1793                                       NISLANDS_SMC_STATETABLE *table)
1794 {
1795         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1796         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1797         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1798         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1799         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1800         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1801         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1802         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1803         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1804         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1805         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1806         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1807         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1808         u32 reg;
1809         int ret;
1810
1811         table->ACPIState = table->initialState;
1812
1813         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1814
1815         if (pi->acpi_vddc) {
1816                 ret = ni_populate_voltage_value(rdev,
1817                                                 &eg_pi->vddc_voltage_table,
1818                                                 pi->acpi_vddc, &table->ACPIState.levels[0].vddc);
1819                 if (!ret) {
1820                         u16 std_vddc;
1821
1822                         ret = ni_get_std_voltage_value(rdev,
1823                                                        &table->ACPIState.levels[0].vddc, &std_vddc);
1824                         if (!ret)
1825                                 ni_populate_std_voltage_value(rdev, std_vddc,
1826                                                               table->ACPIState.levels[0].vddc.index,
1827                                                               &table->ACPIState.levels[0].std_vddc);
1828                 }
1829
1830                 if (pi->pcie_gen2) {
1831                         if (pi->acpi_pcie_gen2)
1832                                 table->ACPIState.levels[0].gen2PCIE = 1;
1833                         else
1834                                 table->ACPIState.levels[0].gen2PCIE = 0;
1835                 } else {
1836                         table->ACPIState.levels[0].gen2PCIE = 0;
1837                 }
1838         } else {
1839                 ret = ni_populate_voltage_value(rdev,
1840                                                 &eg_pi->vddc_voltage_table,
1841                                                 pi->min_vddc_in_table,
1842                                                 &table->ACPIState.levels[0].vddc);
1843                 if (!ret) {
1844                         u16 std_vddc;
1845
1846                         ret = ni_get_std_voltage_value(rdev,
1847                                                        &table->ACPIState.levels[0].vddc,
1848                                                        &std_vddc);
1849                         if (!ret)
1850                                 ni_populate_std_voltage_value(rdev, std_vddc,
1851                                                               table->ACPIState.levels[0].vddc.index,
1852                                                               &table->ACPIState.levels[0].std_vddc);
1853                 }
1854                 table->ACPIState.levels[0].gen2PCIE = 0;
1855         }
1856
1857         if (eg_pi->acpi_vddci) {
1858                 if (eg_pi->vddci_control)
1859                         ni_populate_voltage_value(rdev,
1860                                                   &eg_pi->vddci_voltage_table,
1861                                                   eg_pi->acpi_vddci,
1862                                                   &table->ACPIState.levels[0].vddci);
1863         }
1864
1865
1866         mpll_ad_func_cntl &= ~PDNB;
1867
1868         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1869
1870         if (pi->mem_gddr5)
1871                 mpll_dq_func_cntl &= ~PDNB;
1872         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1873
1874
1875         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1876                              MRDCKA1_RESET |
1877                              MRDCKB0_RESET |
1878                              MRDCKB1_RESET |
1879                              MRDCKC0_RESET |
1880                              MRDCKC1_RESET |
1881                              MRDCKD0_RESET |
1882                              MRDCKD1_RESET);
1883
1884         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1885                               MRDCKA1_PDNB |
1886                               MRDCKB0_PDNB |
1887                               MRDCKB1_PDNB |
1888                               MRDCKC0_PDNB |
1889                               MRDCKC1_PDNB |
1890                               MRDCKD0_PDNB |
1891                               MRDCKD1_PDNB);
1892
1893         dll_cntl |= (MRDCKA0_BYPASS |
1894                      MRDCKA1_BYPASS |
1895                      MRDCKB0_BYPASS |
1896                      MRDCKB1_BYPASS |
1897                      MRDCKC0_BYPASS |
1898                      MRDCKC1_BYPASS |
1899                      MRDCKD0_BYPASS |
1900                      MRDCKD1_BYPASS);
1901
1902         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1903         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1904
1905         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1906         table->ACPIState.levels[0].mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1907         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1908         table->ACPIState.levels[0].mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1909         table->ACPIState.levels[0].mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1910         table->ACPIState.levels[0].mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1911
1912         table->ACPIState.levels[0].mclk.mclk_value = 0;
1913
1914         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1915         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1916         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1917         table->ACPIState.levels[0].sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1918
1919         table->ACPIState.levels[0].sclk.sclk_value = 0;
1920
1921         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.levels[0].mvdd);
1922
1923         if (eg_pi->dynamic_ac_timing)
1924                 table->ACPIState.levels[0].ACIndex = 1;
1925
1926         table->ACPIState.levels[0].dpm2.MaxPS = 0;
1927         table->ACPIState.levels[0].dpm2.NearTDPDec = 0;
1928         table->ACPIState.levels[0].dpm2.AboveSafeInc = 0;
1929         table->ACPIState.levels[0].dpm2.BelowSafeInc = 0;
1930
1931         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1932         table->ACPIState.levels[0].SQPowerThrottle = cpu_to_be32(reg);
1933
1934         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1935         table->ACPIState.levels[0].SQPowerThrottle_2 = cpu_to_be32(reg);
1936
1937         return 0;
1938 }
1939
1940 static int ni_init_smc_table(struct radeon_device *rdev)
1941 {
1942         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1943         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1944         int ret;
1945         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1946         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1947
1948         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1949
1950         ni_populate_smc_voltage_tables(rdev, table);
1951
1952         switch (rdev->pm.int_thermal_type) {
1953         case THERMAL_TYPE_NI:
1954         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1955                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1956                 break;
1957         case THERMAL_TYPE_NONE:
1958                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1959                 break;
1960         default:
1961                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1962                 break;
1963         }
1964
1965         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1966                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1967
1968         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1969                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1970
1971         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1972                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1973
1974         if (pi->mem_gddr5)
1975                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1976
1977         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1978         if (ret)
1979                 return ret;
1980
1981         ret = ni_populate_smc_acpi_state(rdev, table);
1982         if (ret)
1983                 return ret;
1984
1985         table->driverState = table->initialState;
1986
1987         table->ULVState = table->initialState;
1988
1989         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1990                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1991         if (ret)
1992                 return ret;
1993
1994         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1995                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1996 }
1997
1998 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1999                                     u32 engine_clock,
2000                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2001 {
2002         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2003         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2004         struct atom_clock_dividers dividers;
2005         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2006         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2007         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2008         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2009         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2010         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2011         u64 tmp;
2012         u32 reference_clock = rdev->clock.spll.reference_freq;
2013         u32 reference_divider;
2014         u32 fbdiv;
2015         int ret;
2016
2017         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2018                                              engine_clock, false, &dividers);
2019         if (ret)
2020                 return ret;
2021
2022         reference_divider = 1 + dividers.ref_div;
2023
2024
2025         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2026         do_div(tmp, reference_clock);
2027         fbdiv = (u32) tmp;
2028
2029         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2030         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2031         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2032
2033         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2034         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2035
2036         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2037         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2038         spll_func_cntl_3 |= SPLL_DITHEN;
2039
2040         if (pi->sclk_ss) {
2041                 struct radeon_atom_ss ss;
2042                 u32 vco_freq = engine_clock * dividers.post_div;
2043
2044                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2045                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2046                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2047                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2048
2049                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2050                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2051                         cg_spll_spread_spectrum |= SSEN;
2052
2053                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2054                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2055                 }
2056         }
2057
2058         sclk->sclk_value = engine_clock;
2059         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2060         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2061         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2062         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2063         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2064         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2065
2066         return 0;
2067 }
2068
2069 static int ni_populate_sclk_value(struct radeon_device *rdev,
2070                                   u32 engine_clock,
2071                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2072 {
2073         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2074         int ret;
2075
2076         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2077         if (!ret) {
2078                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2079                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2080                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2081                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2082                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2083                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2084                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2085         }
2086
2087         return ret;
2088 }
2089
2090 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2091 {
2092         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2093         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2094         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2095         NISLANDS_SMC_SCLK_VALUE sclk_params;
2096         u32 fb_div;
2097         u32 p_div;
2098         u32 clk_s;
2099         u32 clk_v;
2100         u32 sclk = 0;
2101         int i, ret;
2102         u32 tmp;
2103
2104         if (ni_pi->spll_table_start == 0)
2105                 return -EINVAL;
2106
2107         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2108         if (spll_table == NULL)
2109                 return -ENOMEM;
2110
2111         for (i = 0; i < 256; i++) {
2112                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2113                 if (ret)
2114                         break;
2115
2116                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2117                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2118                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2119                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2120
2121                 fb_div &= ~0x00001FFF;
2122                 fb_div >>= 1;
2123                 clk_v >>= 6;
2124
2125                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2126                         ret = -EINVAL;
2127
2128                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2129                         ret = -EINVAL;
2130
2131                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2132                         ret = -EINVAL;
2133
2134                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2135                         ret = -EINVAL;
2136
2137                 if (ret)
2138                         break;
2139
2140                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2141                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2142                 spll_table->freq[i] = cpu_to_be32(tmp);
2143
2144                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2145                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2146                 spll_table->ss[i] = cpu_to_be32(tmp);
2147
2148                 sclk += 512;
2149         }
2150
2151         if (!ret)
2152                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2153                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2154
2155         kfree(spll_table);
2156
2157         return ret;
2158 }
2159
2160 static int ni_populate_mclk_value(struct radeon_device *rdev,
2161                                   u32 engine_clock,
2162                                   u32 memory_clock,
2163                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2164                                   bool strobe_mode,
2165                                   bool dll_state_on)
2166 {
2167         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2168         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2169         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2170         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2171         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2172         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2173         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2174         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2175         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2176         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2177         struct atom_clock_dividers dividers;
2178         u32 ibias;
2179         u32 dll_speed;
2180         int ret;
2181         u32 mc_seq_misc7;
2182
2183         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2184                                              memory_clock, strobe_mode, &dividers);
2185         if (ret)
2186                 return ret;
2187
2188         if (!strobe_mode) {
2189                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2190
2191                 if (mc_seq_misc7 & 0x8000000)
2192                         dividers.post_div = 1;
2193         }
2194
2195         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2196
2197         mpll_ad_func_cntl &= ~(CLKR_MASK |
2198                                YCLK_POST_DIV_MASK |
2199                                CLKF_MASK |
2200                                CLKFRAC_MASK |
2201                                IBIAS_MASK);
2202         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2203         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2204         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2205         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2206         mpll_ad_func_cntl |= IBIAS(ibias);
2207
2208         if (dividers.vco_mode)
2209                 mpll_ad_func_cntl_2 |= VCO_MODE;
2210         else
2211                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2212
2213         if (pi->mem_gddr5) {
2214                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2215                                        YCLK_POST_DIV_MASK |
2216                                        CLKF_MASK |
2217                                        CLKFRAC_MASK |
2218                                        IBIAS_MASK);
2219                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2220                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2221                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2222                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2223                 mpll_dq_func_cntl |= IBIAS(ibias);
2224
2225                 if (strobe_mode)
2226                         mpll_dq_func_cntl &= ~PDNB;
2227                 else
2228                         mpll_dq_func_cntl |= PDNB;
2229
2230                 if (dividers.vco_mode)
2231                         mpll_dq_func_cntl_2 |= VCO_MODE;
2232                 else
2233                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2234         }
2235
2236         if (pi->mclk_ss) {
2237                 struct radeon_atom_ss ss;
2238                 u32 vco_freq = memory_clock * dividers.post_div;
2239
2240                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2241                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2242                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2243                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2244                         u32 clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2245                         u32 clk_v = ss.percentage *
2246                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2247
2248                         mpll_ss1 &= ~CLKV_MASK;
2249                         mpll_ss1 |= CLKV(clk_v);
2250
2251                         mpll_ss2 &= ~CLKS_MASK;
2252                         mpll_ss2 |= CLKS(clk_s);
2253                 }
2254         }
2255
2256         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2257                                         memory_clock);
2258
2259         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2260         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2261         if (dll_state_on)
2262                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2263                                      MRDCKA1_PDNB |
2264                                      MRDCKB0_PDNB |
2265                                      MRDCKB1_PDNB |
2266                                      MRDCKC0_PDNB |
2267                                      MRDCKC1_PDNB |
2268                                      MRDCKD0_PDNB |
2269                                      MRDCKD1_PDNB);
2270         else
2271                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2272                                       MRDCKA1_PDNB |
2273                                       MRDCKB0_PDNB |
2274                                       MRDCKB1_PDNB |
2275                                       MRDCKC0_PDNB |
2276                                       MRDCKC1_PDNB |
2277                                       MRDCKD0_PDNB |
2278                                       MRDCKD1_PDNB);
2279
2280
2281         mclk->mclk_value = cpu_to_be32(memory_clock);
2282         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2283         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2284         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2285         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2286         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2287         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2288         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2289         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2290
2291         return 0;
2292 }
2293
2294 static void ni_populate_smc_sp(struct radeon_device *rdev,
2295                                struct radeon_ps *radeon_state,
2296                                NISLANDS_SMC_SWSTATE *smc_state)
2297 {
2298         struct ni_ps *ps = ni_get_ps(radeon_state);
2299         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2300         int i;
2301
2302         for (i = 0; i < ps->performance_level_count - 1; i++)
2303                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2304
2305         smc_state->levels[ps->performance_level_count - 1].bSP =
2306                 cpu_to_be32(pi->psp);
2307 }
2308
2309 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2310                                          struct rv7xx_pl *pl,
2311                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2312 {
2313         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2314         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2315         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2316         int ret;
2317         bool dll_state_on;
2318         u16 std_vddc;
2319         u32 tmp = RREG32(DC_STUTTER_CNTL);
2320
2321         level->gen2PCIE = pi->pcie_gen2 ?
2322                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2323
2324         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2325         if (ret)
2326                 return ret;
2327
2328         level->mcFlags =  0;
2329         if (pi->mclk_stutter_mode_threshold &&
2330             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2331             !eg_pi->uvd_enabled &&
2332             (tmp & DC_STUTTER_ENABLE_A) &&
2333             (tmp & DC_STUTTER_ENABLE_B))
2334                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2335
2336         if (pi->mem_gddr5) {
2337                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2338                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2339                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2340                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2341
2342                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2343
2344                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2345                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2346                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2347                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2348                         else
2349                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2350                 } else {
2351                         dll_state_on = false;
2352                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2353                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2354                 }
2355
2356                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2357                                              &level->mclk,
2358                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2359                                              dll_state_on);
2360         } else
2361                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2362
2363         if (ret)
2364                 return ret;
2365
2366         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2367                                         pl->vddc, &level->vddc);
2368         if (ret)
2369                 return ret;
2370
2371         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2372         if (ret)
2373                 return ret;
2374
2375         ni_populate_std_voltage_value(rdev, std_vddc,
2376                                       level->vddc.index, &level->std_vddc);
2377
2378         if (eg_pi->vddci_control) {
2379                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2380                                                 pl->vddci, &level->vddci);
2381                 if (ret)
2382                         return ret;
2383         }
2384
2385         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2386
2387         return ret;
2388 }
2389
2390 static int ni_populate_smc_t(struct radeon_device *rdev,
2391                              struct radeon_ps *radeon_state,
2392                              NISLANDS_SMC_SWSTATE *smc_state)
2393 {
2394         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2395         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2396         struct ni_ps *state = ni_get_ps(radeon_state);
2397         u32 a_t;
2398         u32 t_l, t_h;
2399         u32 high_bsp;
2400         int i, ret;
2401
2402         if (state->performance_level_count >= 9)
2403                 return -EINVAL;
2404
2405         if (state->performance_level_count < 2) {
2406                 a_t = CG_R(0xffff) | CG_L(0);
2407                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2408                 return 0;
2409         }
2410
2411         smc_state->levels[0].aT = cpu_to_be32(0);
2412
2413         for (i = 0; i <= state->performance_level_count - 2; i++) {
2414                 if (eg_pi->uvd_enabled)
2415                         ret = r600_calculate_at(
2416                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2417                                 100 * R600_AH_DFLT,
2418                                 state->performance_levels[i + 1].sclk,
2419                                 state->performance_levels[i].sclk,
2420                                 &t_l,
2421                                 &t_h);
2422                 else
2423                         ret = r600_calculate_at(
2424                                 1000 * (i + 1),
2425                                 100 * R600_AH_DFLT,
2426                                 state->performance_levels[i + 1].sclk,
2427                                 state->performance_levels[i].sclk,
2428                                 &t_l,
2429                                 &t_h);
2430
2431                 if (ret) {
2432                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2433                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2434                 }
2435
2436                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2437                 a_t |= CG_R(t_l * pi->bsp / 20000);
2438                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2439
2440                 high_bsp = (i == state->performance_level_count - 2) ?
2441                         pi->pbsp : pi->bsp;
2442
2443                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2444                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2445         }
2446
2447         return 0;
2448 }
2449
2450 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2451                                                 struct radeon_ps *radeon_state,
2452                                                 NISLANDS_SMC_SWSTATE *smc_state)
2453 {
2454         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2455         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2456         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2457         struct ni_ps *state = ni_get_ps(radeon_state);
2458         u32 prev_sclk;
2459         u32 max_sclk;
2460         u32 min_sclk;
2461         int i, ret;
2462         u32 tdp_limit;
2463         u32 near_tdp_limit;
2464         u32 power_boost_limit;
2465         u8 max_ps_percent;
2466
2467         if (ni_pi->enable_power_containment == false)
2468                 return 0;
2469
2470         if (state->performance_level_count == 0)
2471                 return -EINVAL;
2472
2473         if (smc_state->levelCount != state->performance_level_count)
2474                 return -EINVAL;
2475
2476         ret = ni_calculate_adjusted_tdp_limits(rdev,
2477                                                false, /* ??? */
2478                                                rdev->pm.dpm.tdp_adjustment,
2479                                                &tdp_limit,
2480                                                &near_tdp_limit);
2481         if (ret)
2482                 return ret;
2483
2484         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2485
2486         ret = rv770_write_smc_sram_dword(rdev,
2487                                          pi->state_table_start +
2488                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2489                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2490                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2491                                          pi->sram_end);
2492         if (ret)
2493                 power_boost_limit = 0;
2494
2495         smc_state->levels[0].dpm2.MaxPS = 0;
2496         smc_state->levels[0].dpm2.NearTDPDec = 0;
2497         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2498         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2499         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2500
2501         for (i = 1; i < state->performance_level_count; i++) {
2502                 prev_sclk = state->performance_levels[i-1].sclk;
2503                 max_sclk  = state->performance_levels[i].sclk;
2504                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2505                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2506
2507                 if (max_sclk < prev_sclk)
2508                         return -EINVAL;
2509
2510                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2511                         min_sclk = max_sclk;
2512                 else if (1 == i)
2513                         min_sclk = prev_sclk;
2514                 else
2515                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2516
2517                 if (min_sclk < state->performance_levels[0].sclk)
2518                         min_sclk = state->performance_levels[0].sclk;
2519
2520                 if (min_sclk == 0)
2521                         return -EINVAL;
2522
2523                 smc_state->levels[i].dpm2.MaxPS =
2524                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2525                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2526                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2527                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2528                 smc_state->levels[i].stateFlags |=
2529                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2530                         PPSMC_STATEFLAG_POWERBOOST : 0;
2531         }
2532
2533         return 0;
2534 }
2535
2536 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2537                                          struct radeon_ps *radeon_state,
2538                                          NISLANDS_SMC_SWSTATE *smc_state)
2539 {
2540         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2541         struct ni_ps *state = ni_get_ps(radeon_state);
2542         u32 sq_power_throttle;
2543         u32 sq_power_throttle2;
2544         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2545         int i;
2546
2547         if (state->performance_level_count == 0)
2548                 return -EINVAL;
2549
2550         if (smc_state->levelCount != state->performance_level_count)
2551                 return -EINVAL;
2552
2553         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2554                 return -EINVAL;
2555
2556         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2557                 enable_sq_ramping = false;
2558
2559         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2560                 enable_sq_ramping = false;
2561
2562         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2563                 enable_sq_ramping = false;
2564
2565         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2566                 enable_sq_ramping = false;
2567
2568         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2569                 enable_sq_ramping = false;
2570
2571         for (i = 0; i < state->performance_level_count; i++) {
2572                 sq_power_throttle  = 0;
2573                 sq_power_throttle2 = 0;
2574
2575                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2576                     enable_sq_ramping) {
2577                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2578                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2579                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2580                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2581                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2582                 } else {
2583                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2584                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2585                 }
2586
2587                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2588                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2589         }
2590
2591         return 0;
2592 }
2593
2594 static int ni_enable_power_containment(struct radeon_device *rdev,
2595                                        struct radeon_ps *radeon_new_state,
2596                                        bool enable)
2597 {
2598         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2599         PPSMC_Result smc_result;
2600         int ret = 0;
2601
2602         if (ni_pi->enable_power_containment) {
2603                 if (enable) {
2604                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2605                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2606                                 if (smc_result != PPSMC_Result_OK) {
2607                                         ret = -EINVAL;
2608                                         ni_pi->pc_enabled = false;
2609                                 } else {
2610                                         ni_pi->pc_enabled = true;
2611                                 }
2612                         }
2613                 } else {
2614                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2615                         if (smc_result != PPSMC_Result_OK)
2616                                 ret = -EINVAL;
2617                         ni_pi->pc_enabled = false;
2618                 }
2619         }
2620
2621         return ret;
2622 }
2623
2624 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2625                                          struct radeon_ps *radeon_state,
2626                                          NISLANDS_SMC_SWSTATE *smc_state)
2627 {
2628         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2629         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2630         struct ni_ps *state = ni_get_ps(radeon_state);
2631         int i, ret;
2632         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2633
2634         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2635                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2636
2637         smc_state->levelCount = 0;
2638
2639         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2640                 return -EINVAL;
2641
2642         for (i = 0; i < state->performance_level_count; i++) {
2643                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2644                                                     &smc_state->levels[i]);
2645                 smc_state->levels[i].arbRefreshState =
2646                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2647
2648                 if (ret)
2649                         return ret;
2650
2651                 if (ni_pi->enable_power_containment)
2652                         smc_state->levels[i].displayWatermark =
2653                                 (state->performance_levels[i].sclk < threshold) ?
2654                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2655                 else
2656                         smc_state->levels[i].displayWatermark = (i < 2) ?
2657                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2658
2659                 if (eg_pi->dynamic_ac_timing)
2660                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2661                 else
2662                         smc_state->levels[i].ACIndex = 0;
2663
2664                 smc_state->levelCount++;
2665         }
2666
2667         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2668                                       cpu_to_be32(threshold / 512));
2669
2670         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2671
2672         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2673         if (ret)
2674                 ni_pi->enable_power_containment = false;
2675
2676         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2677         if (ret)
2678                 ni_pi->enable_sq_ramping = false;
2679
2680         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2681 }
2682
2683 static int ni_upload_sw_state(struct radeon_device *rdev,
2684                               struct radeon_ps *radeon_new_state)
2685 {
2686         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2687         u16 address = pi->state_table_start +
2688                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2689         u16 state_size = sizeof(NISLANDS_SMC_SWSTATE) +
2690                 ((NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE - 1) * sizeof(NISLANDS_SMC_HW_PERFORMANCE_LEVEL));
2691         int ret;
2692         NISLANDS_SMC_SWSTATE *smc_state = kzalloc(state_size, GFP_KERNEL);
2693
2694         if (smc_state == NULL)
2695                 return -ENOMEM;
2696
2697         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2698         if (ret)
2699                 goto done;
2700
2701         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2702
2703 done:
2704         kfree(smc_state);
2705
2706         return ret;
2707 }
2708
2709 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2710                                        struct ni_mc_reg_table *table)
2711 {
2712         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2713         u8 i, j, k;
2714         u32 temp_reg;
2715
2716         for (i = 0, j = table->last; i < table->last; i++) {
2717                 switch (table->mc_reg_address[i].s1) {
2718                 case MC_SEQ_MISC1 >> 2:
2719                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2720                                 return -EINVAL;
2721                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2722                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2723                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2724                         for (k = 0; k < table->num_entries; k++)
2725                                 table->mc_reg_table_entry[k].mc_data[j] =
2726                                         ((temp_reg & 0xffff0000)) |
2727                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2728                         j++;
2729                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2730                                 return -EINVAL;
2731
2732                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2733                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2734                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2735                         for(k = 0; k < table->num_entries; k++) {
2736                                 table->mc_reg_table_entry[k].mc_data[j] =
2737                                         (temp_reg & 0xffff0000) |
2738                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2739                                 if (!pi->mem_gddr5)
2740                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2741                         }
2742                         j++;
2743                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2744                                 return -EINVAL;
2745                         break;
2746                 case MC_SEQ_RESERVE_M >> 2:
2747                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2748                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2749                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2750                         for (k = 0; k < table->num_entries; k++)
2751                                 table->mc_reg_table_entry[k].mc_data[j] =
2752                                         (temp_reg & 0xffff0000) |
2753                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2754                         j++;
2755                         if (j > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2756                                 return -EINVAL;
2757                         break;
2758                 default:
2759                         break;
2760                 }
2761         }
2762
2763         table->last = j;
2764
2765         return 0;
2766 }
2767
2768 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2769 {
2770         bool result = true;
2771
2772         switch (in_reg) {
2773         case  MC_SEQ_RAS_TIMING >> 2:
2774                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2775                 break;
2776         case MC_SEQ_CAS_TIMING >> 2:
2777                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2778                 break;
2779         case MC_SEQ_MISC_TIMING >> 2:
2780                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2781                 break;
2782         case MC_SEQ_MISC_TIMING2 >> 2:
2783                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2784                 break;
2785         case MC_SEQ_RD_CTL_D0 >> 2:
2786                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2787                 break;
2788         case MC_SEQ_RD_CTL_D1 >> 2:
2789                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2790                 break;
2791         case MC_SEQ_WR_CTL_D0 >> 2:
2792                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2793                 break;
2794         case MC_SEQ_WR_CTL_D1 >> 2:
2795                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2796                 break;
2797         case MC_PMG_CMD_EMRS >> 2:
2798                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2799                 break;
2800         case MC_PMG_CMD_MRS >> 2:
2801                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2802                 break;
2803         case MC_PMG_CMD_MRS1 >> 2:
2804                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2805                 break;
2806         case MC_SEQ_PMG_TIMING >> 2:
2807                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2808                 break;
2809         case MC_PMG_CMD_MRS2 >> 2:
2810                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2811                 break;
2812         default:
2813                 result = false;
2814                 break;
2815         }
2816
2817         return result;
2818 }
2819
2820 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2821 {
2822         u8 i, j;
2823
2824         for (i = 0; i < table->last; i++) {
2825                 for (j = 1; j < table->num_entries; j++) {
2826                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2827                                 table->valid_flag |= 1 << i;
2828                                 break;
2829                         }
2830                 }
2831         }
2832 }
2833
2834 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2835 {
2836         u32 i;
2837         u16 address;
2838
2839         for (i = 0; i < table->last; i++)
2840                 table->mc_reg_address[i].s0 =
2841                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2842                         address : table->mc_reg_address[i].s1;
2843 }
2844
2845 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2846                                       struct ni_mc_reg_table *ni_table)
2847 {
2848         u8 i, j;
2849
2850         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2851                 return -EINVAL;
2852         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2853                 return -EINVAL;
2854
2855         for (i = 0; i < table->last; i++)
2856                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2857         ni_table->last = table->last;
2858
2859         for (i = 0; i < table->num_entries; i++) {
2860                 ni_table->mc_reg_table_entry[i].mclk_max =
2861                         table->mc_reg_table_entry[i].mclk_max;
2862                 for (j = 0; j < table->last; j++)
2863                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2864                                 table->mc_reg_table_entry[i].mc_data[j];
2865         }
2866         ni_table->num_entries = table->num_entries;
2867
2868         return 0;
2869 }
2870
2871 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2872 {
2873         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2874         int ret;
2875         struct atom_mc_reg_table *table;
2876         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2877         u8 module_index = rv770_get_memory_module_index(rdev);
2878
2879         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2880         if (!table)
2881                 return -ENOMEM;
2882
2883         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2884         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2885         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2886         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2887         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2888         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2889         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2890         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2891         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2892         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2893         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2894         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2895         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2896
2897         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2898
2899         if (ret)
2900                 goto init_mc_done;
2901
2902         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2903
2904         if (ret)
2905                 goto init_mc_done;
2906
2907         ni_set_s0_mc_reg_index(ni_table);
2908
2909         ret = ni_set_mc_special_registers(rdev, ni_table);
2910
2911         if (ret)
2912                 goto init_mc_done;
2913
2914         ni_set_valid_flag(ni_table);
2915
2916 init_mc_done:
2917         kfree(table);
2918
2919         return ret;
2920 }
2921
2922 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2923                                          SMC_NIslands_MCRegisters *mc_reg_table)
2924 {
2925         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2926         u32 i, j;
2927
2928         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2929                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2930                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2931                                 break;
2932                         mc_reg_table->address[i].s0 =
2933                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2934                         mc_reg_table->address[i].s1 =
2935                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2936                         i++;
2937                 }
2938         }
2939         mc_reg_table->last = (u8)i;
2940 }
2941
2942
2943 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2944                                     SMC_NIslands_MCRegisterSet *data,
2945                                     u32 num_entries, u32 valid_flag)
2946 {
2947         u32 i, j;
2948
2949         for (i = 0, j = 0; j < num_entries; j++) {
2950                 if (valid_flag & (1 << j)) {
2951                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2952                         i++;
2953                 }
2954         }
2955 }
2956
2957 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2958                                                  struct rv7xx_pl *pl,
2959                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2960 {
2961         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2962         u32 i = 0;
2963
2964         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2965                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2966                         break;
2967         }
2968
2969         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2970                 --i;
2971
2972         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2973                                 mc_reg_table_data,
2974                                 ni_pi->mc_reg_table.last,
2975                                 ni_pi->mc_reg_table.valid_flag);
2976 }
2977
2978 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2979                                            struct radeon_ps *radeon_state,
2980                                            SMC_NIslands_MCRegisters *mc_reg_table)
2981 {
2982         struct ni_ps *state = ni_get_ps(radeon_state);
2983         int i;
2984
2985         for (i = 0; i < state->performance_level_count; i++) {
2986                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2987                                                      &state->performance_levels[i],
2988                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2989         }
2990 }
2991
2992 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2993                                     struct radeon_ps *radeon_boot_state)
2994 {
2995         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2996         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2997         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2998         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
2999         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3000
3001         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3002
3003         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3004
3005         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3006
3007         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3008                                              &mc_reg_table->data[0]);
3009
3010         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3011                                 &mc_reg_table->data[1],
3012                                 ni_pi->mc_reg_table.last,
3013                                 ni_pi->mc_reg_table.valid_flag);
3014
3015         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3016
3017         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3018                                        (u8 *)mc_reg_table,
3019                                        sizeof(SMC_NIslands_MCRegisters),
3020                                        pi->sram_end);
3021 }
3022
3023 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3024                                   struct radeon_ps *radeon_new_state)
3025 {
3026         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3027         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3028         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3029         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3030         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3031         u16 address;
3032
3033         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3034
3035         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3036
3037         address = eg_pi->mc_reg_table_start +
3038                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3039
3040         return rv770_copy_bytes_to_smc(rdev, address,
3041                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3042                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3043                                        pi->sram_end);
3044 }
3045
3046 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3047                                                    PP_NIslands_CACTABLES *cac_tables)
3048 {
3049         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3050         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3051         u32 leakage = 0;
3052         unsigned int i, j, table_size;
3053         s32 t;
3054         u32 smc_leakage, max_leakage = 0;
3055         u32 scaling_factor;
3056
3057         table_size = eg_pi->vddc_voltage_table.count;
3058
3059         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3060                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3061
3062         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3063
3064         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3065                 for (j = 0; j < table_size; j++) {
3066                         t = (1000 * ((i + 1) * 8));
3067
3068                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3069                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3070
3071                         ni_calculate_leakage_for_v_and_t(rdev,
3072                                                          &ni_pi->cac_data.leakage_coefficients,
3073                                                          eg_pi->vddc_voltage_table.entries[j].value,
3074                                                          t,
3075                                                          ni_pi->cac_data.i_leakage,
3076                                                          &leakage);
3077
3078                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3079                         if (smc_leakage > max_leakage)
3080                                 max_leakage = smc_leakage;
3081
3082                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3083                 }
3084         }
3085
3086         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3087                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3088                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3089         }
3090         return 0;
3091 }
3092
3093 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3094                                             PP_NIslands_CACTABLES *cac_tables)
3095 {
3096         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3097         struct radeon_cac_leakage_table *leakage_table =
3098                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3099         u32 i, j, table_size;
3100         u32 smc_leakage, max_leakage = 0;
3101         u32 scaling_factor;
3102
3103         if (!leakage_table)
3104                 return -EINVAL;
3105
3106         table_size = leakage_table->count;
3107
3108         if (eg_pi->vddc_voltage_table.count != table_size)
3109                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3110                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3111
3112         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3113                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3114
3115         if (table_size == 0)
3116                 return -EINVAL;
3117
3118         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3119
3120         for (j = 0; j < table_size; j++) {
3121                 smc_leakage = leakage_table->entries[j].leakage;
3122
3123                 if (smc_leakage > max_leakage)
3124                         max_leakage = smc_leakage;
3125
3126                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3127                         cac_tables->cac_lkge_lut[i][j] =
3128                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3129         }
3130
3131         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3132                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3133                         cac_tables->cac_lkge_lut[i][j] =
3134                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3135         }
3136         return 0;
3137 }
3138
3139 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3140 {
3141         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3142         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3143         PP_NIslands_CACTABLES *cac_tables = NULL;
3144         int i, ret;
3145         u32 reg;
3146
3147         if (ni_pi->enable_cac == false)
3148                 return 0;
3149
3150         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3151         if (!cac_tables)
3152                 return -ENOMEM;
3153
3154         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3155         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3156                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3157         WREG32(CG_CAC_CTRL, reg);
3158
3159         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3160                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3161
3162         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3163                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3164
3165         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3166         ni_pi->cac_data.pwr_const = 0;
3167         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3168         ni_pi->cac_data.bif_cac_value = 0;
3169         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3170         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3171         ni_pi->cac_data.allow_ovrflw = 0;
3172         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3173         ni_pi->cac_data.num_win_tdp = 0;
3174         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3175
3176         if (ni_pi->driver_calculate_cac_leakage)
3177                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3178         else
3179                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3180
3181         if (ret)
3182                 goto done_free;
3183
3184         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3185         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3186         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3187         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3188         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3189         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3190         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3191         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3192         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3193
3194         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3195                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3196
3197 done_free:
3198         if (ret) {
3199                 ni_pi->enable_cac = false;
3200                 ni_pi->enable_power_containment = false;
3201         }
3202
3203         kfree(cac_tables);
3204
3205         return 0;
3206 }
3207
3208 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3209 {
3210         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3211         u32 reg;
3212
3213         if (!ni_pi->enable_cac ||
3214             !ni_pi->cac_configuration_required)
3215                 return 0;
3216
3217         if (ni_pi->cac_weights == NULL)
3218                 return -EINVAL;
3219
3220         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3221                                                       WEIGHT_TCP_SIG1_MASK |
3222                                                       WEIGHT_TA_SIG_MASK);
3223         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3224                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3225                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3226         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3227
3228         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3229                                                       WEIGHT_TCC_EN1_MASK |
3230                                                       WEIGHT_TCC_EN2_MASK);
3231         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3232                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3233                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3234         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3235
3236         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3237                                                       WEIGHT_CB_EN1_MASK |
3238                                                       WEIGHT_CB_EN2_MASK |
3239                                                       WEIGHT_CB_EN3_MASK);
3240         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3241                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3242                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3243                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3244         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3245
3246         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3247                                                       WEIGHT_DB_SIG1_MASK |
3248                                                       WEIGHT_DB_SIG2_MASK |
3249                                                       WEIGHT_DB_SIG3_MASK);
3250         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3251                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3252                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3253                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3254         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3255
3256         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3257                                                       WEIGHT_SXM_SIG1_MASK |
3258                                                       WEIGHT_SXM_SIG2_MASK |
3259                                                       WEIGHT_SXS_SIG0_MASK |
3260                                                       WEIGHT_SXS_SIG1_MASK);
3261         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3262                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3263                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3264                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3265                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3266         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3267
3268         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3269                                                       WEIGHT_XBR_1_MASK |
3270                                                       WEIGHT_XBR_2_MASK |
3271                                                       WEIGHT_SPI_SIG0_MASK);
3272         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3273                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3274                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3275                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3276         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3277
3278         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3279                                                       WEIGHT_SPI_SIG2_MASK |
3280                                                       WEIGHT_SPI_SIG3_MASK |
3281                                                       WEIGHT_SPI_SIG4_MASK |
3282                                                       WEIGHT_SPI_SIG5_MASK);
3283         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3284                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3285                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3286                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3287                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3288         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3289
3290         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3291                                                       WEIGHT_LDS_SIG1_MASK |
3292                                                       WEIGHT_SC_MASK);
3293         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3294                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3295                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3296         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3297
3298         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3299                                                       WEIGHT_CP_MASK |
3300                                                       WEIGHT_PA_SIG0_MASK |
3301                                                       WEIGHT_PA_SIG1_MASK |
3302                                                       WEIGHT_VGT_SIG0_MASK);
3303         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3304                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3305                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3306                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3307                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3308         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3309
3310         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3311                                                       WEIGHT_VGT_SIG2_MASK |
3312                                                       WEIGHT_DC_SIG0_MASK |
3313                                                       WEIGHT_DC_SIG1_MASK |
3314                                                       WEIGHT_DC_SIG2_MASK);
3315         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3316                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3317                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3318                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3319                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3320         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3321
3322         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3323                                                       WEIGHT_UVD_SIG0_MASK |
3324                                                       WEIGHT_UVD_SIG1_MASK |
3325                                                       WEIGHT_SPARE0_MASK |
3326                                                       WEIGHT_SPARE1_MASK);
3327         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3328                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3329                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3330                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3331                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3332         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3333
3334         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3335                                                       WEIGHT_SQ_VSP0_MASK);
3336         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3337                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3338         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3339
3340         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3341         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3342         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3343
3344         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3345                                                         OVR_VAL_SPARE_0_MASK |
3346                                                         OVR_MODE_SPARE_1_MASK |
3347                                                         OVR_VAL_SPARE_1_MASK);
3348         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3349                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3350                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3351                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3352         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3353
3354         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3355                                            VSP0_MASK |
3356                                            GPR_MASK);
3357         reg |= (VSP(ni_pi->cac_weights->vsp) |
3358                 VSP0(ni_pi->cac_weights->vsp0) |
3359                 GPR(ni_pi->cac_weights->gpr));
3360         WREG32(SQ_CAC_THRESHOLD, reg);
3361
3362         reg = (MCDW_WR_ENABLE |
3363                MCDX_WR_ENABLE |
3364                MCDY_WR_ENABLE |
3365                MCDZ_WR_ENABLE |
3366                INDEX(0x09D4));
3367         WREG32(MC_CG_CONFIG, reg);
3368
3369         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3370                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3371                ALLOW_OVERFLOW);
3372         WREG32(MC_CG_DATAPORT, reg);
3373
3374         return 0;
3375 }
3376
3377 static int ni_enable_smc_cac(struct radeon_device *rdev,
3378                              struct radeon_ps *radeon_new_state,
3379                              bool enable)
3380 {
3381         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3382         int ret = 0;
3383         PPSMC_Result smc_result;
3384
3385         if (ni_pi->enable_cac) {
3386                 if (enable) {
3387                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3388                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3389
3390                                 if (ni_pi->support_cac_long_term_average) {
3391                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3392                                         if (PPSMC_Result_OK != smc_result)
3393                                                 ni_pi->support_cac_long_term_average = false;
3394                                 }
3395
3396                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3397                                 if (PPSMC_Result_OK != smc_result)
3398                                         ret = -EINVAL;
3399
3400                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3401                         }
3402                 } else if (ni_pi->cac_enabled) {
3403                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3404
3405                         ni_pi->cac_enabled = false;
3406
3407                         if (ni_pi->support_cac_long_term_average) {
3408                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3409                                 if (PPSMC_Result_OK != smc_result)
3410                                         ni_pi->support_cac_long_term_average = false;
3411                         }
3412                 }
3413         }
3414
3415         return ret;
3416 }
3417
3418 static int ni_pcie_performance_request(struct radeon_device *rdev,
3419                                        u8 perf_req, bool advertise)
3420 {
3421 #if defined(CONFIG_ACPI)
3422         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3423
3424         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3425             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3426                 if (eg_pi->pcie_performance_request_registered == false)
3427                         radeon_acpi_pcie_notify_device_ready(rdev);
3428                 eg_pi->pcie_performance_request_registered = true;
3429                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3430         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3431                     eg_pi->pcie_performance_request_registered) {
3432                 eg_pi->pcie_performance_request_registered = false;
3433                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3434         }
3435 #endif
3436         return 0;
3437 }
3438
3439 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3440 {
3441         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3442         u32 tmp;
3443
3444         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3445
3446         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3447             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3448                 pi->pcie_gen2 = true;
3449         else
3450                 pi->pcie_gen2 = false;
3451
3452         if (!pi->pcie_gen2)
3453                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3454
3455         return 0;
3456 }
3457
3458 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3459                                             bool enable)
3460 {
3461         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3462         u32 tmp, bif;
3463
3464         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3465
3466         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3467             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3468                 if (enable) {
3469                         if (!pi->boot_in_gen2) {
3470                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3471                                 bif |= CG_CLIENT_REQ(0xd);
3472                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3473                         }
3474                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3475                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3476                         tmp |= LC_GEN2_EN_STRAP;
3477
3478                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3479                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3480                         udelay(10);
3481                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3482                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3483                 } else {
3484                         if (!pi->boot_in_gen2) {
3485                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3486                                 bif |= CG_CLIENT_REQ(0xd);
3487                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3488
3489                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3490                                 tmp &= ~LC_GEN2_EN_STRAP;
3491                         }
3492                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3493                 }
3494         }
3495 }
3496
3497 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3498                                         bool enable)
3499 {
3500         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3501
3502         if (enable)
3503                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3504         else
3505                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3506 }
3507
3508 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3509                                            struct radeon_ps *new_ps,
3510                                            struct radeon_ps *old_ps)
3511 {
3512         struct ni_ps *new_state = ni_get_ps(new_ps);
3513         struct ni_ps *current_state = ni_get_ps(old_ps);
3514
3515         if ((new_ps->vclk == old_ps->vclk) &&
3516             (new_ps->dclk == old_ps->dclk))
3517                 return;
3518
3519         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3520             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3521                 return;
3522
3523         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3524 }
3525
3526 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3527                                           struct radeon_ps *new_ps,
3528                                           struct radeon_ps *old_ps)
3529 {
3530         struct ni_ps *new_state = ni_get_ps(new_ps);
3531         struct ni_ps *current_state = ni_get_ps(old_ps);
3532
3533         if ((new_ps->vclk == old_ps->vclk) &&
3534             (new_ps->dclk == old_ps->dclk))
3535                 return;
3536
3537         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3538             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3539                 return;
3540
3541         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3542 }
3543
3544 void ni_dpm_setup_asic(struct radeon_device *rdev)
3545 {
3546         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3547         int r;
3548
3549         r = ni_mc_load_microcode(rdev);
3550         if (r)
3551                 DRM_ERROR("Failed to load MC firmware!\n");
3552         ni_read_clock_registers(rdev);
3553         btc_read_arb_registers(rdev);
3554         rv770_get_memory_type(rdev);
3555         if (eg_pi->pcie_performance_request)
3556                 ni_advertise_gen2_capability(rdev);
3557         rv770_get_pcie_gen2_status(rdev);
3558         rv770_enable_acpi_pm(rdev);
3559 }
3560
3561 void ni_update_current_ps(struct radeon_device *rdev,
3562                           struct radeon_ps *rps)
3563 {
3564         struct ni_ps *new_ps = ni_get_ps(rps);
3565         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3566         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3567
3568         eg_pi->current_rps = *rps;
3569         ni_pi->current_ps = *new_ps;
3570         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3571 }
3572
3573 void ni_update_requested_ps(struct radeon_device *rdev,
3574                             struct radeon_ps *rps)
3575 {
3576         struct ni_ps *new_ps = ni_get_ps(rps);
3577         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3578         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3579
3580         eg_pi->requested_rps = *rps;
3581         ni_pi->requested_ps = *new_ps;
3582         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3583 }
3584
3585 int ni_dpm_enable(struct radeon_device *rdev)
3586 {
3587         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3588         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3589         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3590         int ret;
3591
3592         if (pi->gfx_clock_gating)
3593                 ni_cg_clockgating_default(rdev);
3594         if (btc_dpm_enabled(rdev))
3595                 return -EINVAL;
3596         if (pi->mg_clock_gating)
3597                 ni_mg_clockgating_default(rdev);
3598         if (eg_pi->ls_clock_gating)
3599                 ni_ls_clockgating_default(rdev);
3600         if (pi->voltage_control) {
3601                 rv770_enable_voltage_control(rdev, true);
3602                 ret = cypress_construct_voltage_tables(rdev);
3603                 if (ret) {
3604                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3605                         return ret;
3606                 }
3607         }
3608         if (eg_pi->dynamic_ac_timing) {
3609                 ret = ni_initialize_mc_reg_table(rdev);
3610                 if (ret)
3611                         eg_pi->dynamic_ac_timing = false;
3612         }
3613         if (pi->dynamic_ss)
3614                 cypress_enable_spread_spectrum(rdev, true);
3615         if (pi->thermal_protection)
3616                 rv770_enable_thermal_protection(rdev, true);
3617         rv770_setup_bsp(rdev);
3618         rv770_program_git(rdev);
3619         rv770_program_tp(rdev);
3620         rv770_program_tpp(rdev);
3621         rv770_program_sstp(rdev);
3622         cypress_enable_display_gap(rdev);
3623         rv770_program_vc(rdev);
3624         if (pi->dynamic_pcie_gen2)
3625                 ni_enable_dynamic_pcie_gen2(rdev, true);
3626         ret = rv770_upload_firmware(rdev);
3627         if (ret) {
3628                 DRM_ERROR("rv770_upload_firmware failed\n");
3629                 return ret;
3630         }
3631         ret = ni_process_firmware_header(rdev);
3632         if (ret) {
3633                 DRM_ERROR("ni_process_firmware_header failed\n");
3634                 return ret;
3635         }
3636         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3637         if (ret) {
3638                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3639                 return ret;
3640         }
3641         ret = ni_init_smc_table(rdev);
3642         if (ret) {
3643                 DRM_ERROR("ni_init_smc_table failed\n");
3644                 return ret;
3645         }
3646         ret = ni_init_smc_spll_table(rdev);
3647         if (ret) {
3648                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3649                 return ret;
3650         }
3651         ret = ni_init_arb_table_index(rdev);
3652         if (ret) {
3653                 DRM_ERROR("ni_init_arb_table_index failed\n");
3654                 return ret;
3655         }
3656         if (eg_pi->dynamic_ac_timing) {
3657                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3658                 if (ret) {
3659                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3660                         return ret;
3661                 }
3662         }
3663         ret = ni_initialize_smc_cac_tables(rdev);
3664         if (ret) {
3665                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3666                 return ret;
3667         }
3668         ret = ni_initialize_hardware_cac_manager(rdev);
3669         if (ret) {
3670                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3671                 return ret;
3672         }
3673         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3674         if (ret) {
3675                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3676                 return ret;
3677         }
3678         ni_program_response_times(rdev);
3679         r7xx_start_smc(rdev);
3680         ret = cypress_notify_smc_display_change(rdev, false);
3681         if (ret) {
3682                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3683                 return ret;
3684         }
3685         cypress_enable_sclk_control(rdev, true);
3686         if (eg_pi->memory_transition)
3687                 cypress_enable_mclk_control(rdev, true);
3688         cypress_start_dpm(rdev);
3689         if (pi->gfx_clock_gating)
3690                 ni_gfx_clockgating_enable(rdev, true);
3691         if (pi->mg_clock_gating)
3692                 ni_mg_clockgating_enable(rdev, true);
3693         if (eg_pi->ls_clock_gating)
3694                 ni_ls_clockgating_enable(rdev, true);
3695
3696         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3697
3698         ni_update_current_ps(rdev, boot_ps);
3699
3700         return 0;
3701 }
3702
3703 void ni_dpm_disable(struct radeon_device *rdev)
3704 {
3705         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3706         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3707         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3708
3709         if (!btc_dpm_enabled(rdev))
3710                 return;
3711         rv770_clear_vc(rdev);
3712         if (pi->thermal_protection)
3713                 rv770_enable_thermal_protection(rdev, false);
3714         ni_enable_power_containment(rdev, boot_ps, false);
3715         ni_enable_smc_cac(rdev, boot_ps, false);
3716         cypress_enable_spread_spectrum(rdev, false);
3717         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3718         if (pi->dynamic_pcie_gen2)
3719                 ni_enable_dynamic_pcie_gen2(rdev, false);
3720
3721         if (rdev->irq.installed &&
3722             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3723                 rdev->irq.dpm_thermal = false;
3724                 radeon_irq_set(rdev);
3725         }
3726
3727         if (pi->gfx_clock_gating)
3728                 ni_gfx_clockgating_enable(rdev, false);
3729         if (pi->mg_clock_gating)
3730                 ni_mg_clockgating_enable(rdev, false);
3731         if (eg_pi->ls_clock_gating)
3732                 ni_ls_clockgating_enable(rdev, false);
3733         ni_stop_dpm(rdev);
3734         btc_reset_to_default(rdev);
3735         ni_stop_smc(rdev);
3736         ni_force_switch_to_arb_f0(rdev);
3737
3738         ni_update_current_ps(rdev, boot_ps);
3739 }
3740
3741 static int ni_power_control_set_level(struct radeon_device *rdev)
3742 {
3743         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3744         int ret;
3745
3746         ret = ni_restrict_performance_levels_before_switch(rdev);
3747         if (ret)
3748                 return ret;
3749         ret = rv770_halt_smc(rdev);
3750         if (ret)
3751                 return ret;
3752         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3753         if (ret)
3754                 return ret;
3755         ret = rv770_resume_smc(rdev);
3756         if (ret)
3757                 return ret;
3758         ret = rv770_set_sw_state(rdev);
3759         if (ret)
3760                 return ret;
3761
3762         return 0;
3763 }
3764
3765 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3766 {
3767         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3768         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3769         struct radeon_ps *new_ps = &requested_ps;
3770
3771         ni_update_requested_ps(rdev, new_ps);
3772
3773         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3774
3775         return 0;
3776 }
3777
3778 int ni_dpm_set_power_state(struct radeon_device *rdev)
3779 {
3780         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3781         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3782         struct radeon_ps *old_ps = &eg_pi->current_rps;
3783         int ret;
3784
3785         ret = ni_restrict_performance_levels_before_switch(rdev);
3786         if (ret) {
3787                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3788                 return ret;
3789         }
3790         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3791         ret = ni_enable_power_containment(rdev, new_ps, false);
3792         if (ret) {
3793                 DRM_ERROR("ni_enable_power_containment failed\n");
3794                 return ret;
3795         }
3796         ret = ni_enable_smc_cac(rdev, new_ps, false);
3797         if (ret) {
3798                 DRM_ERROR("ni_enable_smc_cac failed\n");
3799                 return ret;
3800         }
3801         ret = rv770_halt_smc(rdev);
3802         if (ret) {
3803                 DRM_ERROR("rv770_halt_smc failed\n");
3804                 return ret;
3805         }
3806         if (eg_pi->smu_uvd_hs)
3807                 btc_notify_uvd_to_smc(rdev, new_ps);
3808         ret = ni_upload_sw_state(rdev, new_ps);
3809         if (ret) {
3810                 DRM_ERROR("ni_upload_sw_state failed\n");
3811                 return ret;
3812         }
3813         if (eg_pi->dynamic_ac_timing) {
3814                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3815                 if (ret) {
3816                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3817                         return ret;
3818                 }
3819         }
3820         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3821         if (ret) {
3822                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3823                 return ret;
3824         }
3825         ret = rv770_resume_smc(rdev);
3826         if (ret) {
3827                 DRM_ERROR("rv770_resume_smc failed\n");
3828                 return ret;
3829         }
3830         ret = rv770_set_sw_state(rdev);
3831         if (ret) {
3832                 DRM_ERROR("rv770_set_sw_state failed\n");
3833                 return ret;
3834         }
3835         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3836         ret = ni_enable_smc_cac(rdev, new_ps, true);
3837         if (ret) {
3838                 DRM_ERROR("ni_enable_smc_cac failed\n");
3839                 return ret;
3840         }
3841         ret = ni_enable_power_containment(rdev, new_ps, true);
3842         if (ret) {
3843                 DRM_ERROR("ni_enable_power_containment failed\n");
3844                 return ret;
3845         }
3846
3847         /* update tdp */
3848         ret = ni_power_control_set_level(rdev);
3849         if (ret) {
3850                 DRM_ERROR("ni_power_control_set_level failed\n");
3851                 return ret;
3852         }
3853
3854         return 0;
3855 }
3856
3857 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3858 {
3859         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3860         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3861
3862         ni_update_current_ps(rdev, new_ps);
3863 }
3864
3865 #if 0
3866 void ni_dpm_reset_asic(struct radeon_device *rdev)
3867 {
3868         ni_restrict_performance_levels_before_switch(rdev);
3869         rv770_set_boot_state(rdev);
3870 }
3871 #endif
3872
3873 union power_info {
3874         struct _ATOM_POWERPLAY_INFO info;
3875         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3876         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3877         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3878         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3879         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3880 };
3881
3882 union pplib_clock_info {
3883         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3884         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3885         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3886         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3887 };
3888
3889 union pplib_power_state {
3890         struct _ATOM_PPLIB_STATE v1;
3891         struct _ATOM_PPLIB_STATE_V2 v2;
3892 };
3893
3894 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3895                                           struct radeon_ps *rps,
3896                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3897                                           u8 table_rev)
3898 {
3899         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3900         rps->class = le16_to_cpu(non_clock_info->usClassification);
3901         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3902
3903         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3904                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3905                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3906         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3907                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3908                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3909         } else {
3910                 rps->vclk = 0;
3911                 rps->dclk = 0;
3912         }
3913
3914         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3915                 rdev->pm.dpm.boot_ps = rps;
3916         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3917                 rdev->pm.dpm.uvd_ps = rps;
3918 }
3919
3920 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3921                                       struct radeon_ps *rps, int index,
3922                                       union pplib_clock_info *clock_info)
3923 {
3924         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3925         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3926         struct ni_ps *ps = ni_get_ps(rps);
3927         struct rv7xx_pl *pl = &ps->performance_levels[index];
3928
3929         ps->performance_level_count = index + 1;
3930
3931         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3932         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3933         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3934         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3935
3936         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3937         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3938         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3939
3940         /* patch up vddc if necessary */
3941         if (pl->vddc == 0xff01) {
3942                 if (pi->max_vddc)
3943                         pl->vddc = pi->max_vddc;
3944         }
3945
3946         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3947                 pi->acpi_vddc = pl->vddc;
3948                 eg_pi->acpi_vddci = pl->vddci;
3949                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3950                         pi->acpi_pcie_gen2 = true;
3951                 else
3952                         pi->acpi_pcie_gen2 = false;
3953         }
3954
3955         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3956                 eg_pi->ulv.supported = true;
3957                 eg_pi->ulv.pl = pl;
3958         }
3959
3960         if (pi->min_vddc_in_table > pl->vddc)
3961                 pi->min_vddc_in_table = pl->vddc;
3962
3963         if (pi->max_vddc_in_table < pl->vddc)
3964                 pi->max_vddc_in_table = pl->vddc;
3965
3966         /* patch up boot state */
3967         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3968                 u16 vddc, vddci, mvdd;
3969                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3970                 pl->mclk = rdev->clock.default_mclk;
3971                 pl->sclk = rdev->clock.default_sclk;
3972                 pl->vddc = vddc;
3973                 pl->vddci = vddci;
3974         }
3975
3976         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3977             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3978                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3979                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3980                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3981                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3982         }
3983 }
3984
3985 static int ni_parse_power_table(struct radeon_device *rdev)
3986 {
3987         struct radeon_mode_info *mode_info = &rdev->mode_info;
3988         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3989         union pplib_power_state *power_state;
3990         int i, j;
3991         union pplib_clock_info *clock_info;
3992         union power_info *power_info;
3993         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3994         u16 data_offset;
3995         u8 frev, crev;
3996         struct ni_ps *ps;
3997
3998         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
3999                                    &frev, &crev, &data_offset))
4000                 return -EINVAL;
4001         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4002
4003         rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4004                                   sizeof(struct radeon_ps),
4005                                   GFP_KERNEL);
4006         if (!rdev->pm.dpm.ps)
4007                 return -ENOMEM;
4008
4009         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4010                 power_state = (union pplib_power_state *)
4011                         (mode_info->atom_context->bios + data_offset +
4012                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4013                          i * power_info->pplib.ucStateEntrySize);
4014                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4015                         (mode_info->atom_context->bios + data_offset +
4016                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4017                          (power_state->v1.ucNonClockStateIndex *
4018                           power_info->pplib.ucNonClockSize));
4019                 if (power_info->pplib.ucStateEntrySize - 1) {
4020                         u8 *idx;
4021                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4022                         if (ps == NULL) {
4023                                 kfree(rdev->pm.dpm.ps);
4024                                 return -ENOMEM;
4025                         }
4026                         rdev->pm.dpm.ps[i].ps_priv = ps;
4027                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4028                                                          non_clock_info,
4029                                                          power_info->pplib.ucNonClockSize);
4030                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4031                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4032                                 clock_info = (union pplib_clock_info *)
4033                                         (mode_info->atom_context->bios + data_offset +
4034                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4035                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4036                                 ni_parse_pplib_clock_info(rdev,
4037                                                           &rdev->pm.dpm.ps[i], j,
4038                                                           clock_info);
4039                         }
4040                 }
4041         }
4042         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4043         return 0;
4044 }
4045
4046 int ni_dpm_init(struct radeon_device *rdev)
4047 {
4048         struct rv7xx_power_info *pi;
4049         struct evergreen_power_info *eg_pi;
4050         struct ni_power_info *ni_pi;
4051         struct atom_clock_dividers dividers;
4052         int ret;
4053
4054         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4055         if (ni_pi == NULL)
4056                 return -ENOMEM;
4057         rdev->pm.dpm.priv = ni_pi;
4058         eg_pi = &ni_pi->eg;
4059         pi = &eg_pi->rv7xx;
4060
4061         rv770_get_max_vddc(rdev);
4062
4063         eg_pi->ulv.supported = false;
4064         pi->acpi_vddc = 0;
4065         eg_pi->acpi_vddci = 0;
4066         pi->min_vddc_in_table = 0;
4067         pi->max_vddc_in_table = 0;
4068
4069         ret = r600_get_platform_caps(rdev);
4070         if (ret)
4071                 return ret;
4072
4073         ret = ni_parse_power_table(rdev);
4074         if (ret)
4075                 return ret;
4076         ret = r600_parse_extended_power_table(rdev);
4077         if (ret)
4078                 return ret;
4079
4080         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4081                 kcalloc(4,
4082                         sizeof(struct radeon_clock_voltage_dependency_entry),
4083                         GFP_KERNEL);
4084         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4085                 r600_free_extended_power_table(rdev);
4086                 return -ENOMEM;
4087         }
4088         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4089         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4090         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4091         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4092         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4093         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4094         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4095         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4096         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4097
4098         ni_patch_dependency_tables_based_on_leakage(rdev);
4099
4100         if (rdev->pm.dpm.voltage_response_time == 0)
4101                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4102         if (rdev->pm.dpm.backbias_response_time == 0)
4103                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4104
4105         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4106                                              0, false, &dividers);
4107         if (ret)
4108                 pi->ref_div = dividers.ref_div + 1;
4109         else
4110                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4111
4112         pi->rlp = RV770_RLP_DFLT;
4113         pi->rmp = RV770_RMP_DFLT;
4114         pi->lhp = RV770_LHP_DFLT;
4115         pi->lmp = RV770_LMP_DFLT;
4116
4117         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4118         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4119         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4120         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4121
4122         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4123         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4124         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4125         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4126
4127         eg_pi->smu_uvd_hs = true;
4128
4129         if (rdev->pdev->device == 0x6707) {
4130                 pi->mclk_strobe_mode_threshold = 55000;
4131                 pi->mclk_edc_enable_threshold = 55000;
4132                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4133         } else {
4134                 pi->mclk_strobe_mode_threshold = 40000;
4135                 pi->mclk_edc_enable_threshold = 40000;
4136                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4137         }
4138         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4139
4140         pi->voltage_control =
4141                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4142
4143         pi->mvdd_control =
4144                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4145
4146         eg_pi->vddci_control =
4147                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4148
4149         rv770_get_engine_memory_ss(rdev);
4150
4151         pi->asi = RV770_ASI_DFLT;
4152         pi->pasi = CYPRESS_HASI_DFLT;
4153         pi->vrc = CYPRESS_VRC_DFLT;
4154
4155         pi->power_gating = false;
4156
4157         pi->gfx_clock_gating = true;
4158
4159         pi->mg_clock_gating = true;
4160         pi->mgcgtssm = true;
4161         eg_pi->ls_clock_gating = false;
4162         eg_pi->sclk_deep_sleep = false;
4163
4164         pi->dynamic_pcie_gen2 = true;
4165
4166         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4167                 pi->thermal_protection = true;
4168         else
4169                 pi->thermal_protection = false;
4170
4171         pi->display_gap = true;
4172
4173         pi->dcodt = true;
4174
4175         pi->ulps = true;
4176
4177         eg_pi->dynamic_ac_timing = true;
4178         eg_pi->abm = true;
4179         eg_pi->mcls = true;
4180         eg_pi->light_sleep = true;
4181         eg_pi->memory_transition = true;
4182 #if defined(CONFIG_ACPI)
4183         eg_pi->pcie_performance_request =
4184                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4185 #else
4186         eg_pi->pcie_performance_request = false;
4187 #endif
4188
4189         eg_pi->dll_default_on = false;
4190
4191         eg_pi->sclk_deep_sleep = false;
4192
4193         pi->mclk_stutter_mode_threshold = 0;
4194
4195         pi->sram_end = SMC_RAM_END;
4196
4197         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4198         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4199         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4200         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4201         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4202         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4203         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4204         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4205
4206         ni_pi->cac_data.leakage_coefficients.at = 516;
4207         ni_pi->cac_data.leakage_coefficients.bt = 18;
4208         ni_pi->cac_data.leakage_coefficients.av = 51;
4209         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4210
4211         switch (rdev->pdev->device) {
4212         case 0x6700:
4213         case 0x6701:
4214         case 0x6702:
4215         case 0x6703:
4216         case 0x6718:
4217                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4218                 break;
4219         case 0x6705:
4220         case 0x6719:
4221         case 0x671D:
4222         case 0x671C:
4223         default:
4224                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4225                 break;
4226         case 0x6704:
4227         case 0x6706:
4228         case 0x6707:
4229         case 0x6708:
4230         case 0x6709:
4231                 ni_pi->cac_weights = &cac_weights_cayman_le;
4232                 break;
4233         }
4234
4235         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4236                 ni_pi->enable_power_containment = true;
4237                 ni_pi->enable_cac = true;
4238                 ni_pi->enable_sq_ramping = true;
4239         } else {
4240                 ni_pi->enable_power_containment = false;
4241                 ni_pi->enable_cac = false;
4242                 ni_pi->enable_sq_ramping = false;
4243         }
4244
4245         ni_pi->driver_calculate_cac_leakage = false;
4246         ni_pi->cac_configuration_required = true;
4247
4248         if (ni_pi->cac_configuration_required) {
4249                 ni_pi->support_cac_long_term_average = true;
4250                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4251                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4252         } else {
4253                 ni_pi->support_cac_long_term_average = false;
4254                 ni_pi->lta_window_size = 0;
4255                 ni_pi->lts_truncate = 0;
4256         }
4257
4258         ni_pi->use_power_boost_limit = true;
4259
4260         /* make sure dc limits are valid */
4261         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4262             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4263                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4264                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4265
4266         return 0;
4267 }
4268
4269 void ni_dpm_fini(struct radeon_device *rdev)
4270 {
4271         int i;
4272
4273         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4274                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4275         }
4276         kfree(rdev->pm.dpm.ps);
4277         kfree(rdev->pm.dpm.priv);
4278         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4279         r600_free_extended_power_table(rdev);
4280 }
4281
4282 void ni_dpm_print_power_state(struct radeon_device *rdev,
4283                               struct radeon_ps *rps)
4284 {
4285         struct ni_ps *ps = ni_get_ps(rps);
4286         struct rv7xx_pl *pl;
4287         int i;
4288
4289         r600_dpm_print_class_info(rps->class, rps->class2);
4290         r600_dpm_print_cap_info(rps->caps);
4291         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4292         for (i = 0; i < ps->performance_level_count; i++) {
4293                 pl = &ps->performance_levels[i];
4294                 if (rdev->family >= CHIP_TAHITI)
4295                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4296                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4297                 else
4298                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4299                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4300         }
4301         r600_dpm_print_ps_status(rdev, rps);
4302 }
4303
4304 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4305                                                     struct seq_file *m)
4306 {
4307         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4308         struct radeon_ps *rps = &eg_pi->current_rps;
4309         struct ni_ps *ps = ni_get_ps(rps);
4310         struct rv7xx_pl *pl;
4311         u32 current_index =
4312                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4313                 CURRENT_STATE_INDEX_SHIFT;
4314
4315         if (current_index >= ps->performance_level_count) {
4316                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4317         } else {
4318                 pl = &ps->performance_levels[current_index];
4319                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4320                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4321                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4322         }
4323 }
4324
4325 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4326 {
4327         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4328         struct radeon_ps *rps = &eg_pi->current_rps;
4329         struct ni_ps *ps = ni_get_ps(rps);
4330         struct rv7xx_pl *pl;
4331         u32 current_index =
4332                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4333                 CURRENT_STATE_INDEX_SHIFT;
4334
4335         if (current_index >= ps->performance_level_count) {
4336                 return 0;
4337         } else {
4338                 pl = &ps->performance_levels[current_index];
4339                 return pl->sclk;
4340         }
4341 }
4342
4343 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4344 {
4345         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4346         struct radeon_ps *rps = &eg_pi->current_rps;
4347         struct ni_ps *ps = ni_get_ps(rps);
4348         struct rv7xx_pl *pl;
4349         u32 current_index =
4350                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4351                 CURRENT_STATE_INDEX_SHIFT;
4352
4353         if (current_index >= ps->performance_level_count) {
4354                 return 0;
4355         } else {
4356                 pl = &ps->performance_levels[current_index];
4357                 return pl->mclk;
4358         }
4359 }
4360
4361 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4362 {
4363         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4364         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4365
4366         if (low)
4367                 return requested_state->performance_levels[0].sclk;
4368         else
4369                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4370 }
4371
4372 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4373 {
4374         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4375         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4376
4377         if (low)
4378                 return requested_state->performance_levels[0].mclk;
4379         else
4380                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4381 }
4382