drivers/gpu/drm/radeon/ni_dpm.c

   1 /*
   2  * Copyright 2012 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/math64.h>
  25 #include <linux/pci.h>
  26 #include <linux/seq_file.h>
  27
  28 #include "atom.h"
  29 #include "evergreen.h"
  30 #include "ni_dpm.h"
  31 #include "nid.h"
  32 #include "r600_dpm.h"
  33 #include "rv770.h"
  34 #include "radeon.h"
  35 #include "radeon_asic.h"
  36
  37 #define MC_CG_ARB_FREQ_F0           0x0a
  38 #define MC_CG_ARB_FREQ_F1           0x0b
  39 #define MC_CG_ARB_FREQ_F2           0x0c
  40 #define MC_CG_ARB_FREQ_F3           0x0d
  41
  42 #define SMC_RAM_END 0xC000
  43
  44 static const struct ni_cac_weights cac_weights_cayman_xt =
  45 {
  46         0x15,
  47         0x2,
  48         0x19,
  49         0x2,
  50         0x8,
  51         0x14,
  52         0x2,
  53         0x16,
  54         0xE,
  55         0x17,
  56         0x13,
  57         0x2B,
  58         0x10,
  59         0x7,
  60         0x5,
  61         0x5,
  62         0x5,
  63         0x2,
  64         0x3,
  65         0x9,
  66         0x10,
  67         0x10,
  68         0x2B,
  69         0xA,
  70         0x9,
  71         0x4,
  72         0xD,
  73         0xD,
  74         0x3E,
  75         0x18,
  76         0x14,
  77         0,
  78         0x3,
  79         0x3,
  80         0x5,
  81         0,
  82         0x2,
  83         0,
  84         0,
  85         0,
  86         0,
  87         0,
  88         0,
  89         0,
  90         0,
  91         0,
  92         0x1CC,
  93         0,
  94         0x164,
  95         1,
  96         1,
  97         1,
  98         1,
  99         12,
 100         12,
 101         12,
 102         0x12,
 103         0x1F,
 104         132,
 105         5,
 106         7,
 107         0,
 108         { 0, 0, 0, 0, 0, 0, 0, 0 },
 109         { 0, 0, 0, 0 },
 110         true
 111 };
 112
 113 static const struct ni_cac_weights cac_weights_cayman_pro =
 114 {
 115         0x16,
 116         0x4,
 117         0x10,
 118         0x2,
 119         0xA,
 120         0x16,
 121         0x2,
 122         0x18,
 123         0x10,
 124         0x1A,
 125         0x16,
 126         0x2D,
 127         0x12,
 128         0xA,
 129         0x6,
 130         0x6,
 131         0x6,
 132         0x2,
 133         0x4,
 134         0xB,
 135         0x11,
 136         0x11,
 137         0x2D,
 138         0xC,
 139         0xC,
 140         0x7,
 141         0x10,
 142         0x10,
 143         0x3F,
 144         0x1A,
 145         0x16,
 146         0,
 147         0x7,
 148         0x4,
 149         0x6,
 150         1,
 151         0x2,
 152         0x1,
 153         0,
 154         0,
 155         0,
 156         0,
 157         0,
 158         0,
 159         0x30,
 160         0,
 161         0x1CF,
 162         0,
 163         0x166,
 164         1,
 165         1,
 166         1,
 167         1,
 168         12,
 169         12,
 170         12,
 171         0x15,
 172         0x1F,
 173         132,
 174         6,
 175         6,
 176         0,
 177         { 0, 0, 0, 0, 0, 0, 0, 0 },
 178         { 0, 0, 0, 0 },
 179         true
 180 };
 181
 182 static const struct ni_cac_weights cac_weights_cayman_le =
 183 {
 184         0x7,
 185         0xE,
 186         0x1,
 187         0xA,
 188         0x1,
 189         0x3F,
 190         0x2,
 191         0x18,
 192         0x10,
 193         0x1A,
 194         0x1,
 195         0x3F,
 196         0x1,
 197         0xE,
 198         0x6,
 199         0x6,
 200         0x6,
 201         0x2,
 202         0x4,
 203         0x9,
 204         0x1A,
 205         0x1A,
 206         0x2C,
 207         0xA,
 208         0x11,
 209         0x8,
 210         0x19,
 211         0x19,
 212         0x1,
 213         0x1,
 214         0x1A,
 215         0,
 216         0x8,
 217         0x5,
 218         0x8,
 219         0x1,
 220         0x3,
 221         0x1,
 222         0,
 223         0,
 224         0,
 225         0,
 226         0,
 227         0,
 228         0x38,
 229         0x38,
 230         0x239,
 231         0x3,
 232         0x18A,
 233         1,
 234         1,
 235         1,
 236         1,
 237         12,
 238         12,
 239         12,
 240         0x15,
 241         0x22,
 242         132,
 243         6,
 244         6,
 245         0,
 246         { 0, 0, 0, 0, 0, 0, 0, 0 },
 247         { 0, 0, 0, 0 },
 248         true
 249 };
 250
 251 #define NISLANDS_MGCG_SEQUENCE  300
 252
 253 static const u32 cayman_cgcg_cgls_default[] =
 254 {
 255         0x000008f8, 0x00000010, 0xffffffff,
 256         0x000008fc, 0x00000000, 0xffffffff,
 257         0x000008f8, 0x00000011, 0xffffffff,
 258         0x000008fc, 0x00000000, 0xffffffff,
 259         0x000008f8, 0x00000012, 0xffffffff,
 260         0x000008fc, 0x00000000, 0xffffffff,
 261         0x000008f8, 0x00000013, 0xffffffff,
 262         0x000008fc, 0x00000000, 0xffffffff,
 263         0x000008f8, 0x00000014, 0xffffffff,
 264         0x000008fc, 0x00000000, 0xffffffff,
 265         0x000008f8, 0x00000015, 0xffffffff,
 266         0x000008fc, 0x00000000, 0xffffffff,
 267         0x000008f8, 0x00000016, 0xffffffff,
 268         0x000008fc, 0x00000000, 0xffffffff,
 269         0x000008f8, 0x00000017, 0xffffffff,
 270         0x000008fc, 0x00000000, 0xffffffff,
 271         0x000008f8, 0x00000018, 0xffffffff,
 272         0x000008fc, 0x00000000, 0xffffffff,
 273         0x000008f8, 0x00000019, 0xffffffff,
 274         0x000008fc, 0x00000000, 0xffffffff,
 275         0x000008f8, 0x0000001a, 0xffffffff,
 276         0x000008fc, 0x00000000, 0xffffffff,
 277         0x000008f8, 0x0000001b, 0xffffffff,
 278         0x000008fc, 0x00000000, 0xffffffff,
 279         0x000008f8, 0x00000020, 0xffffffff,
 280         0x000008fc, 0x00000000, 0xffffffff,
 281         0x000008f8, 0x00000021, 0xffffffff,
 282         0x000008fc, 0x00000000, 0xffffffff,
 283         0x000008f8, 0x00000022, 0xffffffff,
 284         0x000008fc, 0x00000000, 0xffffffff,
 285         0x000008f8, 0x00000023, 0xffffffff,
 286         0x000008fc, 0x00000000, 0xffffffff,
 287         0x000008f8, 0x00000024, 0xffffffff,
 288         0x000008fc, 0x00000000, 0xffffffff,
 289         0x000008f8, 0x00000025, 0xffffffff,
 290         0x000008fc, 0x00000000, 0xffffffff,
 291         0x000008f8, 0x00000026, 0xffffffff,
 292         0x000008fc, 0x00000000, 0xffffffff,
 293         0x000008f8, 0x00000027, 0xffffffff,
 294         0x000008fc, 0x00000000, 0xffffffff,
 295         0x000008f8, 0x00000028, 0xffffffff,
 296         0x000008fc, 0x00000000, 0xffffffff,
 297         0x000008f8, 0x00000029, 0xffffffff,
 298         0x000008fc, 0x00000000, 0xffffffff,
 299         0x000008f8, 0x0000002a, 0xffffffff,
 300         0x000008fc, 0x00000000, 0xffffffff,
 301         0x000008f8, 0x0000002b, 0xffffffff,
 302         0x000008fc, 0x00000000, 0xffffffff
 303 };
 304 #define CAYMAN_CGCG_CGLS_DEFAULT_LENGTH sizeof(cayman_cgcg_cgls_default) / (3 * sizeof(u32))
 305
 306 static const u32 cayman_cgcg_cgls_disable[] =
 307 {
 308         0x000008f8, 0x00000010, 0xffffffff,
 309         0x000008fc, 0xffffffff, 0xffffffff,
 310         0x000008f8, 0x00000011, 0xffffffff,
 311         0x000008fc, 0xffffffff, 0xffffffff,
 312         0x000008f8, 0x00000012, 0xffffffff,
 313         0x000008fc, 0xffffffff, 0xffffffff,
 314         0x000008f8, 0x00000013, 0xffffffff,
 315         0x000008fc, 0xffffffff, 0xffffffff,
 316         0x000008f8, 0x00000014, 0xffffffff,
 317         0x000008fc, 0xffffffff, 0xffffffff,
 318         0x000008f8, 0x00000015, 0xffffffff,
 319         0x000008fc, 0xffffffff, 0xffffffff,
 320         0x000008f8, 0x00000016, 0xffffffff,
 321         0x000008fc, 0xffffffff, 0xffffffff,
 322         0x000008f8, 0x00000017, 0xffffffff,
 323         0x000008fc, 0xffffffff, 0xffffffff,
 324         0x000008f8, 0x00000018, 0xffffffff,
 325         0x000008fc, 0xffffffff, 0xffffffff,
 326         0x000008f8, 0x00000019, 0xffffffff,
 327         0x000008fc, 0xffffffff, 0xffffffff,
 328         0x000008f8, 0x0000001a, 0xffffffff,
 329         0x000008fc, 0xffffffff, 0xffffffff,
 330         0x000008f8, 0x0000001b, 0xffffffff,
 331         0x000008fc, 0xffffffff, 0xffffffff,
 332         0x000008f8, 0x00000020, 0xffffffff,
 333         0x000008fc, 0x00000000, 0xffffffff,
 334         0x000008f8, 0x00000021, 0xffffffff,
 335         0x000008fc, 0x00000000, 0xffffffff,
 336         0x000008f8, 0x00000022, 0xffffffff,
 337         0x000008fc, 0x00000000, 0xffffffff,
 338         0x000008f8, 0x00000023, 0xffffffff,
 339         0x000008fc, 0x00000000, 0xffffffff,
 340         0x000008f8, 0x00000024, 0xffffffff,
 341         0x000008fc, 0x00000000, 0xffffffff,
 342         0x000008f8, 0x00000025, 0xffffffff,
 343         0x000008fc, 0x00000000, 0xffffffff,
 344         0x000008f8, 0x00000026, 0xffffffff,
 345         0x000008fc, 0x00000000, 0xffffffff,
 346         0x000008f8, 0x00000027, 0xffffffff,
 347         0x000008fc, 0x00000000, 0xffffffff,
 348         0x000008f8, 0x00000028, 0xffffffff,
 349         0x000008fc, 0x00000000, 0xffffffff,
 350         0x000008f8, 0x00000029, 0xffffffff,
 351         0x000008fc, 0x00000000, 0xffffffff,
 352         0x000008f8, 0x0000002a, 0xffffffff,
 353         0x000008fc, 0x00000000, 0xffffffff,
 354         0x000008f8, 0x0000002b, 0xffffffff,
 355         0x000008fc, 0x00000000, 0xffffffff,
 356         0x00000644, 0x000f7902, 0x001f4180,
 357         0x00000644, 0x000f3802, 0x001f4180
 358 };
 359 #define CAYMAN_CGCG_CGLS_DISABLE_LENGTH sizeof(cayman_cgcg_cgls_disable) / (3 * sizeof(u32))
 360
 361 static const u32 cayman_cgcg_cgls_enable[] =
 362 {
 363         0x00000644, 0x000f7882, 0x001f4080,
 364         0x000008f8, 0x00000010, 0xffffffff,
 365         0x000008fc, 0x00000000, 0xffffffff,
 366         0x000008f8, 0x00000011, 0xffffffff,
 367         0x000008fc, 0x00000000, 0xffffffff,
 368         0x000008f8, 0x00000012, 0xffffffff,
 369         0x000008fc, 0x00000000, 0xffffffff,
 370         0x000008f8, 0x00000013, 0xffffffff,
 371         0x000008fc, 0x00000000, 0xffffffff,
 372         0x000008f8, 0x00000014, 0xffffffff,
 373         0x000008fc, 0x00000000, 0xffffffff,
 374         0x000008f8, 0x00000015, 0xffffffff,
 375         0x000008fc, 0x00000000, 0xffffffff,
 376         0x000008f8, 0x00000016, 0xffffffff,
 377         0x000008fc, 0x00000000, 0xffffffff,
 378         0x000008f8, 0x00000017, 0xffffffff,
 379         0x000008fc, 0x00000000, 0xffffffff,
 380         0x000008f8, 0x00000018, 0xffffffff,
 381         0x000008fc, 0x00000000, 0xffffffff,
 382         0x000008f8, 0x00000019, 0xffffffff,
 383         0x000008fc, 0x00000000, 0xffffffff,
 384         0x000008f8, 0x0000001a, 0xffffffff,
 385         0x000008fc, 0x00000000, 0xffffffff,
 386         0x000008f8, 0x0000001b, 0xffffffff,
 387         0x000008fc, 0x00000000, 0xffffffff,
 388         0x000008f8, 0x00000020, 0xffffffff,
 389         0x000008fc, 0xffffffff, 0xffffffff,
 390         0x000008f8, 0x00000021, 0xffffffff,
 391         0x000008fc, 0xffffffff, 0xffffffff,
 392         0x000008f8, 0x00000022, 0xffffffff,
 393         0x000008fc, 0xffffffff, 0xffffffff,
 394         0x000008f8, 0x00000023, 0xffffffff,
 395         0x000008fc, 0xffffffff, 0xffffffff,
 396         0x000008f8, 0x00000024, 0xffffffff,
 397         0x000008fc, 0xffffffff, 0xffffffff,
 398         0x000008f8, 0x00000025, 0xffffffff,
 399         0x000008fc, 0xffffffff, 0xffffffff,
 400         0x000008f8, 0x00000026, 0xffffffff,
 401         0x000008fc, 0xffffffff, 0xffffffff,
 402         0x000008f8, 0x00000027, 0xffffffff,
 403         0x000008fc, 0xffffffff, 0xffffffff,
 404         0x000008f8, 0x00000028, 0xffffffff,
 405         0x000008fc, 0xffffffff, 0xffffffff,
 406         0x000008f8, 0x00000029, 0xffffffff,
 407         0x000008fc, 0xffffffff, 0xffffffff,
 408         0x000008f8, 0x0000002a, 0xffffffff,
 409         0x000008fc, 0xffffffff, 0xffffffff,
 410         0x000008f8, 0x0000002b, 0xffffffff,
 411         0x000008fc, 0xffffffff, 0xffffffff
 412 };
 413 #define CAYMAN_CGCG_CGLS_ENABLE_LENGTH  sizeof(cayman_cgcg_cgls_enable) / (3 * sizeof(u32))
 414
 415 static const u32 cayman_mgcg_default[] =
 416 {
 417         0x0000802c, 0xc0000000, 0xffffffff,
 418         0x00003fc4, 0xc0000000, 0xffffffff,
 419         0x00005448, 0x00000100, 0xffffffff,
 420         0x000055e4, 0x00000100, 0xffffffff,
 421         0x0000160c, 0x00000100, 0xffffffff,
 422         0x00008984, 0x06000100, 0xffffffff,
 423         0x0000c164, 0x00000100, 0xffffffff,
 424         0x00008a18, 0x00000100, 0xffffffff,
 425         0x0000897c, 0x06000100, 0xffffffff,
 426         0x00008b28, 0x00000100, 0xffffffff,
 427         0x00009144, 0x00800200, 0xffffffff,
 428         0x00009a60, 0x00000100, 0xffffffff,
 429         0x00009868, 0x00000100, 0xffffffff,
 430         0x00008d58, 0x00000100, 0xffffffff,
 431         0x00009510, 0x00000100, 0xffffffff,
 432         0x0000949c, 0x00000100, 0xffffffff,
 433         0x00009654, 0x00000100, 0xffffffff,
 434         0x00009030, 0x00000100, 0xffffffff,
 435         0x00009034, 0x00000100, 0xffffffff,
 436         0x00009038, 0x00000100, 0xffffffff,
 437         0x0000903c, 0x00000100, 0xffffffff,
 438         0x00009040, 0x00000100, 0xffffffff,
 439         0x0000a200, 0x00000100, 0xffffffff,
 440         0x0000a204, 0x00000100, 0xffffffff,
 441         0x0000a208, 0x00000100, 0xffffffff,
 442         0x0000a20c, 0x00000100, 0xffffffff,
 443         0x00009744, 0x00000100, 0xffffffff,
 444         0x00003f80, 0x00000100, 0xffffffff,
 445         0x0000a210, 0x00000100, 0xffffffff,
 446         0x0000a214, 0x00000100, 0xffffffff,
 447         0x000004d8, 0x00000100, 0xffffffff,
 448         0x00009664, 0x00000100, 0xffffffff,
 449         0x00009698, 0x00000100, 0xffffffff,
 450         0x000004d4, 0x00000200, 0xffffffff,
 451         0x000004d0, 0x00000000, 0xffffffff,
 452         0x000030cc, 0x00000104, 0xffffffff,
 453         0x0000d0c0, 0x00000100, 0xffffffff,
 454         0x0000d8c0, 0x00000100, 0xffffffff,
 455         0x0000802c, 0x40000000, 0xffffffff,
 456         0x00003fc4, 0x40000000, 0xffffffff,
 457         0x0000915c, 0x00010000, 0xffffffff,
 458         0x00009160, 0x00030002, 0xffffffff,
 459         0x00009164, 0x00050004, 0xffffffff,
 460         0x00009168, 0x00070006, 0xffffffff,
 461         0x00009178, 0x00070000, 0xffffffff,
 462         0x0000917c, 0x00030002, 0xffffffff,
 463         0x00009180, 0x00050004, 0xffffffff,
 464         0x0000918c, 0x00010006, 0xffffffff,
 465         0x00009190, 0x00090008, 0xffffffff,
 466         0x00009194, 0x00070000, 0xffffffff,
 467         0x00009198, 0x00030002, 0xffffffff,
 468         0x0000919c, 0x00050004, 0xffffffff,
 469         0x000091a8, 0x00010006, 0xffffffff,
 470         0x000091ac, 0x00090008, 0xffffffff,
 471         0x000091b0, 0x00070000, 0xffffffff,
 472         0x000091b4, 0x00030002, 0xffffffff,
 473         0x000091b8, 0x00050004, 0xffffffff,
 474         0x000091c4, 0x00010006, 0xffffffff,
 475         0x000091c8, 0x00090008, 0xffffffff,
 476         0x000091cc, 0x00070000, 0xffffffff,
 477         0x000091d0, 0x00030002, 0xffffffff,
 478         0x000091d4, 0x00050004, 0xffffffff,
 479         0x000091e0, 0x00010006, 0xffffffff,
 480         0x000091e4, 0x00090008, 0xffffffff,
 481         0x000091e8, 0x00000000, 0xffffffff,
 482         0x000091ec, 0x00070000, 0xffffffff,
 483         0x000091f0, 0x00030002, 0xffffffff,
 484         0x000091f4, 0x00050004, 0xffffffff,
 485         0x00009200, 0x00010006, 0xffffffff,
 486         0x00009204, 0x00090008, 0xffffffff,
 487         0x00009208, 0x00070000, 0xffffffff,
 488         0x0000920c, 0x00030002, 0xffffffff,
 489         0x00009210, 0x00050004, 0xffffffff,
 490         0x0000921c, 0x00010006, 0xffffffff,
 491         0x00009220, 0x00090008, 0xffffffff,
 492         0x00009224, 0x00070000, 0xffffffff,
 493         0x00009228, 0x00030002, 0xffffffff,
 494         0x0000922c, 0x00050004, 0xffffffff,
 495         0x00009238, 0x00010006, 0xffffffff,
 496         0x0000923c, 0x00090008, 0xffffffff,
 497         0x00009240, 0x00070000, 0xffffffff,
 498         0x00009244, 0x00030002, 0xffffffff,
 499         0x00009248, 0x00050004, 0xffffffff,
 500         0x00009254, 0x00010006, 0xffffffff,
 501         0x00009258, 0x00090008, 0xffffffff,
 502         0x0000925c, 0x00070000, 0xffffffff,
 503         0x00009260, 0x00030002, 0xffffffff,
 504         0x00009264, 0x00050004, 0xffffffff,
 505         0x00009270, 0x00010006, 0xffffffff,
 506         0x00009274, 0x00090008, 0xffffffff,
 507         0x00009278, 0x00070000, 0xffffffff,
 508         0x0000927c, 0x00030002, 0xffffffff,
 509         0x00009280, 0x00050004, 0xffffffff,
 510         0x0000928c, 0x00010006, 0xffffffff,
 511         0x00009290, 0x00090008, 0xffffffff,
 512         0x000092a8, 0x00070000, 0xffffffff,
 513         0x000092ac, 0x00030002, 0xffffffff,
 514         0x000092b0, 0x00050004, 0xffffffff,
 515         0x000092bc, 0x00010006, 0xffffffff,
 516         0x000092c0, 0x00090008, 0xffffffff,
 517         0x000092c4, 0x00070000, 0xffffffff,
 518         0x000092c8, 0x00030002, 0xffffffff,
 519         0x000092cc, 0x00050004, 0xffffffff,
 520         0x000092d8, 0x00010006, 0xffffffff,
 521         0x000092dc, 0x00090008, 0xffffffff,
 522         0x00009294, 0x00000000, 0xffffffff,
 523         0x0000802c, 0x40010000, 0xffffffff,
 524         0x00003fc4, 0x40010000, 0xffffffff,
 525         0x0000915c, 0x00010000, 0xffffffff,
 526         0x00009160, 0x00030002, 0xffffffff,
 527         0x00009164, 0x00050004, 0xffffffff,
 528         0x00009168, 0x00070006, 0xffffffff,
 529         0x00009178, 0x00070000, 0xffffffff,
 530         0x0000917c, 0x00030002, 0xffffffff,
 531         0x00009180, 0x00050004, 0xffffffff,
 532         0x0000918c, 0x00010006, 0xffffffff,
 533         0x00009190, 0x00090008, 0xffffffff,
 534         0x00009194, 0x00070000, 0xffffffff,
 535         0x00009198, 0x00030002, 0xffffffff,
 536         0x0000919c, 0x00050004, 0xffffffff,
 537         0x000091a8, 0x00010006, 0xffffffff,
 538         0x000091ac, 0x00090008, 0xffffffff,
 539         0x000091b0, 0x00070000, 0xffffffff,
 540         0x000091b4, 0x00030002, 0xffffffff,
 541         0x000091b8, 0x00050004, 0xffffffff,
 542         0x000091c4, 0x00010006, 0xffffffff,
 543         0x000091c8, 0x00090008, 0xffffffff,
 544         0x000091cc, 0x00070000, 0xffffffff,
 545         0x000091d0, 0x00030002, 0xffffffff,
 546         0x000091d4, 0x00050004, 0xffffffff,
 547         0x000091e0, 0x00010006, 0xffffffff,
 548         0x000091e4, 0x00090008, 0xffffffff,
 549         0x000091e8, 0x00000000, 0xffffffff,
 550         0x000091ec, 0x00070000, 0xffffffff,
 551         0x000091f0, 0x00030002, 0xffffffff,
 552         0x000091f4, 0x00050004, 0xffffffff,
 553         0x00009200, 0x00010006, 0xffffffff,
 554         0x00009204, 0x00090008, 0xffffffff,
 555         0x00009208, 0x00070000, 0xffffffff,
 556         0x0000920c, 0x00030002, 0xffffffff,
 557         0x00009210, 0x00050004, 0xffffffff,
 558         0x0000921c, 0x00010006, 0xffffffff,
 559         0x00009220, 0x00090008, 0xffffffff,
 560         0x00009224, 0x00070000, 0xffffffff,
 561         0x00009228, 0x00030002, 0xffffffff,
 562         0x0000922c, 0x00050004, 0xffffffff,
 563         0x00009238, 0x00010006, 0xffffffff,
 564         0x0000923c, 0x00090008, 0xffffffff,
 565         0x00009240, 0x00070000, 0xffffffff,
 566         0x00009244, 0x00030002, 0xffffffff,
 567         0x00009248, 0x00050004, 0xffffffff,
 568         0x00009254, 0x00010006, 0xffffffff,
 569         0x00009258, 0x00090008, 0xffffffff,
 570         0x0000925c, 0x00070000, 0xffffffff,
 571         0x00009260, 0x00030002, 0xffffffff,
 572         0x00009264, 0x00050004, 0xffffffff,
 573         0x00009270, 0x00010006, 0xffffffff,
 574         0x00009274, 0x00090008, 0xffffffff,
 575         0x00009278, 0x00070000, 0xffffffff,
 576         0x0000927c, 0x00030002, 0xffffffff,
 577         0x00009280, 0x00050004, 0xffffffff,
 578         0x0000928c, 0x00010006, 0xffffffff,
 579         0x00009290, 0x00090008, 0xffffffff,
 580         0x000092a8, 0x00070000, 0xffffffff,
 581         0x000092ac, 0x00030002, 0xffffffff,
 582         0x000092b0, 0x00050004, 0xffffffff,
 583         0x000092bc, 0x00010006, 0xffffffff,
 584         0x000092c0, 0x00090008, 0xffffffff,
 585         0x000092c4, 0x00070000, 0xffffffff,
 586         0x000092c8, 0x00030002, 0xffffffff,
 587         0x000092cc, 0x00050004, 0xffffffff,
 588         0x000092d8, 0x00010006, 0xffffffff,
 589         0x000092dc, 0x00090008, 0xffffffff,
 590         0x00009294, 0x00000000, 0xffffffff,
 591         0x0000802c, 0xc0000000, 0xffffffff,
 592         0x00003fc4, 0xc0000000, 0xffffffff,
 593         0x000008f8, 0x00000010, 0xffffffff,
 594         0x000008fc, 0x00000000, 0xffffffff,
 595         0x000008f8, 0x00000011, 0xffffffff,
 596         0x000008fc, 0x00000000, 0xffffffff,
 597         0x000008f8, 0x00000012, 0xffffffff,
 598         0x000008fc, 0x00000000, 0xffffffff,
 599         0x000008f8, 0x00000013, 0xffffffff,
 600         0x000008fc, 0x00000000, 0xffffffff,
 601         0x000008f8, 0x00000014, 0xffffffff,
 602         0x000008fc, 0x00000000, 0xffffffff,
 603         0x000008f8, 0x00000015, 0xffffffff,
 604         0x000008fc, 0x00000000, 0xffffffff,
 605         0x000008f8, 0x00000016, 0xffffffff,
 606         0x000008fc, 0x00000000, 0xffffffff,
 607         0x000008f8, 0x00000017, 0xffffffff,
 608         0x000008fc, 0x00000000, 0xffffffff,
 609         0x000008f8, 0x00000018, 0xffffffff,
 610         0x000008fc, 0x00000000, 0xffffffff,
 611         0x000008f8, 0x00000019, 0xffffffff,
 612         0x000008fc, 0x00000000, 0xffffffff,
 613         0x000008f8, 0x0000001a, 0xffffffff,
 614         0x000008fc, 0x00000000, 0xffffffff,
 615         0x000008f8, 0x0000001b, 0xffffffff,
 616         0x000008fc, 0x00000000, 0xffffffff
 617 };
 618 #define CAYMAN_MGCG_DEFAULT_LENGTH sizeof(cayman_mgcg_default) / (3 * sizeof(u32))
 619
 620 static const u32 cayman_mgcg_disable[] =
 621 {
 622         0x0000802c, 0xc0000000, 0xffffffff,
 623         0x000008f8, 0x00000000, 0xffffffff,
 624         0x000008fc, 0xffffffff, 0xffffffff,
 625         0x000008f8, 0x00000001, 0xffffffff,
 626         0x000008fc, 0xffffffff, 0xffffffff,
 627         0x000008f8, 0x00000002, 0xffffffff,
 628         0x000008fc, 0xffffffff, 0xffffffff,
 629         0x000008f8, 0x00000003, 0xffffffff,
 630         0x000008fc, 0xffffffff, 0xffffffff,
 631         0x00009150, 0x00600000, 0xffffffff
 632 };
 633 #define CAYMAN_MGCG_DISABLE_LENGTH   sizeof(cayman_mgcg_disable) / (3 * sizeof(u32))
 634
 635 static const u32 cayman_mgcg_enable[] =
 636 {
 637         0x0000802c, 0xc0000000, 0xffffffff,
 638         0x000008f8, 0x00000000, 0xffffffff,
 639         0x000008fc, 0x00000000, 0xffffffff,
 640         0x000008f8, 0x00000001, 0xffffffff,
 641         0x000008fc, 0x00000000, 0xffffffff,
 642         0x000008f8, 0x00000002, 0xffffffff,
 643         0x000008fc, 0x00600000, 0xffffffff,
 644         0x000008f8, 0x00000003, 0xffffffff,
 645         0x000008fc, 0x00000000, 0xffffffff,
 646         0x00009150, 0x96944200, 0xffffffff
 647 };
 648
 649 #define CAYMAN_MGCG_ENABLE_LENGTH   sizeof(cayman_mgcg_enable) / (3 * sizeof(u32))
 650
 651 #define NISLANDS_SYSLS_SEQUENCE  100
 652
 653 static const u32 cayman_sysls_default[] =
 654 {
 655         /* Register,   Value,     Mask bits */
 656         0x000055e8, 0x00000000, 0xffffffff,
 657         0x0000d0bc, 0x00000000, 0xffffffff,
 658         0x0000d8bc, 0x00000000, 0xffffffff,
 659         0x000015c0, 0x000c1401, 0xffffffff,
 660         0x0000264c, 0x000c0400, 0xffffffff,
 661         0x00002648, 0x000c0400, 0xffffffff,
 662         0x00002650, 0x000c0400, 0xffffffff,
 663         0x000020b8, 0x000c0400, 0xffffffff,
 664         0x000020bc, 0x000c0400, 0xffffffff,
 665         0x000020c0, 0x000c0c80, 0xffffffff,
 666         0x0000f4a0, 0x000000c0, 0xffffffff,
 667         0x0000f4a4, 0x00680fff, 0xffffffff,
 668         0x00002f50, 0x00000404, 0xffffffff,
 669         0x000004c8, 0x00000001, 0xffffffff,
 670         0x000064ec, 0x00000000, 0xffffffff,
 671         0x00000c7c, 0x00000000, 0xffffffff,
 672         0x00008dfc, 0x00000000, 0xffffffff
 673 };
 674 #define CAYMAN_SYSLS_DEFAULT_LENGTH sizeof(cayman_sysls_default) / (3 * sizeof(u32))
 675
 676 static const u32 cayman_sysls_disable[] =
 677 {
 678         /* Register,   Value,     Mask bits */
 679         0x0000d0c0, 0x00000000, 0xffffffff,
 680         0x0000d8c0, 0x00000000, 0xffffffff,
 681         0x000055e8, 0x00000000, 0xffffffff,
 682         0x0000d0bc, 0x00000000, 0xffffffff,
 683         0x0000d8bc, 0x00000000, 0xffffffff,
 684         0x000015c0, 0x00041401, 0xffffffff,
 685         0x0000264c, 0x00040400, 0xffffffff,
 686         0x00002648, 0x00040400, 0xffffffff,
 687         0x00002650, 0x00040400, 0xffffffff,
 688         0x000020b8, 0x00040400, 0xffffffff,
 689         0x000020bc, 0x00040400, 0xffffffff,
 690         0x000020c0, 0x00040c80, 0xffffffff,
 691         0x0000f4a0, 0x000000c0, 0xffffffff,
 692         0x0000f4a4, 0x00680000, 0xffffffff,
 693         0x00002f50, 0x00000404, 0xffffffff,
 694         0x000004c8, 0x00000001, 0xffffffff,
 695         0x000064ec, 0x00007ffd, 0xffffffff,
 696         0x00000c7c, 0x0000ff00, 0xffffffff,
 697         0x00008dfc, 0x0000007f, 0xffffffff
 698 };
 699 #define CAYMAN_SYSLS_DISABLE_LENGTH sizeof(cayman_sysls_disable) / (3 * sizeof(u32))
 700
 701 static const u32 cayman_sysls_enable[] =
 702 {
 703         /* Register,   Value,     Mask bits */
 704         0x000055e8, 0x00000001, 0xffffffff,
 705         0x0000d0bc, 0x00000100, 0xffffffff,
 706         0x0000d8bc, 0x00000100, 0xffffffff,
 707         0x000015c0, 0x000c1401, 0xffffffff,
 708         0x0000264c, 0x000c0400, 0xffffffff,
 709         0x00002648, 0x000c0400, 0xffffffff,
 710         0x00002650, 0x000c0400, 0xffffffff,
 711         0x000020b8, 0x000c0400, 0xffffffff,
 712         0x000020bc, 0x000c0400, 0xffffffff,
 713         0x000020c0, 0x000c0c80, 0xffffffff,
 714         0x0000f4a0, 0x000000c0, 0xffffffff,
 715         0x0000f4a4, 0x00680fff, 0xffffffff,
 716         0x00002f50, 0x00000903, 0xffffffff,
 717         0x000004c8, 0x00000000, 0xffffffff,
 718         0x000064ec, 0x00000000, 0xffffffff,
 719         0x00000c7c, 0x00000000, 0xffffffff,
 720         0x00008dfc, 0x00000000, 0xffffffff
 721 };
 722 #define CAYMAN_SYSLS_ENABLE_LENGTH sizeof(cayman_sysls_enable) / (3 * sizeof(u32))
 723
 724 extern int ni_mc_load_microcode(struct radeon_device *rdev);
 725
 726 struct ni_power_info *ni_get_pi(struct radeon_device *rdev)
 727 {
 728         struct ni_power_info *pi = rdev->pm.dpm.priv;
 729
 730         return pi;
 731 }
 732
 733 struct ni_ps *ni_get_ps(struct radeon_ps *rps)
 734 {
 735         struct ni_ps *ps = rps->ps_priv;
 736
 737         return ps;
 738 }
 739
 740 static void ni_calculate_leakage_for_v_and_t_formula(const struct ni_leakage_coeffients *coeff,
 741                                                      u16 v, s32 t,
 742                                                      u32 ileakage,
 743                                                      u32 *leakage)
 744 {
 745         s64 kt, kv, leakage_w, i_leakage, vddc, temperature;
 746
 747         i_leakage = div64_s64(drm_int2fixp(ileakage), 1000);
 748         vddc = div64_s64(drm_int2fixp(v), 1000);
 749         temperature = div64_s64(drm_int2fixp(t), 1000);
 750
 751         kt = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->at), 1000),
 752                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bt), 1000), temperature)));
 753         kv = drm_fixp_mul(div64_s64(drm_int2fixp(coeff->av), 1000),
 754                           drm_fixp_exp(drm_fixp_mul(div64_s64(drm_int2fixp(coeff->bv), 1000), vddc)));
 755
 756         leakage_w = drm_fixp_mul(drm_fixp_mul(drm_fixp_mul(i_leakage, kt), kv), vddc);
 757
 758         *leakage = drm_fixp2int(leakage_w * 1000);
 759 }
 760
 761 static void ni_calculate_leakage_for_v_and_t(struct radeon_device *rdev,
 762                                              const struct ni_leakage_coeffients *coeff,
 763                                              u16 v,
 764                                              s32 t,
 765                                              u32 i_leakage,
 766                                              u32 *leakage)
 767 {
 768         ni_calculate_leakage_for_v_and_t_formula(coeff, v, t, i_leakage, leakage);
 769 }
 770
 771 bool ni_dpm_vblank_too_short(struct radeon_device *rdev)
 772 {
 773         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 774         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 775         /* we never hit the non-gddr5 limit so disable it */
 776         u32 switch_limit = pi->mem_gddr5 ? 450 : 0;
 777
 778         if (vblank_time < switch_limit)
 779                 return true;
 780         else
 781                 return false;
 782
 783 }
 784
 785 static void ni_apply_state_adjust_rules(struct radeon_device *rdev,
 786                                         struct radeon_ps *rps)
 787 {
 788         struct ni_ps *ps = ni_get_ps(rps);
 789         struct radeon_clock_and_voltage_limits *max_limits;
 790         bool disable_mclk_switching;
 791         u32 mclk;
 792         u16 vddci;
 793         int i;
 794
 795         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 796             ni_dpm_vblank_too_short(rdev))
 797                 disable_mclk_switching = true;
 798         else
 799                 disable_mclk_switching = false;
 800
 801         if (rdev->pm.dpm.ac_power)
 802                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 803         else
 804                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 805
 806         if (rdev->pm.dpm.ac_power == false) {
 807                 for (i = 0; i < ps->performance_level_count; i++) {
 808                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 809                                 ps->performance_levels[i].mclk = max_limits->mclk;
 810                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 811                                 ps->performance_levels[i].sclk = max_limits->sclk;
 812                         if (ps->performance_levels[i].vddc > max_limits->vddc)
 813                                 ps->performance_levels[i].vddc = max_limits->vddc;
 814                         if (ps->performance_levels[i].vddci > max_limits->vddci)
 815                                 ps->performance_levels[i].vddci = max_limits->vddci;
 816                 }
 817         }
 818
 819         /* XXX validate the min clocks required for display */
 820
 821         /* adjust low state */
 822         if (disable_mclk_switching) {
 823                 ps->performance_levels[0].mclk =
 824                         ps->performance_levels[ps->performance_level_count - 1].mclk;
 825                 ps->performance_levels[0].vddci =
 826                         ps->performance_levels[ps->performance_level_count - 1].vddci;
 827         }
 828
 829         btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 830                                   &ps->performance_levels[0].sclk,
 831                                   &ps->performance_levels[0].mclk);
 832
 833         for (i = 1; i < ps->performance_level_count; i++) {
 834                 if (ps->performance_levels[i].sclk < ps->performance_levels[i - 1].sclk)
 835                         ps->performance_levels[i].sclk = ps->performance_levels[i - 1].sclk;
 836                 if (ps->performance_levels[i].vddc < ps->performance_levels[i - 1].vddc)
 837                         ps->performance_levels[i].vddc = ps->performance_levels[i - 1].vddc;
 838         }
 839
 840         /* adjust remaining states */
 841         if (disable_mclk_switching) {
 842                 mclk = ps->performance_levels[0].mclk;
 843                 vddci = ps->performance_levels[0].vddci;
 844                 for (i = 1; i < ps->performance_level_count; i++) {
 845                         if (mclk < ps->performance_levels[i].mclk)
 846                                 mclk = ps->performance_levels[i].mclk;
 847                         if (vddci < ps->performance_levels[i].vddci)
 848                                 vddci = ps->performance_levels[i].vddci;
 849                 }
 850                 for (i = 0; i < ps->performance_level_count; i++) {
 851                         ps->performance_levels[i].mclk = mclk;
 852                         ps->performance_levels[i].vddci = vddci;
 853                 }
 854         } else {
 855                 for (i = 1; i < ps->performance_level_count; i++) {
 856                         if (ps->performance_levels[i].mclk < ps->performance_levels[i - 1].mclk)
 857                                 ps->performance_levels[i].mclk = ps->performance_levels[i - 1].mclk;
 858                         if (ps->performance_levels[i].vddci < ps->performance_levels[i - 1].vddci)
 859                                 ps->performance_levels[i].vddci = ps->performance_levels[i - 1].vddci;
 860                 }
 861         }
 862
 863         for (i = 1; i < ps->performance_level_count; i++)
 864                 btc_skip_blacklist_clocks(rdev, max_limits->sclk, max_limits->mclk,
 865                                           &ps->performance_levels[i].sclk,
 866                                           &ps->performance_levels[i].mclk);
 867
 868         for (i = 0; i < ps->performance_level_count; i++)
 869                 btc_adjust_clock_combinations(rdev, max_limits,
 870                                               &ps->performance_levels[i]);
 871
 872         for (i = 0; i < ps->performance_level_count; i++) {
 873                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
 874                                                    ps->performance_levels[i].sclk,
 875                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 876                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
 877                                                    ps->performance_levels[i].mclk,
 878                                                    max_limits->vddci, &ps->performance_levels[i].vddci);
 879                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
 880                                                    ps->performance_levels[i].mclk,
 881                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 882                 btc_apply_voltage_dependency_rules(&rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk,
 883                                                    rdev->clock.current_dispclk,
 884                                                    max_limits->vddc,  &ps->performance_levels[i].vddc);
 885         }
 886
 887         for (i = 0; i < ps->performance_level_count; i++) {
 888                 btc_apply_voltage_delta_rules(rdev,
 889                                               max_limits->vddc, max_limits->vddci,
 890                                               &ps->performance_levels[i].vddc,
 891                                               &ps->performance_levels[i].vddci);
 892         }
 893
 894         ps->dc_compatible = true;
 895         for (i = 0; i < ps->performance_level_count; i++) {
 896                 if (ps->performance_levels[i].vddc > rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.vddc)
 897                         ps->dc_compatible = false;
 898
 899                 if (ps->performance_levels[i].vddc < rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2)
 900                         ps->performance_levels[i].flags &= ~ATOM_PPLIB_R600_FLAGS_PCIEGEN2;
 901         }
 902 }
 903
 904 static void ni_cg_clockgating_default(struct radeon_device *rdev)
 905 {
 906         u32 count;
 907         const u32 *ps = NULL;
 908
 909         ps = (const u32 *)&cayman_cgcg_cgls_default;
 910         count = CAYMAN_CGCG_CGLS_DEFAULT_LENGTH;
 911
 912         btc_program_mgcg_hw_sequence(rdev, ps, count);
 913 }
 914
 915 static void ni_gfx_clockgating_enable(struct radeon_device *rdev,
 916                                       bool enable)
 917 {
 918         u32 count;
 919         const u32 *ps = NULL;
 920
 921         if (enable) {
 922                 ps = (const u32 *)&cayman_cgcg_cgls_enable;
 923                 count = CAYMAN_CGCG_CGLS_ENABLE_LENGTH;
 924         } else {
 925                 ps = (const u32 *)&cayman_cgcg_cgls_disable;
 926                 count = CAYMAN_CGCG_CGLS_DISABLE_LENGTH;
 927         }
 928
 929         btc_program_mgcg_hw_sequence(rdev, ps, count);
 930 }
 931
 932 static void ni_mg_clockgating_default(struct radeon_device *rdev)
 933 {
 934         u32 count;
 935         const u32 *ps = NULL;
 936
 937         ps = (const u32 *)&cayman_mgcg_default;
 938         count = CAYMAN_MGCG_DEFAULT_LENGTH;
 939
 940         btc_program_mgcg_hw_sequence(rdev, ps, count);
 941 }
 942
 943 static void ni_mg_clockgating_enable(struct radeon_device *rdev,
 944                                      bool enable)
 945 {
 946         u32 count;
 947         const u32 *ps = NULL;
 948
 949         if (enable) {
 950                 ps = (const u32 *)&cayman_mgcg_enable;
 951                 count = CAYMAN_MGCG_ENABLE_LENGTH;
 952         } else {
 953                 ps = (const u32 *)&cayman_mgcg_disable;
 954                 count = CAYMAN_MGCG_DISABLE_LENGTH;
 955         }
 956
 957         btc_program_mgcg_hw_sequence(rdev, ps, count);
 958 }
 959
 960 static void ni_ls_clockgating_default(struct radeon_device *rdev)
 961 {
 962         u32 count;
 963         const u32 *ps = NULL;
 964
 965         ps = (const u32 *)&cayman_sysls_default;
 966         count = CAYMAN_SYSLS_DEFAULT_LENGTH;
 967
 968         btc_program_mgcg_hw_sequence(rdev, ps, count);
 969 }
 970
 971 static void ni_ls_clockgating_enable(struct radeon_device *rdev,
 972                                      bool enable)
 973 {
 974         u32 count;
 975         const u32 *ps = NULL;
 976
 977         if (enable) {
 978                 ps = (const u32 *)&cayman_sysls_enable;
 979                 count = CAYMAN_SYSLS_ENABLE_LENGTH;
 980         } else {
 981                 ps = (const u32 *)&cayman_sysls_disable;
 982                 count = CAYMAN_SYSLS_DISABLE_LENGTH;
 983         }
 984
 985         btc_program_mgcg_hw_sequence(rdev, ps, count);
 986
 987 }
 988
 989 static int ni_patch_single_dependency_table_based_on_leakage(struct radeon_device *rdev,
 990                                                              struct radeon_clock_voltage_dependency_table *table)
 991 {
 992         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
 993         u32 i;
 994
 995         if (table) {
 996                 for (i = 0; i < table->count; i++) {
 997                         if (0xff01 == table->entries[i].v) {
 998                                 if (pi->max_vddc == 0)
 999                                         return -EINVAL;
1000                                 table->entries[i].v = pi->max_vddc;
1001                         }
1002                 }
1003         }
1004         return 0;
1005 }
1006
1007 static int ni_patch_dependency_tables_based_on_leakage(struct radeon_device *rdev)
1008 {
1009         int ret = 0;
1010
1011         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1012                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
1013
1014         ret = ni_patch_single_dependency_table_based_on_leakage(rdev,
1015                                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
1016         return ret;
1017 }
1018
1019 static void ni_stop_dpm(struct radeon_device *rdev)
1020 {
1021         WREG32_P(GENERAL_PWRMGT, 0, ~GLOBAL_PWRMGT_EN);
1022 }
1023
1024 #if 0
1025 static int ni_notify_hw_of_power_source(struct radeon_device *rdev,
1026                                         bool ac_power)
1027 {
1028         if (ac_power)
1029                 return (rv770_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC) == PPSMC_Result_OK) ?
1030                         0 : -EINVAL;
1031
1032         return 0;
1033 }
1034 #endif
1035
1036 static PPSMC_Result ni_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1037                                                       PPSMC_Msg msg, u32 parameter)
1038 {
1039         WREG32(SMC_SCRATCH0, parameter);
1040         return rv770_send_msg_to_smc(rdev, msg);
1041 }
1042
1043 static int ni_restrict_performance_levels_before_switch(struct radeon_device *rdev)
1044 {
1045         if (rv770_send_msg_to_smc(rdev, PPSMC_MSG_NoForcedLevel) != PPSMC_Result_OK)
1046                 return -EINVAL;
1047
1048         return (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) == PPSMC_Result_OK) ?
1049                 0 : -EINVAL;
1050 }
1051
1052 int ni_dpm_force_performance_level(struct radeon_device *rdev,
1053                                    enum radeon_dpm_forced_level level)
1054 {
1055         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
1056                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1057                         return -EINVAL;
1058
1059                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 1) != PPSMC_Result_OK)
1060                         return -EINVAL;
1061         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
1062                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1063                         return -EINVAL;
1064
1065                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 1) != PPSMC_Result_OK)
1066                         return -EINVAL;
1067         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
1068                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetForcedLevels, 0) != PPSMC_Result_OK)
1069                         return -EINVAL;
1070
1071                 if (ni_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SetEnabledLevels, 0) != PPSMC_Result_OK)
1072                         return -EINVAL;
1073         }
1074
1075         rdev->pm.dpm.forced_level = level;
1076
1077         return 0;
1078 }
1079
1080 static void ni_stop_smc(struct radeon_device *rdev)
1081 {
1082         u32 tmp;
1083         int i;
1084
1085         for (i = 0; i < rdev->usec_timeout; i++) {
1086                 tmp = RREG32(LB_SYNC_RESET_SEL) & LB_SYNC_RESET_SEL_MASK;
1087                 if (tmp != 1)
1088                         break;
1089                 udelay(1);
1090         }
1091
1092         udelay(100);
1093
1094         r7xx_stop_smc(rdev);
1095 }
1096
1097 static int ni_process_firmware_header(struct radeon_device *rdev)
1098 {
1099         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1100         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1101         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1102         u32 tmp;
1103         int ret;
1104
1105         ret = rv770_read_smc_sram_dword(rdev,
1106                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1107                                         NISLANDS_SMC_FIRMWARE_HEADER_stateTable,
1108                                         &tmp, pi->sram_end);
1109
1110         if (ret)
1111                 return ret;
1112
1113         pi->state_table_start = (u16)tmp;
1114
1115         ret = rv770_read_smc_sram_dword(rdev,
1116                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1117                                         NISLANDS_SMC_FIRMWARE_HEADER_softRegisters,
1118                                         &tmp, pi->sram_end);
1119
1120         if (ret)
1121                 return ret;
1122
1123         pi->soft_regs_start = (u16)tmp;
1124
1125         ret = rv770_read_smc_sram_dword(rdev,
1126                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1127                                         NISLANDS_SMC_FIRMWARE_HEADER_mcRegisterTable,
1128                                         &tmp, pi->sram_end);
1129
1130         if (ret)
1131                 return ret;
1132
1133         eg_pi->mc_reg_table_start = (u16)tmp;
1134
1135         ret = rv770_read_smc_sram_dword(rdev,
1136                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1137                                         NISLANDS_SMC_FIRMWARE_HEADER_fanTable,
1138                                         &tmp, pi->sram_end);
1139
1140         if (ret)
1141                 return ret;
1142
1143         ni_pi->fan_table_start = (u16)tmp;
1144
1145         ret = rv770_read_smc_sram_dword(rdev,
1146                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1147                                         NISLANDS_SMC_FIRMWARE_HEADER_mcArbDramAutoRefreshTable,
1148                                         &tmp, pi->sram_end);
1149
1150         if (ret)
1151                 return ret;
1152
1153         ni_pi->arb_table_start = (u16)tmp;
1154
1155         ret = rv770_read_smc_sram_dword(rdev,
1156                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1157                                         NISLANDS_SMC_FIRMWARE_HEADER_cacTable,
1158                                         &tmp, pi->sram_end);
1159
1160         if (ret)
1161                 return ret;
1162
1163         ni_pi->cac_table_start = (u16)tmp;
1164
1165         ret = rv770_read_smc_sram_dword(rdev,
1166                                         NISLANDS_SMC_FIRMWARE_HEADER_LOCATION +
1167                                         NISLANDS_SMC_FIRMWARE_HEADER_spllTable,
1168                                         &tmp, pi->sram_end);
1169
1170         if (ret)
1171                 return ret;
1172
1173         ni_pi->spll_table_start = (u16)tmp;
1174
1175
1176         return ret;
1177 }
1178
1179 static void ni_read_clock_registers(struct radeon_device *rdev)
1180 {
1181         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1182
1183         ni_pi->clock_registers.cg_spll_func_cntl = RREG32(CG_SPLL_FUNC_CNTL);
1184         ni_pi->clock_registers.cg_spll_func_cntl_2 = RREG32(CG_SPLL_FUNC_CNTL_2);
1185         ni_pi->clock_registers.cg_spll_func_cntl_3 = RREG32(CG_SPLL_FUNC_CNTL_3);
1186         ni_pi->clock_registers.cg_spll_func_cntl_4 = RREG32(CG_SPLL_FUNC_CNTL_4);
1187         ni_pi->clock_registers.cg_spll_spread_spectrum = RREG32(CG_SPLL_SPREAD_SPECTRUM);
1188         ni_pi->clock_registers.cg_spll_spread_spectrum_2 = RREG32(CG_SPLL_SPREAD_SPECTRUM_2);
1189         ni_pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1190         ni_pi->clock_registers.mpll_ad_func_cntl_2 = RREG32(MPLL_AD_FUNC_CNTL_2);
1191         ni_pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1192         ni_pi->clock_registers.mpll_dq_func_cntl_2 = RREG32(MPLL_DQ_FUNC_CNTL_2);
1193         ni_pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1194         ni_pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1195         ni_pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1196         ni_pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1197 }
1198
1199 #if 0
1200 static int ni_enter_ulp_state(struct radeon_device *rdev)
1201 {
1202         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1203
1204         if (pi->gfx_clock_gating) {
1205                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~DYN_GFX_CLK_OFF_EN);
1206                 WREG32_P(SCLK_PWRMGT_CNTL, GFX_CLK_FORCE_ON, ~GFX_CLK_FORCE_ON);
1207                 WREG32_P(SCLK_PWRMGT_CNTL, 0, ~GFX_CLK_FORCE_ON);
1208                 RREG32(GB_ADDR_CONFIG);
1209         }
1210
1211         WREG32_P(SMC_MSG, HOST_SMC_MSG(PPSMC_MSG_SwitchToMinimumPower),
1212                  ~HOST_SMC_MSG_MASK);
1213
1214         udelay(25000);
1215
1216         return 0;
1217 }
1218 #endif
1219
1220 static void ni_program_response_times(struct radeon_device *rdev)
1221 {
1222         u32 voltage_response_time, backbias_response_time, acpi_delay_time, vbi_time_out;
1223         u32 vddc_dly, bb_dly, acpi_dly, vbi_dly, mclk_switch_limit;
1224         u32 reference_clock;
1225
1226         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mvdd_chg_time, 1);
1227
1228         voltage_response_time = (u32)rdev->pm.dpm.voltage_response_time;
1229         backbias_response_time = (u32)rdev->pm.dpm.backbias_response_time;
1230
1231         if (voltage_response_time == 0)
1232                 voltage_response_time = 1000;
1233
1234         if (backbias_response_time == 0)
1235                 backbias_response_time = 1000;
1236
1237         acpi_delay_time = 15000;
1238         vbi_time_out = 100000;
1239
1240         reference_clock = radeon_get_xclk(rdev);
1241
1242         vddc_dly = (voltage_response_time  * reference_clock) / 1600;
1243         bb_dly   = (backbias_response_time * reference_clock) / 1600;
1244         acpi_dly = (acpi_delay_time * reference_clock) / 1600;
1245         vbi_dly  = (vbi_time_out * reference_clock) / 1600;
1246
1247         mclk_switch_limit = (460 * reference_clock) / 100;
1248
1249         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_vreg,  vddc_dly);
1250         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_bbias, bb_dly);
1251         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_delay_acpi,  acpi_dly);
1252         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_chg_timeout, vbi_dly);
1253         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mc_block_delay, 0xAA);
1254         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_mclk_switch_lim, mclk_switch_limit);
1255 }
1256
1257 static void ni_populate_smc_voltage_table(struct radeon_device *rdev,
1258                                           struct atom_voltage_table *voltage_table,
1259                                           NISLANDS_SMC_STATETABLE *table)
1260 {
1261         unsigned int i;
1262
1263         for (i = 0; i < voltage_table->count; i++) {
1264                 table->highSMIO[i] = 0;
1265                 table->lowSMIO[i] |= cpu_to_be32(voltage_table->entries[i].smio_low);
1266         }
1267 }
1268
1269 static void ni_populate_smc_voltage_tables(struct radeon_device *rdev,
1270                                            NISLANDS_SMC_STATETABLE *table)
1271 {
1272         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1273         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1274         unsigned char i;
1275
1276         if (eg_pi->vddc_voltage_table.count) {
1277                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddc_voltage_table, table);
1278                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] = 0;
1279                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDC] =
1280                         cpu_to_be32(eg_pi->vddc_voltage_table.mask_low);
1281
1282                 for (i = 0; i < eg_pi->vddc_voltage_table.count; i++) {
1283                         if (pi->max_vddc_in_table <= eg_pi->vddc_voltage_table.entries[i].value) {
1284                                 table->maxVDDCIndexInPPTable = i;
1285                                 break;
1286                         }
1287                 }
1288         }
1289
1290         if (eg_pi->vddci_voltage_table.count) {
1291                 ni_populate_smc_voltage_table(rdev, &eg_pi->vddci_voltage_table, table);
1292
1293                 table->voltageMaskTable.highMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] = 0;
1294                 table->voltageMaskTable.lowMask[NISLANDS_SMC_VOLTAGEMASK_VDDCI] =
1295                         cpu_to_be32(eg_pi->vddci_voltage_table.mask_low);
1296         }
1297 }
1298
1299 static int ni_populate_voltage_value(struct radeon_device *rdev,
1300                                      struct atom_voltage_table *table,
1301                                      u16 value,
1302                                      NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1303 {
1304         unsigned int i;
1305
1306         for (i = 0; i < table->count; i++) {
1307                 if (value <= table->entries[i].value) {
1308                         voltage->index = (u8)i;
1309                         voltage->value = cpu_to_be16(table->entries[i].value);
1310                         break;
1311                 }
1312         }
1313
1314         if (i >= table->count)
1315                 return -EINVAL;
1316
1317         return 0;
1318 }
1319
1320 static void ni_populate_mvdd_value(struct radeon_device *rdev,
1321                                    u32 mclk,
1322                                    NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1323 {
1324         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1325         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1326
1327         if (!pi->mvdd_control) {
1328                 voltage->index = eg_pi->mvdd_high_index;
1329                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1330                 return;
1331         }
1332
1333         if (mclk <= pi->mvdd_split_frequency) {
1334                 voltage->index = eg_pi->mvdd_low_index;
1335                 voltage->value = cpu_to_be16(MVDD_LOW_VALUE);
1336         } else {
1337                 voltage->index = eg_pi->mvdd_high_index;
1338                 voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1339         }
1340 }
1341
1342 static int ni_get_std_voltage_value(struct radeon_device *rdev,
1343                                     NISLANDS_SMC_VOLTAGE_VALUE *voltage,
1344                                     u16 *std_voltage)
1345 {
1346         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries &&
1347             ((u32)voltage->index < rdev->pm.dpm.dyn_state.cac_leakage_table.count))
1348                 *std_voltage = rdev->pm.dpm.dyn_state.cac_leakage_table.entries[voltage->index].vddc;
1349         else
1350                 *std_voltage = be16_to_cpu(voltage->value);
1351
1352         return 0;
1353 }
1354
1355 static void ni_populate_std_voltage_value(struct radeon_device *rdev,
1356                                           u16 value, u8 index,
1357                                           NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1358 {
1359         voltage->index = index;
1360         voltage->value = cpu_to_be16(value);
1361 }
1362
1363 static u32 ni_get_smc_power_scaling_factor(struct radeon_device *rdev)
1364 {
1365         u32 xclk_period;
1366         u32 xclk = radeon_get_xclk(rdev);
1367         u32 tmp = RREG32(CG_CAC_CTRL) & TID_CNT_MASK;
1368
1369         xclk_period = (1000000000UL / xclk);
1370         xclk_period /= 10000UL;
1371
1372         return tmp * xclk_period;
1373 }
1374
1375 static u32 ni_scale_power_for_smc(u32 power_in_watts, u32 scaling_factor)
1376 {
1377         return (power_in_watts * scaling_factor) << 2;
1378 }
1379
1380 static u32 ni_calculate_power_boost_limit(struct radeon_device *rdev,
1381                                           struct radeon_ps *radeon_state,
1382                                           u32 near_tdp_limit)
1383 {
1384         struct ni_ps *state = ni_get_ps(radeon_state);
1385         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1386         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1387         u32 power_boost_limit = 0;
1388         int ret;
1389
1390         if (ni_pi->enable_power_containment &&
1391             ni_pi->use_power_boost_limit) {
1392                 NISLANDS_SMC_VOLTAGE_VALUE vddc;
1393                 u16 std_vddc_med;
1394                 u16 std_vddc_high;
1395                 u64 tmp, n, d;
1396
1397                 if (state->performance_level_count < 3)
1398                         return 0;
1399
1400                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1401                                                 state->performance_levels[state->performance_level_count - 2].vddc,
1402                                                 &vddc);
1403                 if (ret)
1404                         return 0;
1405
1406                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_med);
1407                 if (ret)
1408                         return 0;
1409
1410                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1411                                                 state->performance_levels[state->performance_level_count - 1].vddc,
1412                                                 &vddc);
1413                 if (ret)
1414                         return 0;
1415
1416                 ret = ni_get_std_voltage_value(rdev, &vddc, &std_vddc_high);
1417                 if (ret)
1418                         return 0;
1419
1420                 n = ((u64)near_tdp_limit * ((u64)std_vddc_med * (u64)std_vddc_med) * 90);
1421                 d = ((u64)std_vddc_high * (u64)std_vddc_high * 100);
1422                 tmp = div64_u64(n, d);
1423
1424                 if (tmp >> 32)
1425                         return 0;
1426                 power_boost_limit = (u32)tmp;
1427         }
1428
1429         return power_boost_limit;
1430 }
1431
1432 static int ni_calculate_adjusted_tdp_limits(struct radeon_device *rdev,
1433                                             bool adjust_polarity,
1434                                             u32 tdp_adjustment,
1435                                             u32 *tdp_limit,
1436                                             u32 *near_tdp_limit)
1437 {
1438         if (tdp_adjustment > (u32)rdev->pm.dpm.tdp_od_limit)
1439                 return -EINVAL;
1440
1441         if (adjust_polarity) {
1442                 *tdp_limit = ((100 + tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1443                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit + (*tdp_limit - rdev->pm.dpm.tdp_limit);
1444         } else {
1445                 *tdp_limit = ((100 - tdp_adjustment) * rdev->pm.dpm.tdp_limit) / 100;
1446                 *near_tdp_limit = rdev->pm.dpm.near_tdp_limit - (rdev->pm.dpm.tdp_limit - *tdp_limit);
1447         }
1448
1449         return 0;
1450 }
1451
1452 static int ni_populate_smc_tdp_limits(struct radeon_device *rdev,
1453                                       struct radeon_ps *radeon_state)
1454 {
1455         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1456         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1457
1458         if (ni_pi->enable_power_containment) {
1459                 NISLANDS_SMC_STATETABLE *smc_table = &ni_pi->smc_statetable;
1460                 u32 scaling_factor = ni_get_smc_power_scaling_factor(rdev);
1461                 u32 tdp_limit;
1462                 u32 near_tdp_limit;
1463                 u32 power_boost_limit;
1464                 int ret;
1465
1466                 if (scaling_factor == 0)
1467                         return -EINVAL;
1468
1469                 memset(smc_table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1470
1471                 ret = ni_calculate_adjusted_tdp_limits(rdev,
1472                                                        false, /* ??? */
1473                                                        rdev->pm.dpm.tdp_adjustment,
1474                                                        &tdp_limit,
1475                                                        &near_tdp_limit);
1476                 if (ret)
1477                         return ret;
1478
1479                 power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state,
1480                                                                    near_tdp_limit);
1481
1482                 smc_table->dpm2Params.TDPLimit =
1483                         cpu_to_be32(ni_scale_power_for_smc(tdp_limit, scaling_factor));
1484                 smc_table->dpm2Params.NearTDPLimit =
1485                         cpu_to_be32(ni_scale_power_for_smc(near_tdp_limit, scaling_factor));
1486                 smc_table->dpm2Params.SafePowerLimit =
1487                         cpu_to_be32(ni_scale_power_for_smc((near_tdp_limit * NISLANDS_DPM2_TDP_SAFE_LIMIT_PERCENT) / 100,
1488                                                            scaling_factor));
1489                 smc_table->dpm2Params.PowerBoostLimit =
1490                         cpu_to_be32(ni_scale_power_for_smc(power_boost_limit, scaling_factor));
1491
1492                 ret = rv770_copy_bytes_to_smc(rdev,
1493                                               (u16)(pi->state_table_start + offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
1494                                                     offsetof(PP_NIslands_DPM2Parameters, TDPLimit)),
1495                                               (u8 *)(&smc_table->dpm2Params.TDPLimit),
1496                                               sizeof(u32) * 4, pi->sram_end);
1497                 if (ret)
1498                         return ret;
1499         }
1500
1501         return 0;
1502 }
1503
1504 int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
1505                                 u32 arb_freq_src, u32 arb_freq_dest)
1506 {
1507         u32 mc_arb_dram_timing;
1508         u32 mc_arb_dram_timing2;
1509         u32 burst_time;
1510         u32 mc_cg_config;
1511
1512         switch (arb_freq_src) {
1513         case MC_CG_ARB_FREQ_F0:
1514                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
1515                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1516                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE0_MASK) >> STATE0_SHIFT;
1517                 break;
1518         case MC_CG_ARB_FREQ_F1:
1519                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_1);
1520                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_1);
1521                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE1_MASK) >> STATE1_SHIFT;
1522                 break;
1523         case MC_CG_ARB_FREQ_F2:
1524                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_2);
1525                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_2);
1526                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE2_MASK) >> STATE2_SHIFT;
1527                 break;
1528         case MC_CG_ARB_FREQ_F3:
1529                 mc_arb_dram_timing  = RREG32(MC_ARB_DRAM_TIMING_3);
1530                 mc_arb_dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2_3);
1531                 burst_time = (RREG32(MC_ARB_BURST_TIME) & STATE3_MASK) >> STATE3_SHIFT;
1532                 break;
1533         default:
1534                 return -EINVAL;
1535         }
1536
1537         switch (arb_freq_dest) {
1538         case MC_CG_ARB_FREQ_F0:
1539                 WREG32(MC_ARB_DRAM_TIMING, mc_arb_dram_timing);
1540                 WREG32(MC_ARB_DRAM_TIMING2, mc_arb_dram_timing2);
1541                 WREG32_P(MC_ARB_BURST_TIME, STATE0(burst_time), ~STATE0_MASK);
1542                 break;
1543         case MC_CG_ARB_FREQ_F1:
1544                 WREG32(MC_ARB_DRAM_TIMING_1, mc_arb_dram_timing);
1545                 WREG32(MC_ARB_DRAM_TIMING2_1, mc_arb_dram_timing2);
1546                 WREG32_P(MC_ARB_BURST_TIME, STATE1(burst_time), ~STATE1_MASK);
1547                 break;
1548         case MC_CG_ARB_FREQ_F2:
1549                 WREG32(MC_ARB_DRAM_TIMING_2, mc_arb_dram_timing);
1550                 WREG32(MC_ARB_DRAM_TIMING2_2, mc_arb_dram_timing2);
1551                 WREG32_P(MC_ARB_BURST_TIME, STATE2(burst_time), ~STATE2_MASK);
1552                 break;
1553         case MC_CG_ARB_FREQ_F3:
1554                 WREG32(MC_ARB_DRAM_TIMING_3, mc_arb_dram_timing);
1555                 WREG32(MC_ARB_DRAM_TIMING2_3, mc_arb_dram_timing2);
1556                 WREG32_P(MC_ARB_BURST_TIME, STATE3(burst_time), ~STATE3_MASK);
1557                 break;
1558         default:
1559                 return -EINVAL;
1560         }
1561
1562         mc_cg_config = RREG32(MC_CG_CONFIG) | 0x0000000F;
1563         WREG32(MC_CG_CONFIG, mc_cg_config);
1564         WREG32_P(MC_ARB_CG, CG_ARB_REQ(arb_freq_dest), ~CG_ARB_REQ_MASK);
1565
1566         return 0;
1567 }
1568
1569 static int ni_init_arb_table_index(struct radeon_device *rdev)
1570 {
1571         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1572         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1573         u32 tmp;
1574         int ret;
1575
1576         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1577                                         &tmp, pi->sram_end);
1578         if (ret)
1579                 return ret;
1580
1581         tmp &= 0x00FFFFFF;
1582         tmp |= ((u32)MC_CG_ARB_FREQ_F1) << 24;
1583
1584         return rv770_write_smc_sram_dword(rdev, ni_pi->arb_table_start,
1585                                           tmp, pi->sram_end);
1586 }
1587
1588 static int ni_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
1589 {
1590         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
1591 }
1592
1593 static int ni_force_switch_to_arb_f0(struct radeon_device *rdev)
1594 {
1595         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1596         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1597         u32 tmp;
1598         int ret;
1599
1600         ret = rv770_read_smc_sram_dword(rdev, ni_pi->arb_table_start,
1601                                         &tmp, pi->sram_end);
1602         if (ret)
1603                 return ret;
1604
1605         tmp = (tmp >> 24) & 0xff;
1606
1607         if (tmp == MC_CG_ARB_FREQ_F0)
1608                 return 0;
1609
1610         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
1611 }
1612
1613 static int ni_populate_memory_timing_parameters(struct radeon_device *rdev,
1614                                                 struct rv7xx_pl *pl,
1615                                                 SMC_NIslands_MCArbDramTimingRegisterSet *arb_regs)
1616 {
1617         u32 dram_timing;
1618         u32 dram_timing2;
1619
1620         arb_regs->mc_arb_rfsh_rate =
1621                 (u8)rv770_calculate_memory_refresh_rate(rdev, pl->sclk);
1622
1623
1624         radeon_atom_set_engine_dram_timings(rdev, pl->sclk, pl->mclk);
1625
1626         dram_timing = RREG32(MC_ARB_DRAM_TIMING);
1627         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
1628
1629         arb_regs->mc_arb_dram_timing  = cpu_to_be32(dram_timing);
1630         arb_regs->mc_arb_dram_timing2 = cpu_to_be32(dram_timing2);
1631
1632         return 0;
1633 }
1634
1635 static int ni_do_program_memory_timing_parameters(struct radeon_device *rdev,
1636                                                   struct radeon_ps *radeon_state,
1637                                                   unsigned int first_arb_set)
1638 {
1639         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1640         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1641         struct ni_ps *state = ni_get_ps(radeon_state);
1642         SMC_NIslands_MCArbDramTimingRegisterSet arb_regs = { 0 };
1643         int i, ret = 0;
1644
1645         for (i = 0; i < state->performance_level_count; i++) {
1646                 ret = ni_populate_memory_timing_parameters(rdev, &state->performance_levels[i], &arb_regs);
1647                 if (ret)
1648                         break;
1649
1650                 ret = rv770_copy_bytes_to_smc(rdev,
1651                                               (u16)(ni_pi->arb_table_start +
1652                                                     offsetof(SMC_NIslands_MCArbDramTimingRegisters, data) +
1653                                                     sizeof(SMC_NIslands_MCArbDramTimingRegisterSet) * (first_arb_set + i)),
1654                                               (u8 *)&arb_regs,
1655                                               (u16)sizeof(SMC_NIslands_MCArbDramTimingRegisterSet),
1656                                               pi->sram_end);
1657                 if (ret)
1658                         break;
1659         }
1660         return ret;
1661 }
1662
1663 static int ni_program_memory_timing_parameters(struct radeon_device *rdev,
1664                                                struct radeon_ps *radeon_new_state)
1665 {
1666         return ni_do_program_memory_timing_parameters(rdev, radeon_new_state,
1667                                                       NISLANDS_DRIVER_STATE_ARB_INDEX);
1668 }
1669
1670 static void ni_populate_initial_mvdd_value(struct radeon_device *rdev,
1671                                            struct NISLANDS_SMC_VOLTAGE_VALUE *voltage)
1672 {
1673         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1674
1675         voltage->index = eg_pi->mvdd_high_index;
1676         voltage->value = cpu_to_be16(MVDD_HIGH_VALUE);
1677 }
1678
1679 static int ni_populate_smc_initial_state(struct radeon_device *rdev,
1680                                          struct radeon_ps *radeon_initial_state,
1681                                          NISLANDS_SMC_STATETABLE *table)
1682 {
1683         struct ni_ps *initial_state = ni_get_ps(radeon_initial_state);
1684         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1685         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1686         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1687         u32 reg;
1688         int ret;
1689
1690         table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL =
1691                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl);
1692         table->initialState.level.mclk.vMPLL_AD_FUNC_CNTL_2 =
1693                 cpu_to_be32(ni_pi->clock_registers.mpll_ad_func_cntl_2);
1694         table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL =
1695                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl);
1696         table->initialState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 =
1697                 cpu_to_be32(ni_pi->clock_registers.mpll_dq_func_cntl_2);
1698         table->initialState.level.mclk.vMCLK_PWRMGT_CNTL =
1699                 cpu_to_be32(ni_pi->clock_registers.mclk_pwrmgt_cntl);
1700         table->initialState.level.mclk.vDLL_CNTL =
1701                 cpu_to_be32(ni_pi->clock_registers.dll_cntl);
1702         table->initialState.level.mclk.vMPLL_SS =
1703                 cpu_to_be32(ni_pi->clock_registers.mpll_ss1);
1704         table->initialState.level.mclk.vMPLL_SS2 =
1705                 cpu_to_be32(ni_pi->clock_registers.mpll_ss2);
1706         table->initialState.level.mclk.mclk_value =
1707                 cpu_to_be32(initial_state->performance_levels[0].mclk);
1708
1709         table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL =
1710                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl);
1711         table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_2 =
1712                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_2);
1713         table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_3 =
1714                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_3);
1715         table->initialState.level.sclk.vCG_SPLL_FUNC_CNTL_4 =
1716                 cpu_to_be32(ni_pi->clock_registers.cg_spll_func_cntl_4);
1717         table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM =
1718                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum);
1719         table->initialState.level.sclk.vCG_SPLL_SPREAD_SPECTRUM_2 =
1720                 cpu_to_be32(ni_pi->clock_registers.cg_spll_spread_spectrum_2);
1721         table->initialState.level.sclk.sclk_value =
1722                 cpu_to_be32(initial_state->performance_levels[0].sclk);
1723         table->initialState.level.arbRefreshState =
1724                 NISLANDS_INITIAL_STATE_ARB_INDEX;
1725
1726         table->initialState.level.ACIndex = 0;
1727
1728         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
1729                                         initial_state->performance_levels[0].vddc,
1730                                         &table->initialState.level.vddc);
1731         if (!ret) {
1732                 u16 std_vddc;
1733
1734                 ret = ni_get_std_voltage_value(rdev,
1735                                                &table->initialState.level.vddc,
1736                                                &std_vddc);
1737                 if (!ret)
1738                         ni_populate_std_voltage_value(rdev, std_vddc,
1739                                                       table->initialState.level.vddc.index,
1740                                                       &table->initialState.level.std_vddc);
1741         }
1742
1743         if (eg_pi->vddci_control)
1744                 ni_populate_voltage_value(rdev,
1745                                           &eg_pi->vddci_voltage_table,
1746                                           initial_state->performance_levels[0].vddci,
1747                                           &table->initialState.level.vddci);
1748
1749         ni_populate_initial_mvdd_value(rdev, &table->initialState.level.mvdd);
1750
1751         reg = CG_R(0xffff) | CG_L(0);
1752         table->initialState.level.aT = cpu_to_be32(reg);
1753
1754         table->initialState.level.bSP = cpu_to_be32(pi->dsp);
1755
1756         if (pi->boot_in_gen2)
1757                 table->initialState.level.gen2PCIE = 1;
1758         else
1759                 table->initialState.level.gen2PCIE = 0;
1760
1761         if (pi->mem_gddr5) {
1762                 table->initialState.level.strobeMode =
1763                         cypress_get_strobe_mode_settings(rdev,
1764                                                          initial_state->performance_levels[0].mclk);
1765
1766                 if (initial_state->performance_levels[0].mclk > pi->mclk_edc_enable_threshold)
1767                         table->initialState.level.mcFlags = NISLANDS_SMC_MC_EDC_RD_FLAG | NISLANDS_SMC_MC_EDC_WR_FLAG;
1768                 else
1769                         table->initialState.level.mcFlags =  0;
1770         }
1771
1772         table->initialState.levelCount = 1;
1773
1774         table->initialState.flags |= PPSMC_SWSTATE_FLAG_DC;
1775
1776         table->initialState.level.dpm2.MaxPS = 0;
1777         table->initialState.level.dpm2.NearTDPDec = 0;
1778         table->initialState.level.dpm2.AboveSafeInc = 0;
1779         table->initialState.level.dpm2.BelowSafeInc = 0;
1780
1781         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1782         table->initialState.level.SQPowerThrottle = cpu_to_be32(reg);
1783
1784         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1785         table->initialState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1786
1787         return 0;
1788 }
1789
1790 static int ni_populate_smc_acpi_state(struct radeon_device *rdev,
1791                                       NISLANDS_SMC_STATETABLE *table)
1792 {
1793         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1794         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
1795         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1796         u32 mpll_ad_func_cntl   = ni_pi->clock_registers.mpll_ad_func_cntl;
1797         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
1798         u32 mpll_dq_func_cntl   = ni_pi->clock_registers.mpll_dq_func_cntl;
1799         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
1800         u32 spll_func_cntl      = ni_pi->clock_registers.cg_spll_func_cntl;
1801         u32 spll_func_cntl_2    = ni_pi->clock_registers.cg_spll_func_cntl_2;
1802         u32 spll_func_cntl_3    = ni_pi->clock_registers.cg_spll_func_cntl_3;
1803         u32 spll_func_cntl_4    = ni_pi->clock_registers.cg_spll_func_cntl_4;
1804         u32 mclk_pwrmgt_cntl    = ni_pi->clock_registers.mclk_pwrmgt_cntl;
1805         u32 dll_cntl            = ni_pi->clock_registers.dll_cntl;
1806         u32 reg;
1807         int ret;
1808
1809         table->ACPIState = table->initialState;
1810
1811         table->ACPIState.flags &= ~PPSMC_SWSTATE_FLAG_DC;
1812
1813         if (pi->acpi_vddc) {
1814                 ret = ni_populate_voltage_value(rdev,
1815                                                 &eg_pi->vddc_voltage_table,
1816                                                 pi->acpi_vddc, &table->ACPIState.level.vddc);
1817                 if (!ret) {
1818                         u16 std_vddc;
1819
1820                         ret = ni_get_std_voltage_value(rdev,
1821                                                        &table->ACPIState.level.vddc, &std_vddc);
1822                         if (!ret)
1823                                 ni_populate_std_voltage_value(rdev, std_vddc,
1824                                                               table->ACPIState.level.vddc.index,
1825                                                               &table->ACPIState.level.std_vddc);
1826                 }
1827
1828                 if (pi->pcie_gen2) {
1829                         if (pi->acpi_pcie_gen2)
1830                                 table->ACPIState.level.gen2PCIE = 1;
1831                         else
1832                                 table->ACPIState.level.gen2PCIE = 0;
1833                 } else {
1834                         table->ACPIState.level.gen2PCIE = 0;
1835                 }
1836         } else {
1837                 ret = ni_populate_voltage_value(rdev,
1838                                                 &eg_pi->vddc_voltage_table,
1839                                                 pi->min_vddc_in_table,
1840                                                 &table->ACPIState.level.vddc);
1841                 if (!ret) {
1842                         u16 std_vddc;
1843
1844                         ret = ni_get_std_voltage_value(rdev,
1845                                                        &table->ACPIState.level.vddc,
1846                                                        &std_vddc);
1847                         if (!ret)
1848                                 ni_populate_std_voltage_value(rdev, std_vddc,
1849                                                               table->ACPIState.level.vddc.index,
1850                                                               &table->ACPIState.level.std_vddc);
1851                 }
1852                 table->ACPIState.level.gen2PCIE = 0;
1853         }
1854
1855         if (eg_pi->acpi_vddci) {
1856                 if (eg_pi->vddci_control)
1857                         ni_populate_voltage_value(rdev,
1858                                                   &eg_pi->vddci_voltage_table,
1859                                                   eg_pi->acpi_vddci,
1860                                                   &table->ACPIState.level.vddci);
1861         }
1862
1863
1864         mpll_ad_func_cntl &= ~PDNB;
1865
1866         mpll_ad_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN;
1867
1868         if (pi->mem_gddr5)
1869                 mpll_dq_func_cntl &= ~PDNB;
1870         mpll_dq_func_cntl_2 |= BIAS_GEN_PDNB | RESET_EN | BYPASS;
1871
1872
1873         mclk_pwrmgt_cntl |= (MRDCKA0_RESET |
1874                              MRDCKA1_RESET |
1875                              MRDCKB0_RESET |
1876                              MRDCKB1_RESET |
1877                              MRDCKC0_RESET |
1878                              MRDCKC1_RESET |
1879                              MRDCKD0_RESET |
1880                              MRDCKD1_RESET);
1881
1882         mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
1883                               MRDCKA1_PDNB |
1884                               MRDCKB0_PDNB |
1885                               MRDCKB1_PDNB |
1886                               MRDCKC0_PDNB |
1887                               MRDCKC1_PDNB |
1888                               MRDCKD0_PDNB |
1889                               MRDCKD1_PDNB);
1890
1891         dll_cntl |= (MRDCKA0_BYPASS |
1892                      MRDCKA1_BYPASS |
1893                      MRDCKB0_BYPASS |
1894                      MRDCKB1_BYPASS |
1895                      MRDCKC0_BYPASS |
1896                      MRDCKC1_BYPASS |
1897                      MRDCKD0_BYPASS |
1898                      MRDCKD1_BYPASS);
1899
1900         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
1901         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
1902
1903         table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
1904         table->ACPIState.level.mclk.vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
1905         table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
1906         table->ACPIState.level.mclk.vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
1907         table->ACPIState.level.mclk.vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
1908         table->ACPIState.level.mclk.vDLL_CNTL = cpu_to_be32(dll_cntl);
1909
1910         table->ACPIState.level.mclk.mclk_value = 0;
1911
1912         table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL = cpu_to_be32(spll_func_cntl);
1913         table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(spll_func_cntl_2);
1914         table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(spll_func_cntl_3);
1915         table->ACPIState.level.sclk.vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(spll_func_cntl_4);
1916
1917         table->ACPIState.level.sclk.sclk_value = 0;
1918
1919         ni_populate_mvdd_value(rdev, 0, &table->ACPIState.level.mvdd);
1920
1921         if (eg_pi->dynamic_ac_timing)
1922                 table->ACPIState.level.ACIndex = 1;
1923
1924         table->ACPIState.level.dpm2.MaxPS = 0;
1925         table->ACPIState.level.dpm2.NearTDPDec = 0;
1926         table->ACPIState.level.dpm2.AboveSafeInc = 0;
1927         table->ACPIState.level.dpm2.BelowSafeInc = 0;
1928
1929         reg = MIN_POWER_MASK | MAX_POWER_MASK;
1930         table->ACPIState.level.SQPowerThrottle = cpu_to_be32(reg);
1931
1932         reg = MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
1933         table->ACPIState.level.SQPowerThrottle_2 = cpu_to_be32(reg);
1934
1935         return 0;
1936 }
1937
1938 static int ni_init_smc_table(struct radeon_device *rdev)
1939 {
1940         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
1941         struct ni_power_info *ni_pi = ni_get_pi(rdev);
1942         int ret;
1943         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
1944         NISLANDS_SMC_STATETABLE *table = &ni_pi->smc_statetable;
1945
1946         memset(table, 0, sizeof(NISLANDS_SMC_STATETABLE));
1947
1948         ni_populate_smc_voltage_tables(rdev, table);
1949
1950         switch (rdev->pm.int_thermal_type) {
1951         case THERMAL_TYPE_NI:
1952         case THERMAL_TYPE_EMC2103_WITH_INTERNAL:
1953                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_INTERNAL;
1954                 break;
1955         case THERMAL_TYPE_NONE:
1956                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_NONE;
1957                 break;
1958         default:
1959                 table->thermalProtectType = PPSMC_THERMAL_PROTECT_TYPE_EXTERNAL;
1960                 break;
1961         }
1962
1963         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
1964                 table->systemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
1965
1966         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1967                 table->systemFlags |= PPSMC_SYSTEMFLAG_REGULATOR_HOT;
1968
1969         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
1970                 table->systemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
1971
1972         if (pi->mem_gddr5)
1973                 table->systemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
1974
1975         ret = ni_populate_smc_initial_state(rdev, radeon_boot_state, table);
1976         if (ret)
1977                 return ret;
1978
1979         ret = ni_populate_smc_acpi_state(rdev, table);
1980         if (ret)
1981                 return ret;
1982
1983         table->driverState.flags = table->initialState.flags;
1984         table->driverState.levelCount = table->initialState.levelCount;
1985         table->driverState.levels[0] = table->initialState.level;
1986
1987         table->ULVState = table->initialState;
1988
1989         ret = ni_do_program_memory_timing_parameters(rdev, radeon_boot_state,
1990                                                      NISLANDS_INITIAL_STATE_ARB_INDEX);
1991         if (ret)
1992                 return ret;
1993
1994         return rv770_copy_bytes_to_smc(rdev, pi->state_table_start, (u8 *)table,
1995                                        sizeof(NISLANDS_SMC_STATETABLE), pi->sram_end);
1996 }
1997
1998 static int ni_calculate_sclk_params(struct radeon_device *rdev,
1999                                     u32 engine_clock,
2000                                     NISLANDS_SMC_SCLK_VALUE *sclk)
2001 {
2002         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2003         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2004         struct atom_clock_dividers dividers;
2005         u32 spll_func_cntl = ni_pi->clock_registers.cg_spll_func_cntl;
2006         u32 spll_func_cntl_2 = ni_pi->clock_registers.cg_spll_func_cntl_2;
2007         u32 spll_func_cntl_3 = ni_pi->clock_registers.cg_spll_func_cntl_3;
2008         u32 spll_func_cntl_4 = ni_pi->clock_registers.cg_spll_func_cntl_4;
2009         u32 cg_spll_spread_spectrum = ni_pi->clock_registers.cg_spll_spread_spectrum;
2010         u32 cg_spll_spread_spectrum_2 = ni_pi->clock_registers.cg_spll_spread_spectrum_2;
2011         u64 tmp;
2012         u32 reference_clock = rdev->clock.spll.reference_freq;
2013         u32 reference_divider;
2014         u32 fbdiv;
2015         int ret;
2016
2017         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
2018                                              engine_clock, false, &dividers);
2019         if (ret)
2020                 return ret;
2021
2022         reference_divider = 1 + dividers.ref_div;
2023
2024
2025         tmp = (u64) engine_clock * reference_divider * dividers.post_div * 16834;
2026         do_div(tmp, reference_clock);
2027         fbdiv = (u32) tmp;
2028
2029         spll_func_cntl &= ~(SPLL_PDIV_A_MASK | SPLL_REF_DIV_MASK);
2030         spll_func_cntl |= SPLL_REF_DIV(dividers.ref_div);
2031         spll_func_cntl |= SPLL_PDIV_A(dividers.post_div);
2032
2033         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2034         spll_func_cntl_2 |= SCLK_MUX_SEL(2);
2035
2036         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
2037         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
2038         spll_func_cntl_3 |= SPLL_DITHEN;
2039
2040         if (pi->sclk_ss) {
2041                 struct radeon_atom_ss ss;
2042                 u32 vco_freq = engine_clock * dividers.post_div;
2043
2044                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2045                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
2046                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
2047                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
2048
2049                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
2050                         cg_spll_spread_spectrum |= CLK_S(clk_s);
2051                         cg_spll_spread_spectrum |= SSEN;
2052
2053                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
2054                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
2055                 }
2056         }
2057
2058         sclk->sclk_value = engine_clock;
2059         sclk->vCG_SPLL_FUNC_CNTL = spll_func_cntl;
2060         sclk->vCG_SPLL_FUNC_CNTL_2 = spll_func_cntl_2;
2061         sclk->vCG_SPLL_FUNC_CNTL_3 = spll_func_cntl_3;
2062         sclk->vCG_SPLL_FUNC_CNTL_4 = spll_func_cntl_4;
2063         sclk->vCG_SPLL_SPREAD_SPECTRUM = cg_spll_spread_spectrum;
2064         sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cg_spll_spread_spectrum_2;
2065
2066         return 0;
2067 }
2068
2069 static int ni_populate_sclk_value(struct radeon_device *rdev,
2070                                   u32 engine_clock,
2071                                   NISLANDS_SMC_SCLK_VALUE *sclk)
2072 {
2073         NISLANDS_SMC_SCLK_VALUE sclk_tmp;
2074         int ret;
2075
2076         ret = ni_calculate_sclk_params(rdev, engine_clock, &sclk_tmp);
2077         if (!ret) {
2078                 sclk->sclk_value = cpu_to_be32(sclk_tmp.sclk_value);
2079                 sclk->vCG_SPLL_FUNC_CNTL = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL);
2080                 sclk->vCG_SPLL_FUNC_CNTL_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_2);
2081                 sclk->vCG_SPLL_FUNC_CNTL_3 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_3);
2082                 sclk->vCG_SPLL_FUNC_CNTL_4 = cpu_to_be32(sclk_tmp.vCG_SPLL_FUNC_CNTL_4);
2083                 sclk->vCG_SPLL_SPREAD_SPECTRUM = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM);
2084                 sclk->vCG_SPLL_SPREAD_SPECTRUM_2 = cpu_to_be32(sclk_tmp.vCG_SPLL_SPREAD_SPECTRUM_2);
2085         }
2086
2087         return ret;
2088 }
2089
2090 static int ni_init_smc_spll_table(struct radeon_device *rdev)
2091 {
2092         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2093         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2094         SMC_NISLANDS_SPLL_DIV_TABLE *spll_table;
2095         NISLANDS_SMC_SCLK_VALUE sclk_params;
2096         u32 fb_div;
2097         u32 p_div;
2098         u32 clk_s;
2099         u32 clk_v;
2100         u32 sclk = 0;
2101         int i, ret;
2102         u32 tmp;
2103
2104         if (ni_pi->spll_table_start == 0)
2105                 return -EINVAL;
2106
2107         spll_table = kzalloc(sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), GFP_KERNEL);
2108         if (spll_table == NULL)
2109                 return -ENOMEM;
2110
2111         for (i = 0; i < 256; i++) {
2112                 ret = ni_calculate_sclk_params(rdev, sclk, &sclk_params);
2113                 if (ret)
2114                         break;
2115
2116                 p_div = (sclk_params.vCG_SPLL_FUNC_CNTL & SPLL_PDIV_A_MASK) >> SPLL_PDIV_A_SHIFT;
2117                 fb_div = (sclk_params.vCG_SPLL_FUNC_CNTL_3 & SPLL_FB_DIV_MASK) >> SPLL_FB_DIV_SHIFT;
2118                 clk_s = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM & CLK_S_MASK) >> CLK_S_SHIFT;
2119                 clk_v = (sclk_params.vCG_SPLL_SPREAD_SPECTRUM_2 & CLK_V_MASK) >> CLK_V_SHIFT;
2120
2121                 fb_div &= ~0x00001FFF;
2122                 fb_div >>= 1;
2123                 clk_v >>= 6;
2124
2125                 if (p_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT))
2126                         ret = -EINVAL;
2127
2128                 if (clk_s & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT))
2129                         ret = -EINVAL;
2130
2131                 if (fb_div & ~(SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT))
2132                         ret = -EINVAL;
2133
2134                 if (clk_v & ~(SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK >> SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT))
2135                         ret = -EINVAL;
2136
2137                 if (ret)
2138                         break;
2139
2140                 tmp = ((fb_div << SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_FBDIV_MASK) |
2141                         ((p_div << SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_PDIV_MASK);
2142                 spll_table->freq[i] = cpu_to_be32(tmp);
2143
2144                 tmp = ((clk_v << SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKV_MASK) |
2145                         ((clk_s << SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_SHIFT) & SMC_NISLANDS_SPLL_DIV_TABLE_CLKS_MASK);
2146                 spll_table->ss[i] = cpu_to_be32(tmp);
2147
2148                 sclk += 512;
2149         }
2150
2151         if (!ret)
2152                 ret = rv770_copy_bytes_to_smc(rdev, ni_pi->spll_table_start, (u8 *)spll_table,
2153                                               sizeof(SMC_NISLANDS_SPLL_DIV_TABLE), pi->sram_end);
2154
2155         kfree(spll_table);
2156
2157         return ret;
2158 }
2159
2160 static int ni_populate_mclk_value(struct radeon_device *rdev,
2161                                   u32 engine_clock,
2162                                   u32 memory_clock,
2163                                   NISLANDS_SMC_MCLK_VALUE *mclk,
2164                                   bool strobe_mode,
2165                                   bool dll_state_on)
2166 {
2167         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2168         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2169         u32 mpll_ad_func_cntl = ni_pi->clock_registers.mpll_ad_func_cntl;
2170         u32 mpll_ad_func_cntl_2 = ni_pi->clock_registers.mpll_ad_func_cntl_2;
2171         u32 mpll_dq_func_cntl = ni_pi->clock_registers.mpll_dq_func_cntl;
2172         u32 mpll_dq_func_cntl_2 = ni_pi->clock_registers.mpll_dq_func_cntl_2;
2173         u32 mclk_pwrmgt_cntl = ni_pi->clock_registers.mclk_pwrmgt_cntl;
2174         u32 dll_cntl = ni_pi->clock_registers.dll_cntl;
2175         u32 mpll_ss1 = ni_pi->clock_registers.mpll_ss1;
2176         u32 mpll_ss2 = ni_pi->clock_registers.mpll_ss2;
2177         struct atom_clock_dividers dividers;
2178         u32 ibias;
2179         u32 dll_speed;
2180         int ret;
2181         u32 mc_seq_misc7;
2182
2183         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_MEMORY_PLL_PARAM,
2184                                              memory_clock, strobe_mode, &dividers);
2185         if (ret)
2186                 return ret;
2187
2188         if (!strobe_mode) {
2189                 mc_seq_misc7 = RREG32(MC_SEQ_MISC7);
2190
2191                 if (mc_seq_misc7 & 0x8000000)
2192                         dividers.post_div = 1;
2193         }
2194
2195         ibias = cypress_map_clkf_to_ibias(rdev, dividers.whole_fb_div);
2196
2197         mpll_ad_func_cntl &= ~(CLKR_MASK |
2198                                YCLK_POST_DIV_MASK |
2199                                CLKF_MASK |
2200                                CLKFRAC_MASK |
2201                                IBIAS_MASK);
2202         mpll_ad_func_cntl |= CLKR(dividers.ref_div);
2203         mpll_ad_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2204         mpll_ad_func_cntl |= CLKF(dividers.whole_fb_div);
2205         mpll_ad_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2206         mpll_ad_func_cntl |= IBIAS(ibias);
2207
2208         if (dividers.vco_mode)
2209                 mpll_ad_func_cntl_2 |= VCO_MODE;
2210         else
2211                 mpll_ad_func_cntl_2 &= ~VCO_MODE;
2212
2213         if (pi->mem_gddr5) {
2214                 mpll_dq_func_cntl &= ~(CLKR_MASK |
2215                                        YCLK_POST_DIV_MASK |
2216                                        CLKF_MASK |
2217                                        CLKFRAC_MASK |
2218                                        IBIAS_MASK);
2219                 mpll_dq_func_cntl |= CLKR(dividers.ref_div);
2220                 mpll_dq_func_cntl |= YCLK_POST_DIV(dividers.post_div);
2221                 mpll_dq_func_cntl |= CLKF(dividers.whole_fb_div);
2222                 mpll_dq_func_cntl |= CLKFRAC(dividers.frac_fb_div);
2223                 mpll_dq_func_cntl |= IBIAS(ibias);
2224
2225                 if (strobe_mode)
2226                         mpll_dq_func_cntl &= ~PDNB;
2227                 else
2228                         mpll_dq_func_cntl |= PDNB;
2229
2230                 if (dividers.vco_mode)
2231                         mpll_dq_func_cntl_2 |= VCO_MODE;
2232                 else
2233                         mpll_dq_func_cntl_2 &= ~VCO_MODE;
2234         }
2235
2236         if (pi->mclk_ss) {
2237                 struct radeon_atom_ss ss;
2238                 u32 vco_freq = memory_clock * dividers.post_div;
2239
2240                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2241                                                      ASIC_INTERNAL_MEMORY_SS, vco_freq)) {
2242                         u32 reference_clock = rdev->clock.mpll.reference_freq;
2243                         u32 decoded_ref = rv740_get_decoded_reference_divider(dividers.ref_div);
2244                         u32 clk_s, clk_v;
2245
2246                         if (!decoded_ref)
2247                                 return -EINVAL;
2248                         clk_s = reference_clock * 5 / (decoded_ref * ss.rate);
2249                         clk_v = ss.percentage *
2250                                 (0x4000 * dividers.whole_fb_div + 0x800 * dividers.frac_fb_div) / (clk_s * 625);
2251
2252                         mpll_ss1 &= ~CLKV_MASK;
2253                         mpll_ss1 |= CLKV(clk_v);
2254
2255                         mpll_ss2 &= ~CLKS_MASK;
2256                         mpll_ss2 |= CLKS(clk_s);
2257                 }
2258         }
2259
2260         dll_speed = rv740_get_dll_speed(pi->mem_gddr5,
2261                                         memory_clock);
2262
2263         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2264         mclk_pwrmgt_cntl |= DLL_SPEED(dll_speed);
2265         if (dll_state_on)
2266                 mclk_pwrmgt_cntl |= (MRDCKA0_PDNB |
2267                                      MRDCKA1_PDNB |
2268                                      MRDCKB0_PDNB |
2269                                      MRDCKB1_PDNB |
2270                                      MRDCKC0_PDNB |
2271                                      MRDCKC1_PDNB |
2272                                      MRDCKD0_PDNB |
2273                                      MRDCKD1_PDNB);
2274         else
2275                 mclk_pwrmgt_cntl &= ~(MRDCKA0_PDNB |
2276                                       MRDCKA1_PDNB |
2277                                       MRDCKB0_PDNB |
2278                                       MRDCKB1_PDNB |
2279                                       MRDCKC0_PDNB |
2280                                       MRDCKC1_PDNB |
2281                                       MRDCKD0_PDNB |
2282                                       MRDCKD1_PDNB);
2283
2284
2285         mclk->mclk_value = cpu_to_be32(memory_clock);
2286         mclk->vMPLL_AD_FUNC_CNTL = cpu_to_be32(mpll_ad_func_cntl);
2287         mclk->vMPLL_AD_FUNC_CNTL_2 = cpu_to_be32(mpll_ad_func_cntl_2);
2288         mclk->vMPLL_DQ_FUNC_CNTL = cpu_to_be32(mpll_dq_func_cntl);
2289         mclk->vMPLL_DQ_FUNC_CNTL_2 = cpu_to_be32(mpll_dq_func_cntl_2);
2290         mclk->vMCLK_PWRMGT_CNTL = cpu_to_be32(mclk_pwrmgt_cntl);
2291         mclk->vDLL_CNTL = cpu_to_be32(dll_cntl);
2292         mclk->vMPLL_SS = cpu_to_be32(mpll_ss1);
2293         mclk->vMPLL_SS2 = cpu_to_be32(mpll_ss2);
2294
2295         return 0;
2296 }
2297
2298 static void ni_populate_smc_sp(struct radeon_device *rdev,
2299                                struct radeon_ps *radeon_state,
2300                                NISLANDS_SMC_SWSTATE *smc_state)
2301 {
2302         struct ni_ps *ps = ni_get_ps(radeon_state);
2303         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2304         int i;
2305
2306         for (i = 0; i < ps->performance_level_count - 1; i++)
2307                 smc_state->levels[i].bSP = cpu_to_be32(pi->dsp);
2308
2309         smc_state->levels[ps->performance_level_count - 1].bSP =
2310                 cpu_to_be32(pi->psp);
2311 }
2312
2313 static int ni_convert_power_level_to_smc(struct radeon_device *rdev,
2314                                          struct rv7xx_pl *pl,
2315                                          NISLANDS_SMC_HW_PERFORMANCE_LEVEL *level)
2316 {
2317         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2318         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2319         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2320         int ret;
2321         bool dll_state_on;
2322         u16 std_vddc;
2323         u32 tmp = RREG32(DC_STUTTER_CNTL);
2324
2325         level->gen2PCIE = pi->pcie_gen2 ?
2326                 ((pl->flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2) ? 1 : 0) : 0;
2327
2328         ret = ni_populate_sclk_value(rdev, pl->sclk, &level->sclk);
2329         if (ret)
2330                 return ret;
2331
2332         level->mcFlags =  0;
2333         if (pi->mclk_stutter_mode_threshold &&
2334             (pl->mclk <= pi->mclk_stutter_mode_threshold) &&
2335             !eg_pi->uvd_enabled &&
2336             (tmp & DC_STUTTER_ENABLE_A) &&
2337             (tmp & DC_STUTTER_ENABLE_B))
2338                 level->mcFlags |= NISLANDS_SMC_MC_STUTTER_EN;
2339
2340         if (pi->mem_gddr5) {
2341                 if (pl->mclk > pi->mclk_edc_enable_threshold)
2342                         level->mcFlags |= NISLANDS_SMC_MC_EDC_RD_FLAG;
2343                 if (pl->mclk > eg_pi->mclk_edc_wr_enable_threshold)
2344                         level->mcFlags |= NISLANDS_SMC_MC_EDC_WR_FLAG;
2345
2346                 level->strobeMode = cypress_get_strobe_mode_settings(rdev, pl->mclk);
2347
2348                 if (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) {
2349                         if (cypress_get_mclk_frequency_ratio(rdev, pl->mclk, true) >=
2350                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2351                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2352                         else
2353                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2354                 } else {
2355                         dll_state_on = false;
2356                         if (pl->mclk > ni_pi->mclk_rtt_mode_threshold)
2357                                 level->mcFlags |= NISLANDS_SMC_MC_RTT_ENABLE;
2358                 }
2359
2360                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk,
2361                                              &level->mclk,
2362                                              (level->strobeMode & NISLANDS_SMC_STROBE_ENABLE) != 0,
2363                                              dll_state_on);
2364         } else
2365                 ret = ni_populate_mclk_value(rdev, pl->sclk, pl->mclk, &level->mclk, 1, 1);
2366
2367         if (ret)
2368                 return ret;
2369
2370         ret = ni_populate_voltage_value(rdev, &eg_pi->vddc_voltage_table,
2371                                         pl->vddc, &level->vddc);
2372         if (ret)
2373                 return ret;
2374
2375         ret = ni_get_std_voltage_value(rdev, &level->vddc, &std_vddc);
2376         if (ret)
2377                 return ret;
2378
2379         ni_populate_std_voltage_value(rdev, std_vddc,
2380                                       level->vddc.index, &level->std_vddc);
2381
2382         if (eg_pi->vddci_control) {
2383                 ret = ni_populate_voltage_value(rdev, &eg_pi->vddci_voltage_table,
2384                                                 pl->vddci, &level->vddci);
2385                 if (ret)
2386                         return ret;
2387         }
2388
2389         ni_populate_mvdd_value(rdev, pl->mclk, &level->mvdd);
2390
2391         return ret;
2392 }
2393
2394 static int ni_populate_smc_t(struct radeon_device *rdev,
2395                              struct radeon_ps *radeon_state,
2396                              NISLANDS_SMC_SWSTATE *smc_state)
2397 {
2398         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2399         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2400         struct ni_ps *state = ni_get_ps(radeon_state);
2401         u32 a_t;
2402         u32 t_l, t_h;
2403         u32 high_bsp;
2404         int i, ret;
2405
2406         if (state->performance_level_count >= 9)
2407                 return -EINVAL;
2408
2409         if (state->performance_level_count < 2) {
2410                 a_t = CG_R(0xffff) | CG_L(0);
2411                 smc_state->levels[0].aT = cpu_to_be32(a_t);
2412                 return 0;
2413         }
2414
2415         smc_state->levels[0].aT = cpu_to_be32(0);
2416
2417         for (i = 0; i <= state->performance_level_count - 2; i++) {
2418                 if (eg_pi->uvd_enabled)
2419                         ret = r600_calculate_at(
2420                                 1000 * (i * (eg_pi->smu_uvd_hs ? 2 : 8) + 2),
2421                                 100 * R600_AH_DFLT,
2422                                 state->performance_levels[i + 1].sclk,
2423                                 state->performance_levels[i].sclk,
2424                                 &t_l,
2425                                 &t_h);
2426                 else
2427                         ret = r600_calculate_at(
2428                                 1000 * (i + 1),
2429                                 100 * R600_AH_DFLT,
2430                                 state->performance_levels[i + 1].sclk,
2431                                 state->performance_levels[i].sclk,
2432                                 &t_l,
2433                                 &t_h);
2434
2435                 if (ret) {
2436                         t_h = (i + 1) * 1000 - 50 * R600_AH_DFLT;
2437                         t_l = (i + 1) * 1000 + 50 * R600_AH_DFLT;
2438                 }
2439
2440                 a_t = be32_to_cpu(smc_state->levels[i].aT) & ~CG_R_MASK;
2441                 a_t |= CG_R(t_l * pi->bsp / 20000);
2442                 smc_state->levels[i].aT = cpu_to_be32(a_t);
2443
2444                 high_bsp = (i == state->performance_level_count - 2) ?
2445                         pi->pbsp : pi->bsp;
2446
2447                 a_t = CG_R(0xffff) | CG_L(t_h * high_bsp / 20000);
2448                 smc_state->levels[i + 1].aT = cpu_to_be32(a_t);
2449         }
2450
2451         return 0;
2452 }
2453
2454 static int ni_populate_power_containment_values(struct radeon_device *rdev,
2455                                                 struct radeon_ps *radeon_state,
2456                                                 NISLANDS_SMC_SWSTATE *smc_state)
2457 {
2458         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2459         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2460         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2461         struct ni_ps *state = ni_get_ps(radeon_state);
2462         u32 prev_sclk;
2463         u32 max_sclk;
2464         u32 min_sclk;
2465         int i, ret;
2466         u32 tdp_limit;
2467         u32 near_tdp_limit;
2468         u32 power_boost_limit;
2469         u8 max_ps_percent;
2470
2471         if (ni_pi->enable_power_containment == false)
2472                 return 0;
2473
2474         if (state->performance_level_count == 0)
2475                 return -EINVAL;
2476
2477         if (smc_state->levelCount != state->performance_level_count)
2478                 return -EINVAL;
2479
2480         ret = ni_calculate_adjusted_tdp_limits(rdev,
2481                                                false, /* ??? */
2482                                                rdev->pm.dpm.tdp_adjustment,
2483                                                &tdp_limit,
2484                                                &near_tdp_limit);
2485         if (ret)
2486                 return ret;
2487
2488         power_boost_limit = ni_calculate_power_boost_limit(rdev, radeon_state, near_tdp_limit);
2489
2490         ret = rv770_write_smc_sram_dword(rdev,
2491                                          pi->state_table_start +
2492                                          offsetof(NISLANDS_SMC_STATETABLE, dpm2Params) +
2493                                          offsetof(PP_NIslands_DPM2Parameters, PowerBoostLimit),
2494                                          ni_scale_power_for_smc(power_boost_limit, ni_get_smc_power_scaling_factor(rdev)),
2495                                          pi->sram_end);
2496         if (ret)
2497                 power_boost_limit = 0;
2498
2499         smc_state->levels[0].dpm2.MaxPS = 0;
2500         smc_state->levels[0].dpm2.NearTDPDec = 0;
2501         smc_state->levels[0].dpm2.AboveSafeInc = 0;
2502         smc_state->levels[0].dpm2.BelowSafeInc = 0;
2503         smc_state->levels[0].stateFlags |= power_boost_limit ? PPSMC_STATEFLAG_POWERBOOST : 0;
2504
2505         for (i = 1; i < state->performance_level_count; i++) {
2506                 prev_sclk = state->performance_levels[i-1].sclk;
2507                 max_sclk  = state->performance_levels[i].sclk;
2508                 max_ps_percent = (i != (state->performance_level_count - 1)) ?
2509                         NISLANDS_DPM2_MAXPS_PERCENT_M : NISLANDS_DPM2_MAXPS_PERCENT_H;
2510
2511                 if (max_sclk < prev_sclk)
2512                         return -EINVAL;
2513
2514                 if ((max_ps_percent == 0) || (prev_sclk == max_sclk) || eg_pi->uvd_enabled)
2515                         min_sclk = max_sclk;
2516                 else if (1 == i)
2517                         min_sclk = prev_sclk;
2518                 else
2519                         min_sclk = (prev_sclk * (u32)max_ps_percent) / 100;
2520
2521                 if (min_sclk < state->performance_levels[0].sclk)
2522                         min_sclk = state->performance_levels[0].sclk;
2523
2524                 if (min_sclk == 0)
2525                         return -EINVAL;
2526
2527                 smc_state->levels[i].dpm2.MaxPS =
2528                         (u8)((NISLANDS_DPM2_MAX_PULSE_SKIP * (max_sclk - min_sclk)) / max_sclk);
2529                 smc_state->levels[i].dpm2.NearTDPDec = NISLANDS_DPM2_NEAR_TDP_DEC;
2530                 smc_state->levels[i].dpm2.AboveSafeInc = NISLANDS_DPM2_ABOVE_SAFE_INC;
2531                 smc_state->levels[i].dpm2.BelowSafeInc = NISLANDS_DPM2_BELOW_SAFE_INC;
2532                 smc_state->levels[i].stateFlags |=
2533                         ((i != (state->performance_level_count - 1)) && power_boost_limit) ?
2534                         PPSMC_STATEFLAG_POWERBOOST : 0;
2535         }
2536
2537         return 0;
2538 }
2539
2540 static int ni_populate_sq_ramping_values(struct radeon_device *rdev,
2541                                          struct radeon_ps *radeon_state,
2542                                          NISLANDS_SMC_SWSTATE *smc_state)
2543 {
2544         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2545         struct ni_ps *state = ni_get_ps(radeon_state);
2546         u32 sq_power_throttle;
2547         u32 sq_power_throttle2;
2548         bool enable_sq_ramping = ni_pi->enable_sq_ramping;
2549         int i;
2550
2551         if (state->performance_level_count == 0)
2552                 return -EINVAL;
2553
2554         if (smc_state->levelCount != state->performance_level_count)
2555                 return -EINVAL;
2556
2557         if (rdev->pm.dpm.sq_ramping_threshold == 0)
2558                 return -EINVAL;
2559
2560         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER > (MAX_POWER_MASK >> MAX_POWER_SHIFT))
2561                 enable_sq_ramping = false;
2562
2563         if (NISLANDS_DPM2_SQ_RAMP_MIN_POWER > (MIN_POWER_MASK >> MIN_POWER_SHIFT))
2564                 enable_sq_ramping = false;
2565
2566         if (NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA > (MAX_POWER_DELTA_MASK >> MAX_POWER_DELTA_SHIFT))
2567                 enable_sq_ramping = false;
2568
2569         if (NISLANDS_DPM2_SQ_RAMP_STI_SIZE > (STI_SIZE_MASK >> STI_SIZE_SHIFT))
2570                 enable_sq_ramping = false;
2571
2572         if (NISLANDS_DPM2_SQ_RAMP_LTI_RATIO > (LTI_RATIO_MASK >> LTI_RATIO_SHIFT))
2573                 enable_sq_ramping = false;
2574
2575         for (i = 0; i < state->performance_level_count; i++) {
2576                 sq_power_throttle  = 0;
2577                 sq_power_throttle2 = 0;
2578
2579                 if ((state->performance_levels[i].sclk >= rdev->pm.dpm.sq_ramping_threshold) &&
2580                     enable_sq_ramping) {
2581                         sq_power_throttle |= MAX_POWER(NISLANDS_DPM2_SQ_RAMP_MAX_POWER);
2582                         sq_power_throttle |= MIN_POWER(NISLANDS_DPM2_SQ_RAMP_MIN_POWER);
2583                         sq_power_throttle2 |= MAX_POWER_DELTA(NISLANDS_DPM2_SQ_RAMP_MAX_POWER_DELTA);
2584                         sq_power_throttle2 |= STI_SIZE(NISLANDS_DPM2_SQ_RAMP_STI_SIZE);
2585                         sq_power_throttle2 |= LTI_RATIO(NISLANDS_DPM2_SQ_RAMP_LTI_RATIO);
2586                 } else {
2587                         sq_power_throttle |= MAX_POWER_MASK | MIN_POWER_MASK;
2588                         sq_power_throttle2 |= MAX_POWER_DELTA_MASK | STI_SIZE_MASK | LTI_RATIO_MASK;
2589                 }
2590
2591                 smc_state->levels[i].SQPowerThrottle   = cpu_to_be32(sq_power_throttle);
2592                 smc_state->levels[i].SQPowerThrottle_2 = cpu_to_be32(sq_power_throttle2);
2593         }
2594
2595         return 0;
2596 }
2597
2598 static int ni_enable_power_containment(struct radeon_device *rdev,
2599                                        struct radeon_ps *radeon_new_state,
2600                                        bool enable)
2601 {
2602         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2603         PPSMC_Result smc_result;
2604         int ret = 0;
2605
2606         if (ni_pi->enable_power_containment) {
2607                 if (enable) {
2608                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
2609                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingActive);
2610                                 if (smc_result != PPSMC_Result_OK) {
2611                                         ret = -EINVAL;
2612                                         ni_pi->pc_enabled = false;
2613                                 } else {
2614                                         ni_pi->pc_enabled = true;
2615                                 }
2616                         }
2617                 } else {
2618                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_TDPClampingInactive);
2619                         if (smc_result != PPSMC_Result_OK)
2620                                 ret = -EINVAL;
2621                         ni_pi->pc_enabled = false;
2622                 }
2623         }
2624
2625         return ret;
2626 }
2627
2628 static int ni_convert_power_state_to_smc(struct radeon_device *rdev,
2629                                          struct radeon_ps *radeon_state,
2630                                          NISLANDS_SMC_SWSTATE *smc_state)
2631 {
2632         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
2633         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2634         struct ni_ps *state = ni_get_ps(radeon_state);
2635         int i, ret;
2636         u32 threshold = state->performance_levels[state->performance_level_count - 1].sclk * 100 / 100;
2637
2638         if (!(radeon_state->caps & ATOM_PPLIB_DISALLOW_ON_DC))
2639                 smc_state->flags |= PPSMC_SWSTATE_FLAG_DC;
2640
2641         smc_state->levelCount = 0;
2642
2643         if (state->performance_level_count > NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE)
2644                 return -EINVAL;
2645
2646         for (i = 0; i < state->performance_level_count; i++) {
2647                 ret = ni_convert_power_level_to_smc(rdev, &state->performance_levels[i],
2648                                                     &smc_state->levels[i]);
2649                 smc_state->levels[i].arbRefreshState =
2650                         (u8)(NISLANDS_DRIVER_STATE_ARB_INDEX + i);
2651
2652                 if (ret)
2653                         return ret;
2654
2655                 if (ni_pi->enable_power_containment)
2656                         smc_state->levels[i].displayWatermark =
2657                                 (state->performance_levels[i].sclk < threshold) ?
2658                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2659                 else
2660                         smc_state->levels[i].displayWatermark = (i < 2) ?
2661                                 PPSMC_DISPLAY_WATERMARK_LOW : PPSMC_DISPLAY_WATERMARK_HIGH;
2662
2663                 if (eg_pi->dynamic_ac_timing)
2664                         smc_state->levels[i].ACIndex = NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i;
2665                 else
2666                         smc_state->levels[i].ACIndex = 0;
2667
2668                 smc_state->levelCount++;
2669         }
2670
2671         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_watermark_threshold,
2672                                       cpu_to_be32(threshold / 512));
2673
2674         ni_populate_smc_sp(rdev, radeon_state, smc_state);
2675
2676         ret = ni_populate_power_containment_values(rdev, radeon_state, smc_state);
2677         if (ret)
2678                 ni_pi->enable_power_containment = false;
2679
2680         ret = ni_populate_sq_ramping_values(rdev, radeon_state, smc_state);
2681         if (ret)
2682                 ni_pi->enable_sq_ramping = false;
2683
2684         return ni_populate_smc_t(rdev, radeon_state, smc_state);
2685 }
2686
2687 static int ni_upload_sw_state(struct radeon_device *rdev,
2688                               struct radeon_ps *radeon_new_state)
2689 {
2690         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2691         u16 address = pi->state_table_start +
2692                 offsetof(NISLANDS_SMC_STATETABLE, driverState);
2693         NISLANDS_SMC_SWSTATE *smc_state;
2694         size_t state_size = struct_size(smc_state, levels,
2695                         NISLANDS_MAX_SMC_PERFORMANCE_LEVELS_PER_SWSTATE);
2696         int ret;
2697
2698         smc_state = kzalloc(state_size, GFP_KERNEL);
2699         if (smc_state == NULL)
2700                 return -ENOMEM;
2701
2702         ret = ni_convert_power_state_to_smc(rdev, radeon_new_state, smc_state);
2703         if (ret)
2704                 goto done;
2705
2706         ret = rv770_copy_bytes_to_smc(rdev, address, (u8 *)smc_state, state_size, pi->sram_end);
2707
2708 done:
2709         kfree(smc_state);
2710
2711         return ret;
2712 }
2713
2714 static int ni_set_mc_special_registers(struct radeon_device *rdev,
2715                                        struct ni_mc_reg_table *table)
2716 {
2717         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2718         u8 i, j, k;
2719         u32 temp_reg;
2720
2721         for (i = 0, j = table->last; i < table->last; i++) {
2722                 switch (table->mc_reg_address[i].s1) {
2723                 case MC_SEQ_MISC1 >> 2:
2724                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2725                                 return -EINVAL;
2726                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
2727                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
2728                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2729                         for (k = 0; k < table->num_entries; k++)
2730                                 table->mc_reg_table_entry[k].mc_data[j] =
2731                                         ((temp_reg & 0xffff0000)) |
2732                                         ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
2733                         j++;
2734                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2735                                 return -EINVAL;
2736
2737                         temp_reg = RREG32(MC_PMG_CMD_MRS);
2738                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
2739                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2740                         for(k = 0; k < table->num_entries; k++) {
2741                                 table->mc_reg_table_entry[k].mc_data[j] =
2742                                         (temp_reg & 0xffff0000) |
2743                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2744                                 if (!pi->mem_gddr5)
2745                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
2746                         }
2747                         j++;
2748                         break;
2749                 case MC_SEQ_RESERVE_M >> 2:
2750                         if (j >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2751                                 return -EINVAL;
2752                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
2753                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
2754                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2755                         for (k = 0; k < table->num_entries; k++)
2756                                 table->mc_reg_table_entry[k].mc_data[j] =
2757                                         (temp_reg & 0xffff0000) |
2758                                         (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
2759                         j++;
2760                         break;
2761                 default:
2762                         break;
2763                 }
2764         }
2765
2766         table->last = j;
2767
2768         return 0;
2769 }
2770
2771 static bool ni_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
2772 {
2773         bool result = true;
2774
2775         switch (in_reg) {
2776         case  MC_SEQ_RAS_TIMING >> 2:
2777                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
2778                 break;
2779         case MC_SEQ_CAS_TIMING >> 2:
2780                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
2781                 break;
2782         case MC_SEQ_MISC_TIMING >> 2:
2783                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
2784                 break;
2785         case MC_SEQ_MISC_TIMING2 >> 2:
2786                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
2787                 break;
2788         case MC_SEQ_RD_CTL_D0 >> 2:
2789                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
2790                 break;
2791         case MC_SEQ_RD_CTL_D1 >> 2:
2792                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
2793                 break;
2794         case MC_SEQ_WR_CTL_D0 >> 2:
2795                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
2796                 break;
2797         case MC_SEQ_WR_CTL_D1 >> 2:
2798                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
2799                 break;
2800         case MC_PMG_CMD_EMRS >> 2:
2801                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
2802                 break;
2803         case MC_PMG_CMD_MRS >> 2:
2804                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
2805                 break;
2806         case MC_PMG_CMD_MRS1 >> 2:
2807                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
2808                 break;
2809         case MC_SEQ_PMG_TIMING >> 2:
2810                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
2811                 break;
2812         case MC_PMG_CMD_MRS2 >> 2:
2813                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
2814                 break;
2815         default:
2816                 result = false;
2817                 break;
2818         }
2819
2820         return result;
2821 }
2822
2823 static void ni_set_valid_flag(struct ni_mc_reg_table *table)
2824 {
2825         u8 i, j;
2826
2827         for (i = 0; i < table->last; i++) {
2828                 for (j = 1; j < table->num_entries; j++) {
2829                         if (table->mc_reg_table_entry[j-1].mc_data[i] != table->mc_reg_table_entry[j].mc_data[i]) {
2830                                 table->valid_flag |= 1 << i;
2831                                 break;
2832                         }
2833                 }
2834         }
2835 }
2836
2837 static void ni_set_s0_mc_reg_index(struct ni_mc_reg_table *table)
2838 {
2839         u32 i;
2840         u16 address;
2841
2842         for (i = 0; i < table->last; i++)
2843                 table->mc_reg_address[i].s0 =
2844                         ni_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
2845                         address : table->mc_reg_address[i].s1;
2846 }
2847
2848 static int ni_copy_vbios_mc_reg_table(struct atom_mc_reg_table *table,
2849                                       struct ni_mc_reg_table *ni_table)
2850 {
2851         u8 i, j;
2852
2853         if (table->last > SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2854                 return -EINVAL;
2855         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
2856                 return -EINVAL;
2857
2858         for (i = 0; i < table->last; i++)
2859                 ni_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
2860         ni_table->last = table->last;
2861
2862         for (i = 0; i < table->num_entries; i++) {
2863                 ni_table->mc_reg_table_entry[i].mclk_max =
2864                         table->mc_reg_table_entry[i].mclk_max;
2865                 for (j = 0; j < table->last; j++)
2866                         ni_table->mc_reg_table_entry[i].mc_data[j] =
2867                                 table->mc_reg_table_entry[i].mc_data[j];
2868         }
2869         ni_table->num_entries = table->num_entries;
2870
2871         return 0;
2872 }
2873
2874 static int ni_initialize_mc_reg_table(struct radeon_device *rdev)
2875 {
2876         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2877         int ret;
2878         struct atom_mc_reg_table *table;
2879         struct ni_mc_reg_table *ni_table = &ni_pi->mc_reg_table;
2880         u8 module_index = rv770_get_memory_module_index(rdev);
2881
2882         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
2883         if (!table)
2884                 return -ENOMEM;
2885
2886         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
2887         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
2888         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
2889         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
2890         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
2891         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
2892         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
2893         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
2894         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
2895         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
2896         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
2897         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
2898         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
2899
2900         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
2901
2902         if (ret)
2903                 goto init_mc_done;
2904
2905         ret = ni_copy_vbios_mc_reg_table(table, ni_table);
2906
2907         if (ret)
2908                 goto init_mc_done;
2909
2910         ni_set_s0_mc_reg_index(ni_table);
2911
2912         ret = ni_set_mc_special_registers(rdev, ni_table);
2913
2914         if (ret)
2915                 goto init_mc_done;
2916
2917         ni_set_valid_flag(ni_table);
2918
2919 init_mc_done:
2920         kfree(table);
2921
2922         return ret;
2923 }
2924
2925 static void ni_populate_mc_reg_addresses(struct radeon_device *rdev,
2926                                          SMC_NIslands_MCRegisters *mc_reg_table)
2927 {
2928         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2929         u32 i, j;
2930
2931         for (i = 0, j = 0; j < ni_pi->mc_reg_table.last; j++) {
2932                 if (ni_pi->mc_reg_table.valid_flag & (1 << j)) {
2933                         if (i >= SMC_NISLANDS_MC_REGISTER_ARRAY_SIZE)
2934                                 break;
2935                         mc_reg_table->address[i].s0 =
2936                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s0);
2937                         mc_reg_table->address[i].s1 =
2938                                 cpu_to_be16(ni_pi->mc_reg_table.mc_reg_address[j].s1);
2939                         i++;
2940                 }
2941         }
2942         mc_reg_table->last = (u8)i;
2943 }
2944
2945
2946 static void ni_convert_mc_registers(struct ni_mc_reg_entry *entry,
2947                                     SMC_NIslands_MCRegisterSet *data,
2948                                     u32 num_entries, u32 valid_flag)
2949 {
2950         u32 i, j;
2951
2952         for (i = 0, j = 0; j < num_entries; j++) {
2953                 if (valid_flag & (1 << j)) {
2954                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
2955                         i++;
2956                 }
2957         }
2958 }
2959
2960 static void ni_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
2961                                                  struct rv7xx_pl *pl,
2962                                                  SMC_NIslands_MCRegisterSet *mc_reg_table_data)
2963 {
2964         struct ni_power_info *ni_pi = ni_get_pi(rdev);
2965         u32 i = 0;
2966
2967         for (i = 0; i < ni_pi->mc_reg_table.num_entries; i++) {
2968                 if (pl->mclk <= ni_pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
2969                         break;
2970         }
2971
2972         if ((i == ni_pi->mc_reg_table.num_entries) && (i > 0))
2973                 --i;
2974
2975         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[i],
2976                                 mc_reg_table_data,
2977                                 ni_pi->mc_reg_table.last,
2978                                 ni_pi->mc_reg_table.valid_flag);
2979 }
2980
2981 static void ni_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
2982                                            struct radeon_ps *radeon_state,
2983                                            SMC_NIslands_MCRegisters *mc_reg_table)
2984 {
2985         struct ni_ps *state = ni_get_ps(radeon_state);
2986         int i;
2987
2988         for (i = 0; i < state->performance_level_count; i++) {
2989                 ni_convert_mc_reg_table_entry_to_smc(rdev,
2990                                                      &state->performance_levels[i],
2991                                                      &mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT + i]);
2992         }
2993 }
2994
2995 static int ni_populate_mc_reg_table(struct radeon_device *rdev,
2996                                     struct radeon_ps *radeon_boot_state)
2997 {
2998         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
2999         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3000         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3001         struct ni_ps *boot_state = ni_get_ps(radeon_boot_state);
3002         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3003
3004         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3005
3006         rv770_write_smc_soft_register(rdev, NI_SMC_SOFT_REGISTER_seq_index, 1);
3007
3008         ni_populate_mc_reg_addresses(rdev, mc_reg_table);
3009
3010         ni_convert_mc_reg_table_entry_to_smc(rdev, &boot_state->performance_levels[0],
3011                                              &mc_reg_table->data[0]);
3012
3013         ni_convert_mc_registers(&ni_pi->mc_reg_table.mc_reg_table_entry[0],
3014                                 &mc_reg_table->data[1],
3015                                 ni_pi->mc_reg_table.last,
3016                                 ni_pi->mc_reg_table.valid_flag);
3017
3018         ni_convert_mc_reg_table_to_smc(rdev, radeon_boot_state, mc_reg_table);
3019
3020         return rv770_copy_bytes_to_smc(rdev, eg_pi->mc_reg_table_start,
3021                                        (u8 *)mc_reg_table,
3022                                        sizeof(SMC_NIslands_MCRegisters),
3023                                        pi->sram_end);
3024 }
3025
3026 static int ni_upload_mc_reg_table(struct radeon_device *rdev,
3027                                   struct radeon_ps *radeon_new_state)
3028 {
3029         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3030         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3031         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3032         struct ni_ps *ni_new_state = ni_get_ps(radeon_new_state);
3033         SMC_NIslands_MCRegisters *mc_reg_table = &ni_pi->smc_mc_reg_table;
3034         u16 address;
3035
3036         memset(mc_reg_table, 0, sizeof(SMC_NIslands_MCRegisters));
3037
3038         ni_convert_mc_reg_table_to_smc(rdev, radeon_new_state, mc_reg_table);
3039
3040         address = eg_pi->mc_reg_table_start +
3041                 (u16)offsetof(SMC_NIslands_MCRegisters, data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT]);
3042
3043         return rv770_copy_bytes_to_smc(rdev, address,
3044                                        (u8 *)&mc_reg_table->data[NISLANDS_MCREGISTERTABLE_FIRST_DRIVERSTATE_SLOT],
3045                                        sizeof(SMC_NIslands_MCRegisterSet) * ni_new_state->performance_level_count,
3046                                        pi->sram_end);
3047 }
3048
3049 static int ni_init_driver_calculated_leakage_table(struct radeon_device *rdev,
3050                                                    PP_NIslands_CACTABLES *cac_tables)
3051 {
3052         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3053         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3054         u32 leakage = 0;
3055         unsigned int i, j, table_size;
3056         s32 t;
3057         u32 smc_leakage, max_leakage = 0;
3058         u32 scaling_factor;
3059
3060         table_size = eg_pi->vddc_voltage_table.count;
3061
3062         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3063                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3064
3065         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3066
3067         for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++) {
3068                 for (j = 0; j < table_size; j++) {
3069                         t = (1000 * ((i + 1) * 8));
3070
3071                         if (t < ni_pi->cac_data.leakage_minimum_temperature)
3072                                 t = ni_pi->cac_data.leakage_minimum_temperature;
3073
3074                         ni_calculate_leakage_for_v_and_t(rdev,
3075                                                          &ni_pi->cac_data.leakage_coefficients,
3076                                                          eg_pi->vddc_voltage_table.entries[j].value,
3077                                                          t,
3078                                                          ni_pi->cac_data.i_leakage,
3079                                                          &leakage);
3080
3081                         smc_leakage = ni_scale_power_for_smc(leakage, scaling_factor) / 1000;
3082                         if (smc_leakage > max_leakage)
3083                                 max_leakage = smc_leakage;
3084
3085                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(smc_leakage);
3086                 }
3087         }
3088
3089         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3090                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3091                         cac_tables->cac_lkge_lut[i][j] = cpu_to_be32(max_leakage);
3092         }
3093         return 0;
3094 }
3095
3096 static int ni_init_simplified_leakage_table(struct radeon_device *rdev,
3097                                             PP_NIslands_CACTABLES *cac_tables)
3098 {
3099         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3100         struct radeon_cac_leakage_table *leakage_table =
3101                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3102         u32 i, j, table_size;
3103         u32 smc_leakage, max_leakage = 0;
3104         u32 scaling_factor;
3105
3106         if (!leakage_table)
3107                 return -EINVAL;
3108
3109         table_size = leakage_table->count;
3110
3111         if (eg_pi->vddc_voltage_table.count != table_size)
3112                 table_size = (eg_pi->vddc_voltage_table.count < leakage_table->count) ?
3113                         eg_pi->vddc_voltage_table.count : leakage_table->count;
3114
3115         if (SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES < table_size)
3116                 table_size = SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES;
3117
3118         if (table_size == 0)
3119                 return -EINVAL;
3120
3121         scaling_factor = ni_get_smc_power_scaling_factor(rdev);
3122
3123         for (j = 0; j < table_size; j++) {
3124                 smc_leakage = leakage_table->entries[j].leakage;
3125
3126                 if (smc_leakage > max_leakage)
3127                         max_leakage = smc_leakage;
3128
3129                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3130                         cac_tables->cac_lkge_lut[i][j] =
3131                                 cpu_to_be32(ni_scale_power_for_smc(smc_leakage, scaling_factor));
3132         }
3133
3134         for (j = table_size; j < SMC_NISLANDS_LKGE_LUT_NUM_OF_VOLT_ENTRIES; j++) {
3135                 for (i = 0; i < SMC_NISLANDS_LKGE_LUT_NUM_OF_TEMP_ENTRIES; i++)
3136                         cac_tables->cac_lkge_lut[i][j] =
3137                                 cpu_to_be32(ni_scale_power_for_smc(max_leakage, scaling_factor));
3138         }
3139         return 0;
3140 }
3141
3142 static int ni_initialize_smc_cac_tables(struct radeon_device *rdev)
3143 {
3144         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3145         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3146         PP_NIslands_CACTABLES *cac_tables = NULL;
3147         int i, ret;
3148         u32 reg;
3149
3150         if (ni_pi->enable_cac == false)
3151                 return 0;
3152
3153         cac_tables = kzalloc(sizeof(PP_NIslands_CACTABLES), GFP_KERNEL);
3154         if (!cac_tables)
3155                 return -ENOMEM;
3156
3157         reg = RREG32(CG_CAC_CTRL) & ~(TID_CNT_MASK | TID_UNIT_MASK);
3158         reg |= (TID_CNT(ni_pi->cac_weights->tid_cnt) |
3159                 TID_UNIT(ni_pi->cac_weights->tid_unit));
3160         WREG32(CG_CAC_CTRL, reg);
3161
3162         for (i = 0; i < NISLANDS_DCCAC_MAX_LEVELS; i++)
3163                 ni_pi->dc_cac_table[i] = ni_pi->cac_weights->dc_cac[i];
3164
3165         for (i = 0; i < SMC_NISLANDS_BIF_LUT_NUM_OF_ENTRIES; i++)
3166                 cac_tables->cac_bif_lut[i] = ni_pi->cac_weights->pcie_cac[i];
3167
3168         ni_pi->cac_data.i_leakage = rdev->pm.dpm.cac_leakage;
3169         ni_pi->cac_data.pwr_const = 0;
3170         ni_pi->cac_data.dc_cac_value = ni_pi->dc_cac_table[NISLANDS_DCCAC_LEVEL_0];
3171         ni_pi->cac_data.bif_cac_value = 0;
3172         ni_pi->cac_data.mc_wr_weight = ni_pi->cac_weights->mc_write_weight;
3173         ni_pi->cac_data.mc_rd_weight = ni_pi->cac_weights->mc_read_weight;
3174         ni_pi->cac_data.allow_ovrflw = 0;
3175         ni_pi->cac_data.l2num_win_tdp = ni_pi->lta_window_size;
3176         ni_pi->cac_data.num_win_tdp = 0;
3177         ni_pi->cac_data.lts_truncate_n = ni_pi->lts_truncate;
3178
3179         if (ni_pi->driver_calculate_cac_leakage)
3180                 ret = ni_init_driver_calculated_leakage_table(rdev, cac_tables);
3181         else
3182                 ret = ni_init_simplified_leakage_table(rdev, cac_tables);
3183
3184         if (ret)
3185                 goto done_free;
3186
3187         cac_tables->pwr_const      = cpu_to_be32(ni_pi->cac_data.pwr_const);
3188         cac_tables->dc_cacValue    = cpu_to_be32(ni_pi->cac_data.dc_cac_value);
3189         cac_tables->bif_cacValue   = cpu_to_be32(ni_pi->cac_data.bif_cac_value);
3190         cac_tables->AllowOvrflw    = ni_pi->cac_data.allow_ovrflw;
3191         cac_tables->MCWrWeight     = ni_pi->cac_data.mc_wr_weight;
3192         cac_tables->MCRdWeight     = ni_pi->cac_data.mc_rd_weight;
3193         cac_tables->numWin_TDP     = ni_pi->cac_data.num_win_tdp;
3194         cac_tables->l2numWin_TDP   = ni_pi->cac_data.l2num_win_tdp;
3195         cac_tables->lts_truncate_n = ni_pi->cac_data.lts_truncate_n;
3196
3197         ret = rv770_copy_bytes_to_smc(rdev, ni_pi->cac_table_start, (u8 *)cac_tables,
3198                                       sizeof(PP_NIslands_CACTABLES), pi->sram_end);
3199
3200 done_free:
3201         if (ret) {
3202                 ni_pi->enable_cac = false;
3203                 ni_pi->enable_power_containment = false;
3204         }
3205
3206         kfree(cac_tables);
3207
3208         return 0;
3209 }
3210
3211 static int ni_initialize_hardware_cac_manager(struct radeon_device *rdev)
3212 {
3213         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3214         u32 reg;
3215
3216         if (!ni_pi->enable_cac ||
3217             !ni_pi->cac_configuration_required)
3218                 return 0;
3219
3220         if (ni_pi->cac_weights == NULL)
3221                 return -EINVAL;
3222
3223         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_0) & ~(WEIGHT_TCP_SIG0_MASK |
3224                                                       WEIGHT_TCP_SIG1_MASK |
3225                                                       WEIGHT_TA_SIG_MASK);
3226         reg |= (WEIGHT_TCP_SIG0(ni_pi->cac_weights->weight_tcp_sig0) |
3227                 WEIGHT_TCP_SIG1(ni_pi->cac_weights->weight_tcp_sig1) |
3228                 WEIGHT_TA_SIG(ni_pi->cac_weights->weight_ta_sig));
3229         WREG32_CG(CG_CAC_REGION_1_WEIGHT_0, reg);
3230
3231         reg = RREG32_CG(CG_CAC_REGION_1_WEIGHT_1) & ~(WEIGHT_TCC_EN0_MASK |
3232                                                       WEIGHT_TCC_EN1_MASK |
3233                                                       WEIGHT_TCC_EN2_MASK);
3234         reg |= (WEIGHT_TCC_EN0(ni_pi->cac_weights->weight_tcc_en0) |
3235                 WEIGHT_TCC_EN1(ni_pi->cac_weights->weight_tcc_en1) |
3236                 WEIGHT_TCC_EN2(ni_pi->cac_weights->weight_tcc_en2));
3237         WREG32_CG(CG_CAC_REGION_1_WEIGHT_1, reg);
3238
3239         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_0) & ~(WEIGHT_CB_EN0_MASK |
3240                                                       WEIGHT_CB_EN1_MASK |
3241                                                       WEIGHT_CB_EN2_MASK |
3242                                                       WEIGHT_CB_EN3_MASK);
3243         reg |= (WEIGHT_CB_EN0(ni_pi->cac_weights->weight_cb_en0) |
3244                 WEIGHT_CB_EN1(ni_pi->cac_weights->weight_cb_en1) |
3245                 WEIGHT_CB_EN2(ni_pi->cac_weights->weight_cb_en2) |
3246                 WEIGHT_CB_EN3(ni_pi->cac_weights->weight_cb_en3));
3247         WREG32_CG(CG_CAC_REGION_2_WEIGHT_0, reg);
3248
3249         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_1) & ~(WEIGHT_DB_SIG0_MASK |
3250                                                       WEIGHT_DB_SIG1_MASK |
3251                                                       WEIGHT_DB_SIG2_MASK |
3252                                                       WEIGHT_DB_SIG3_MASK);
3253         reg |= (WEIGHT_DB_SIG0(ni_pi->cac_weights->weight_db_sig0) |
3254                 WEIGHT_DB_SIG1(ni_pi->cac_weights->weight_db_sig1) |
3255                 WEIGHT_DB_SIG2(ni_pi->cac_weights->weight_db_sig2) |
3256                 WEIGHT_DB_SIG3(ni_pi->cac_weights->weight_db_sig3));
3257         WREG32_CG(CG_CAC_REGION_2_WEIGHT_1, reg);
3258
3259         reg = RREG32_CG(CG_CAC_REGION_2_WEIGHT_2) & ~(WEIGHT_SXM_SIG0_MASK |
3260                                                       WEIGHT_SXM_SIG1_MASK |
3261                                                       WEIGHT_SXM_SIG2_MASK |
3262                                                       WEIGHT_SXS_SIG0_MASK |
3263                                                       WEIGHT_SXS_SIG1_MASK);
3264         reg |= (WEIGHT_SXM_SIG0(ni_pi->cac_weights->weight_sxm_sig0) |
3265                 WEIGHT_SXM_SIG1(ni_pi->cac_weights->weight_sxm_sig1) |
3266                 WEIGHT_SXM_SIG2(ni_pi->cac_weights->weight_sxm_sig2) |
3267                 WEIGHT_SXS_SIG0(ni_pi->cac_weights->weight_sxs_sig0) |
3268                 WEIGHT_SXS_SIG1(ni_pi->cac_weights->weight_sxs_sig1));
3269         WREG32_CG(CG_CAC_REGION_2_WEIGHT_2, reg);
3270
3271         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_0) & ~(WEIGHT_XBR_0_MASK |
3272                                                       WEIGHT_XBR_1_MASK |
3273                                                       WEIGHT_XBR_2_MASK |
3274                                                       WEIGHT_SPI_SIG0_MASK);
3275         reg |= (WEIGHT_XBR_0(ni_pi->cac_weights->weight_xbr_0) |
3276                 WEIGHT_XBR_1(ni_pi->cac_weights->weight_xbr_1) |
3277                 WEIGHT_XBR_2(ni_pi->cac_weights->weight_xbr_2) |
3278                 WEIGHT_SPI_SIG0(ni_pi->cac_weights->weight_spi_sig0));
3279         WREG32_CG(CG_CAC_REGION_3_WEIGHT_0, reg);
3280
3281         reg = RREG32_CG(CG_CAC_REGION_3_WEIGHT_1) & ~(WEIGHT_SPI_SIG1_MASK |
3282                                                       WEIGHT_SPI_SIG2_MASK |
3283                                                       WEIGHT_SPI_SIG3_MASK |
3284                                                       WEIGHT_SPI_SIG4_MASK |
3285                                                       WEIGHT_SPI_SIG5_MASK);
3286         reg |= (WEIGHT_SPI_SIG1(ni_pi->cac_weights->weight_spi_sig1) |
3287                 WEIGHT_SPI_SIG2(ni_pi->cac_weights->weight_spi_sig2) |
3288                 WEIGHT_SPI_SIG3(ni_pi->cac_weights->weight_spi_sig3) |
3289                 WEIGHT_SPI_SIG4(ni_pi->cac_weights->weight_spi_sig4) |
3290                 WEIGHT_SPI_SIG5(ni_pi->cac_weights->weight_spi_sig5));
3291         WREG32_CG(CG_CAC_REGION_3_WEIGHT_1, reg);
3292
3293         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_0) & ~(WEIGHT_LDS_SIG0_MASK |
3294                                                       WEIGHT_LDS_SIG1_MASK |
3295                                                       WEIGHT_SC_MASK);
3296         reg |= (WEIGHT_LDS_SIG0(ni_pi->cac_weights->weight_lds_sig0) |
3297                 WEIGHT_LDS_SIG1(ni_pi->cac_weights->weight_lds_sig1) |
3298                 WEIGHT_SC(ni_pi->cac_weights->weight_sc));
3299         WREG32_CG(CG_CAC_REGION_4_WEIGHT_0, reg);
3300
3301         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_1) & ~(WEIGHT_BIF_MASK |
3302                                                       WEIGHT_CP_MASK |
3303                                                       WEIGHT_PA_SIG0_MASK |
3304                                                       WEIGHT_PA_SIG1_MASK |
3305                                                       WEIGHT_VGT_SIG0_MASK);
3306         reg |= (WEIGHT_BIF(ni_pi->cac_weights->weight_bif) |
3307                 WEIGHT_CP(ni_pi->cac_weights->weight_cp) |
3308                 WEIGHT_PA_SIG0(ni_pi->cac_weights->weight_pa_sig0) |
3309                 WEIGHT_PA_SIG1(ni_pi->cac_weights->weight_pa_sig1) |
3310                 WEIGHT_VGT_SIG0(ni_pi->cac_weights->weight_vgt_sig0));
3311         WREG32_CG(CG_CAC_REGION_4_WEIGHT_1, reg);
3312
3313         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_2) & ~(WEIGHT_VGT_SIG1_MASK |
3314                                                       WEIGHT_VGT_SIG2_MASK |
3315                                                       WEIGHT_DC_SIG0_MASK |
3316                                                       WEIGHT_DC_SIG1_MASK |
3317                                                       WEIGHT_DC_SIG2_MASK);
3318         reg |= (WEIGHT_VGT_SIG1(ni_pi->cac_weights->weight_vgt_sig1) |
3319                 WEIGHT_VGT_SIG2(ni_pi->cac_weights->weight_vgt_sig2) |
3320                 WEIGHT_DC_SIG0(ni_pi->cac_weights->weight_dc_sig0) |
3321                 WEIGHT_DC_SIG1(ni_pi->cac_weights->weight_dc_sig1) |
3322                 WEIGHT_DC_SIG2(ni_pi->cac_weights->weight_dc_sig2));
3323         WREG32_CG(CG_CAC_REGION_4_WEIGHT_2, reg);
3324
3325         reg = RREG32_CG(CG_CAC_REGION_4_WEIGHT_3) & ~(WEIGHT_DC_SIG3_MASK |
3326                                                       WEIGHT_UVD_SIG0_MASK |
3327                                                       WEIGHT_UVD_SIG1_MASK |
3328                                                       WEIGHT_SPARE0_MASK |
3329                                                       WEIGHT_SPARE1_MASK);
3330         reg |= (WEIGHT_DC_SIG3(ni_pi->cac_weights->weight_dc_sig3) |
3331                 WEIGHT_UVD_SIG0(ni_pi->cac_weights->weight_uvd_sig0) |
3332                 WEIGHT_UVD_SIG1(ni_pi->cac_weights->weight_uvd_sig1) |
3333                 WEIGHT_SPARE0(ni_pi->cac_weights->weight_spare0) |
3334                 WEIGHT_SPARE1(ni_pi->cac_weights->weight_spare1));
3335         WREG32_CG(CG_CAC_REGION_4_WEIGHT_3, reg);
3336
3337         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_0) & ~(WEIGHT_SQ_VSP_MASK |
3338                                                       WEIGHT_SQ_VSP0_MASK);
3339         reg |= (WEIGHT_SQ_VSP(ni_pi->cac_weights->weight_sq_vsp) |
3340                 WEIGHT_SQ_VSP0(ni_pi->cac_weights->weight_sq_vsp0));
3341         WREG32_CG(CG_CAC_REGION_5_WEIGHT_0, reg);
3342
3343         reg = RREG32_CG(CG_CAC_REGION_5_WEIGHT_1) & ~(WEIGHT_SQ_GPR_MASK);
3344         reg |= WEIGHT_SQ_GPR(ni_pi->cac_weights->weight_sq_gpr);
3345         WREG32_CG(CG_CAC_REGION_5_WEIGHT_1, reg);
3346
3347         reg = RREG32_CG(CG_CAC_REGION_4_OVERRIDE_4) & ~(OVR_MODE_SPARE_0_MASK |
3348                                                         OVR_VAL_SPARE_0_MASK |
3349                                                         OVR_MODE_SPARE_1_MASK |
3350                                                         OVR_VAL_SPARE_1_MASK);
3351         reg |= (OVR_MODE_SPARE_0(ni_pi->cac_weights->ovr_mode_spare_0) |
3352                 OVR_VAL_SPARE_0(ni_pi->cac_weights->ovr_val_spare_0) |
3353                 OVR_MODE_SPARE_1(ni_pi->cac_weights->ovr_mode_spare_1) |
3354                 OVR_VAL_SPARE_1(ni_pi->cac_weights->ovr_val_spare_1));
3355         WREG32_CG(CG_CAC_REGION_4_OVERRIDE_4, reg);
3356
3357         reg = RREG32(SQ_CAC_THRESHOLD) & ~(VSP_MASK |
3358                                            VSP0_MASK |
3359                                            GPR_MASK);
3360         reg |= (VSP(ni_pi->cac_weights->vsp) |
3361                 VSP0(ni_pi->cac_weights->vsp0) |
3362                 GPR(ni_pi->cac_weights->gpr));
3363         WREG32(SQ_CAC_THRESHOLD, reg);
3364
3365         reg = (MCDW_WR_ENABLE |
3366                MCDX_WR_ENABLE |
3367                MCDY_WR_ENABLE |
3368                MCDZ_WR_ENABLE |
3369                INDEX(0x09D4));
3370         WREG32(MC_CG_CONFIG, reg);
3371
3372         reg = (READ_WEIGHT(ni_pi->cac_weights->mc_read_weight) |
3373                WRITE_WEIGHT(ni_pi->cac_weights->mc_write_weight) |
3374                ALLOW_OVERFLOW);
3375         WREG32(MC_CG_DATAPORT, reg);
3376
3377         return 0;
3378 }
3379
3380 static int ni_enable_smc_cac(struct radeon_device *rdev,
3381                              struct radeon_ps *radeon_new_state,
3382                              bool enable)
3383 {
3384         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3385         int ret = 0;
3386         PPSMC_Result smc_result;
3387
3388         if (ni_pi->enable_cac) {
3389                 if (enable) {
3390                         if (!r600_is_uvd_state(radeon_new_state->class, radeon_new_state->class2)) {
3391                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_CollectCAC_PowerCorreln);
3392
3393                                 if (ni_pi->support_cac_long_term_average) {
3394                                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgEnable);
3395                                         if (PPSMC_Result_OK != smc_result)
3396                                                 ni_pi->support_cac_long_term_average = false;
3397                                 }
3398
3399                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
3400                                 if (PPSMC_Result_OK != smc_result)
3401                                         ret = -EINVAL;
3402
3403                                 ni_pi->cac_enabled = (PPSMC_Result_OK == smc_result) ? true : false;
3404                         }
3405                 } else if (ni_pi->cac_enabled) {
3406                         smc_result = rv770_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
3407
3408                         ni_pi->cac_enabled = false;
3409
3410                         if (ni_pi->support_cac_long_term_average) {
3411                                 smc_result = rv770_send_msg_to_smc(rdev, PPSMC_CACLongTermAvgDisable);
3412                                 if (PPSMC_Result_OK != smc_result)
3413                                         ni_pi->support_cac_long_term_average = false;
3414                         }
3415                 }
3416         }
3417
3418         return ret;
3419 }
3420
3421 static int ni_pcie_performance_request(struct radeon_device *rdev,
3422                                        u8 perf_req, bool advertise)
3423 {
3424 #if defined(CONFIG_ACPI)
3425         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3426
3427         if ((perf_req == PCIE_PERF_REQ_PECI_GEN1) ||
3428             (perf_req == PCIE_PERF_REQ_PECI_GEN2)) {
3429                 if (eg_pi->pcie_performance_request_registered == false)
3430                         radeon_acpi_pcie_notify_device_ready(rdev);
3431                 eg_pi->pcie_performance_request_registered = true;
3432                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3433         } else if ((perf_req == PCIE_PERF_REQ_REMOVE_REGISTRY) &&
3434                     eg_pi->pcie_performance_request_registered) {
3435                 eg_pi->pcie_performance_request_registered = false;
3436                 return radeon_acpi_pcie_performance_request(rdev, perf_req, advertise);
3437         }
3438 #endif
3439         return 0;
3440 }
3441
3442 static int ni_advertise_gen2_capability(struct radeon_device *rdev)
3443 {
3444         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3445         u32 tmp;
3446
3447         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3448
3449         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3450             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2))
3451                 pi->pcie_gen2 = true;
3452         else
3453                 pi->pcie_gen2 = false;
3454
3455         if (!pi->pcie_gen2)
3456                 ni_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, true);
3457
3458         return 0;
3459 }
3460
3461 static void ni_enable_bif_dynamic_pcie_gen2(struct radeon_device *rdev,
3462                                             bool enable)
3463 {
3464         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3465         u32 tmp, bif;
3466
3467         tmp = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
3468
3469         if ((tmp & LC_OTHER_SIDE_EVER_SENT_GEN2) &&
3470             (tmp & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
3471                 if (enable) {
3472                         if (!pi->boot_in_gen2) {
3473                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3474                                 bif |= CG_CLIENT_REQ(0xd);
3475                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3476                         }
3477                         tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3478                         tmp |= LC_HW_VOLTAGE_IF_CONTROL(1);
3479                         tmp |= LC_GEN2_EN_STRAP;
3480
3481                         tmp |= LC_CLR_FAILED_SPD_CHANGE_CNT;
3482                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3483                         udelay(10);
3484                         tmp &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
3485                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3486                 } else {
3487                         if (!pi->boot_in_gen2) {
3488                                 bif = RREG32(CG_BIF_REQ_AND_RSP) & ~CG_CLIENT_REQ_MASK;
3489                                 bif |= CG_CLIENT_REQ(0xd);
3490                                 WREG32(CG_BIF_REQ_AND_RSP, bif);
3491
3492                                 tmp &= ~LC_HW_VOLTAGE_IF_CONTROL_MASK;
3493                                 tmp &= ~LC_GEN2_EN_STRAP;
3494                         }
3495                         WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, tmp);
3496                 }
3497         }
3498 }
3499
3500 static void ni_enable_dynamic_pcie_gen2(struct radeon_device *rdev,
3501                                         bool enable)
3502 {
3503         ni_enable_bif_dynamic_pcie_gen2(rdev, enable);
3504
3505         if (enable)
3506                 WREG32_P(GENERAL_PWRMGT, ENABLE_GEN2PCIE, ~ENABLE_GEN2PCIE);
3507         else
3508                 WREG32_P(GENERAL_PWRMGT, 0, ~ENABLE_GEN2PCIE);
3509 }
3510
3511 void ni_set_uvd_clock_before_set_eng_clock(struct radeon_device *rdev,
3512                                            struct radeon_ps *new_ps,
3513                                            struct radeon_ps *old_ps)
3514 {
3515         struct ni_ps *new_state = ni_get_ps(new_ps);
3516         struct ni_ps *current_state = ni_get_ps(old_ps);
3517
3518         if ((new_ps->vclk == old_ps->vclk) &&
3519             (new_ps->dclk == old_ps->dclk))
3520                 return;
3521
3522         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk >=
3523             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3524                 return;
3525
3526         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3527 }
3528
3529 void ni_set_uvd_clock_after_set_eng_clock(struct radeon_device *rdev,
3530                                           struct radeon_ps *new_ps,
3531                                           struct radeon_ps *old_ps)
3532 {
3533         struct ni_ps *new_state = ni_get_ps(new_ps);
3534         struct ni_ps *current_state = ni_get_ps(old_ps);
3535
3536         if ((new_ps->vclk == old_ps->vclk) &&
3537             (new_ps->dclk == old_ps->dclk))
3538                 return;
3539
3540         if (new_state->performance_levels[new_state->performance_level_count - 1].sclk <
3541             current_state->performance_levels[current_state->performance_level_count - 1].sclk)
3542                 return;
3543
3544         radeon_set_uvd_clocks(rdev, new_ps->vclk, new_ps->dclk);
3545 }
3546
3547 void ni_dpm_setup_asic(struct radeon_device *rdev)
3548 {
3549         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3550         int r;
3551
3552         r = ni_mc_load_microcode(rdev);
3553         if (r)
3554                 DRM_ERROR("Failed to load MC firmware!\n");
3555         ni_read_clock_registers(rdev);
3556         btc_read_arb_registers(rdev);
3557         rv770_get_memory_type(rdev);
3558         if (eg_pi->pcie_performance_request)
3559                 ni_advertise_gen2_capability(rdev);
3560         rv770_get_pcie_gen2_status(rdev);
3561         rv770_enable_acpi_pm(rdev);
3562 }
3563
3564 void ni_update_current_ps(struct radeon_device *rdev,
3565                           struct radeon_ps *rps)
3566 {
3567         struct ni_ps *new_ps = ni_get_ps(rps);
3568         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3569         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3570
3571         eg_pi->current_rps = *rps;
3572         ni_pi->current_ps = *new_ps;
3573         eg_pi->current_rps.ps_priv = &ni_pi->current_ps;
3574 }
3575
3576 void ni_update_requested_ps(struct radeon_device *rdev,
3577                             struct radeon_ps *rps)
3578 {
3579         struct ni_ps *new_ps = ni_get_ps(rps);
3580         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3581         struct ni_power_info *ni_pi = ni_get_pi(rdev);
3582
3583         eg_pi->requested_rps = *rps;
3584         ni_pi->requested_ps = *new_ps;
3585         eg_pi->requested_rps.ps_priv = &ni_pi->requested_ps;
3586 }
3587
3588 int ni_dpm_enable(struct radeon_device *rdev)
3589 {
3590         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3591         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3592         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3593         int ret;
3594
3595         if (pi->gfx_clock_gating)
3596                 ni_cg_clockgating_default(rdev);
3597         if (btc_dpm_enabled(rdev))
3598                 return -EINVAL;
3599         if (pi->mg_clock_gating)
3600                 ni_mg_clockgating_default(rdev);
3601         if (eg_pi->ls_clock_gating)
3602                 ni_ls_clockgating_default(rdev);
3603         if (pi->voltage_control) {
3604                 rv770_enable_voltage_control(rdev, true);
3605                 ret = cypress_construct_voltage_tables(rdev);
3606                 if (ret) {
3607                         DRM_ERROR("cypress_construct_voltage_tables failed\n");
3608                         return ret;
3609                 }
3610         }
3611         if (eg_pi->dynamic_ac_timing) {
3612                 ret = ni_initialize_mc_reg_table(rdev);
3613                 if (ret)
3614                         eg_pi->dynamic_ac_timing = false;
3615         }
3616         if (pi->dynamic_ss)
3617                 cypress_enable_spread_spectrum(rdev, true);
3618         if (pi->thermal_protection)
3619                 rv770_enable_thermal_protection(rdev, true);
3620         rv770_setup_bsp(rdev);
3621         rv770_program_git(rdev);
3622         rv770_program_tp(rdev);
3623         rv770_program_tpp(rdev);
3624         rv770_program_sstp(rdev);
3625         cypress_enable_display_gap(rdev);
3626         rv770_program_vc(rdev);
3627         if (pi->dynamic_pcie_gen2)
3628                 ni_enable_dynamic_pcie_gen2(rdev, true);
3629         ret = rv770_upload_firmware(rdev);
3630         if (ret) {
3631                 DRM_ERROR("rv770_upload_firmware failed\n");
3632                 return ret;
3633         }
3634         ret = ni_process_firmware_header(rdev);
3635         if (ret) {
3636                 DRM_ERROR("ni_process_firmware_header failed\n");
3637                 return ret;
3638         }
3639         ret = ni_initial_switch_from_arb_f0_to_f1(rdev);
3640         if (ret) {
3641                 DRM_ERROR("ni_initial_switch_from_arb_f0_to_f1 failed\n");
3642                 return ret;
3643         }
3644         ret = ni_init_smc_table(rdev);
3645         if (ret) {
3646                 DRM_ERROR("ni_init_smc_table failed\n");
3647                 return ret;
3648         }
3649         ret = ni_init_smc_spll_table(rdev);
3650         if (ret) {
3651                 DRM_ERROR("ni_init_smc_spll_table failed\n");
3652                 return ret;
3653         }
3654         ret = ni_init_arb_table_index(rdev);
3655         if (ret) {
3656                 DRM_ERROR("ni_init_arb_table_index failed\n");
3657                 return ret;
3658         }
3659         if (eg_pi->dynamic_ac_timing) {
3660                 ret = ni_populate_mc_reg_table(rdev, boot_ps);
3661                 if (ret) {
3662                         DRM_ERROR("ni_populate_mc_reg_table failed\n");
3663                         return ret;
3664                 }
3665         }
3666         ret = ni_initialize_smc_cac_tables(rdev);
3667         if (ret) {
3668                 DRM_ERROR("ni_initialize_smc_cac_tables failed\n");
3669                 return ret;
3670         }
3671         ret = ni_initialize_hardware_cac_manager(rdev);
3672         if (ret) {
3673                 DRM_ERROR("ni_initialize_hardware_cac_manager failed\n");
3674                 return ret;
3675         }
3676         ret = ni_populate_smc_tdp_limits(rdev, boot_ps);
3677         if (ret) {
3678                 DRM_ERROR("ni_populate_smc_tdp_limits failed\n");
3679                 return ret;
3680         }
3681         ni_program_response_times(rdev);
3682         r7xx_start_smc(rdev);
3683         ret = cypress_notify_smc_display_change(rdev, false);
3684         if (ret) {
3685                 DRM_ERROR("cypress_notify_smc_display_change failed\n");
3686                 return ret;
3687         }
3688         cypress_enable_sclk_control(rdev, true);
3689         if (eg_pi->memory_transition)
3690                 cypress_enable_mclk_control(rdev, true);
3691         cypress_start_dpm(rdev);
3692         if (pi->gfx_clock_gating)
3693                 ni_gfx_clockgating_enable(rdev, true);
3694         if (pi->mg_clock_gating)
3695                 ni_mg_clockgating_enable(rdev, true);
3696         if (eg_pi->ls_clock_gating)
3697                 ni_ls_clockgating_enable(rdev, true);
3698
3699         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
3700
3701         ni_update_current_ps(rdev, boot_ps);
3702
3703         return 0;
3704 }
3705
3706 void ni_dpm_disable(struct radeon_device *rdev)
3707 {
3708         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3709         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3710         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
3711
3712         if (!btc_dpm_enabled(rdev))
3713                 return;
3714         rv770_clear_vc(rdev);
3715         if (pi->thermal_protection)
3716                 rv770_enable_thermal_protection(rdev, false);
3717         ni_enable_power_containment(rdev, boot_ps, false);
3718         ni_enable_smc_cac(rdev, boot_ps, false);
3719         cypress_enable_spread_spectrum(rdev, false);
3720         rv770_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
3721         if (pi->dynamic_pcie_gen2)
3722                 ni_enable_dynamic_pcie_gen2(rdev, false);
3723
3724         if (rdev->irq.installed &&
3725             r600_is_internal_thermal_sensor(rdev->pm.int_thermal_type)) {
3726                 rdev->irq.dpm_thermal = false;
3727                 radeon_irq_set(rdev);
3728         }
3729
3730         if (pi->gfx_clock_gating)
3731                 ni_gfx_clockgating_enable(rdev, false);
3732         if (pi->mg_clock_gating)
3733                 ni_mg_clockgating_enable(rdev, false);
3734         if (eg_pi->ls_clock_gating)
3735                 ni_ls_clockgating_enable(rdev, false);
3736         ni_stop_dpm(rdev);
3737         btc_reset_to_default(rdev);
3738         ni_stop_smc(rdev);
3739         ni_force_switch_to_arb_f0(rdev);
3740
3741         ni_update_current_ps(rdev, boot_ps);
3742 }
3743
3744 static int ni_power_control_set_level(struct radeon_device *rdev)
3745 {
3746         struct radeon_ps *new_ps = rdev->pm.dpm.requested_ps;
3747         int ret;
3748
3749         ret = ni_restrict_performance_levels_before_switch(rdev);
3750         if (ret)
3751                 return ret;
3752         ret = rv770_halt_smc(rdev);
3753         if (ret)
3754                 return ret;
3755         ret = ni_populate_smc_tdp_limits(rdev, new_ps);
3756         if (ret)
3757                 return ret;
3758         ret = rv770_resume_smc(rdev);
3759         if (ret)
3760                 return ret;
3761         ret = rv770_set_sw_state(rdev);
3762         if (ret)
3763                 return ret;
3764
3765         return 0;
3766 }
3767
3768 int ni_dpm_pre_set_power_state(struct radeon_device *rdev)
3769 {
3770         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3771         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
3772         struct radeon_ps *new_ps = &requested_ps;
3773
3774         ni_update_requested_ps(rdev, new_ps);
3775
3776         ni_apply_state_adjust_rules(rdev, &eg_pi->requested_rps);
3777
3778         return 0;
3779 }
3780
3781 int ni_dpm_set_power_state(struct radeon_device *rdev)
3782 {
3783         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3784         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3785         struct radeon_ps *old_ps = &eg_pi->current_rps;
3786         int ret;
3787
3788         ret = ni_restrict_performance_levels_before_switch(rdev);
3789         if (ret) {
3790                 DRM_ERROR("ni_restrict_performance_levels_before_switch failed\n");
3791                 return ret;
3792         }
3793         ni_set_uvd_clock_before_set_eng_clock(rdev, new_ps, old_ps);
3794         ret = ni_enable_power_containment(rdev, new_ps, false);
3795         if (ret) {
3796                 DRM_ERROR("ni_enable_power_containment failed\n");
3797                 return ret;
3798         }
3799         ret = ni_enable_smc_cac(rdev, new_ps, false);
3800         if (ret) {
3801                 DRM_ERROR("ni_enable_smc_cac failed\n");
3802                 return ret;
3803         }
3804         ret = rv770_halt_smc(rdev);
3805         if (ret) {
3806                 DRM_ERROR("rv770_halt_smc failed\n");
3807                 return ret;
3808         }
3809         if (eg_pi->smu_uvd_hs)
3810                 btc_notify_uvd_to_smc(rdev, new_ps);
3811         ret = ni_upload_sw_state(rdev, new_ps);
3812         if (ret) {
3813                 DRM_ERROR("ni_upload_sw_state failed\n");
3814                 return ret;
3815         }
3816         if (eg_pi->dynamic_ac_timing) {
3817                 ret = ni_upload_mc_reg_table(rdev, new_ps);
3818                 if (ret) {
3819                         DRM_ERROR("ni_upload_mc_reg_table failed\n");
3820                         return ret;
3821                 }
3822         }
3823         ret = ni_program_memory_timing_parameters(rdev, new_ps);
3824         if (ret) {
3825                 DRM_ERROR("ni_program_memory_timing_parameters failed\n");
3826                 return ret;
3827         }
3828         ret = rv770_resume_smc(rdev);
3829         if (ret) {
3830                 DRM_ERROR("rv770_resume_smc failed\n");
3831                 return ret;
3832         }
3833         ret = rv770_set_sw_state(rdev);
3834         if (ret) {
3835                 DRM_ERROR("rv770_set_sw_state failed\n");
3836                 return ret;
3837         }
3838         ni_set_uvd_clock_after_set_eng_clock(rdev, new_ps, old_ps);
3839         ret = ni_enable_smc_cac(rdev, new_ps, true);
3840         if (ret) {
3841                 DRM_ERROR("ni_enable_smc_cac failed\n");
3842                 return ret;
3843         }
3844         ret = ni_enable_power_containment(rdev, new_ps, true);
3845         if (ret) {
3846                 DRM_ERROR("ni_enable_power_containment failed\n");
3847                 return ret;
3848         }
3849
3850         /* update tdp */
3851         ret = ni_power_control_set_level(rdev);
3852         if (ret) {
3853                 DRM_ERROR("ni_power_control_set_level failed\n");
3854                 return ret;
3855         }
3856
3857         return 0;
3858 }
3859
3860 void ni_dpm_post_set_power_state(struct radeon_device *rdev)
3861 {
3862         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3863         struct radeon_ps *new_ps = &eg_pi->requested_rps;
3864
3865         ni_update_current_ps(rdev, new_ps);
3866 }
3867
3868 #if 0
3869 void ni_dpm_reset_asic(struct radeon_device *rdev)
3870 {
3871         ni_restrict_performance_levels_before_switch(rdev);
3872         rv770_set_boot_state(rdev);
3873 }
3874 #endif
3875
3876 union power_info {
3877         struct _ATOM_POWERPLAY_INFO info;
3878         struct _ATOM_POWERPLAY_INFO_V2 info_2;
3879         struct _ATOM_POWERPLAY_INFO_V3 info_3;
3880         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
3881         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
3882         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
3883 };
3884
3885 union pplib_clock_info {
3886         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
3887         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
3888         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
3889         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
3890 };
3891
3892 union pplib_power_state {
3893         struct _ATOM_PPLIB_STATE v1;
3894         struct _ATOM_PPLIB_STATE_V2 v2;
3895 };
3896
3897 static void ni_parse_pplib_non_clock_info(struct radeon_device *rdev,
3898                                           struct radeon_ps *rps,
3899                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
3900                                           u8 table_rev)
3901 {
3902         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
3903         rps->class = le16_to_cpu(non_clock_info->usClassification);
3904         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
3905
3906         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
3907                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
3908                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
3909         } else if (r600_is_uvd_state(rps->class, rps->class2)) {
3910                 rps->vclk = RV770_DEFAULT_VCLK_FREQ;
3911                 rps->dclk = RV770_DEFAULT_DCLK_FREQ;
3912         } else {
3913                 rps->vclk = 0;
3914                 rps->dclk = 0;
3915         }
3916
3917         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
3918                 rdev->pm.dpm.boot_ps = rps;
3919         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
3920                 rdev->pm.dpm.uvd_ps = rps;
3921 }
3922
3923 static void ni_parse_pplib_clock_info(struct radeon_device *rdev,
3924                                       struct radeon_ps *rps, int index,
3925                                       union pplib_clock_info *clock_info)
3926 {
3927         struct rv7xx_power_info *pi = rv770_get_pi(rdev);
3928         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
3929         struct ni_ps *ps = ni_get_ps(rps);
3930         struct rv7xx_pl *pl = &ps->performance_levels[index];
3931
3932         ps->performance_level_count = index + 1;
3933
3934         pl->sclk = le16_to_cpu(clock_info->evergreen.usEngineClockLow);
3935         pl->sclk |= clock_info->evergreen.ucEngineClockHigh << 16;
3936         pl->mclk = le16_to_cpu(clock_info->evergreen.usMemoryClockLow);
3937         pl->mclk |= clock_info->evergreen.ucMemoryClockHigh << 16;
3938
3939         pl->vddc = le16_to_cpu(clock_info->evergreen.usVDDC);
3940         pl->vddci = le16_to_cpu(clock_info->evergreen.usVDDCI);
3941         pl->flags = le32_to_cpu(clock_info->evergreen.ulFlags);
3942
3943         /* patch up vddc if necessary */
3944         if (pl->vddc == 0xff01) {
3945                 if (pi->max_vddc)
3946                         pl->vddc = pi->max_vddc;
3947         }
3948
3949         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
3950                 pi->acpi_vddc = pl->vddc;
3951                 eg_pi->acpi_vddci = pl->vddci;
3952                 if (ps->performance_levels[0].flags & ATOM_PPLIB_R600_FLAGS_PCIEGEN2)
3953                         pi->acpi_pcie_gen2 = true;
3954                 else
3955                         pi->acpi_pcie_gen2 = false;
3956         }
3957
3958         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
3959                 eg_pi->ulv.supported = true;
3960                 eg_pi->ulv.pl = pl;
3961         }
3962
3963         if (pi->min_vddc_in_table > pl->vddc)
3964                 pi->min_vddc_in_table = pl->vddc;
3965
3966         if (pi->max_vddc_in_table < pl->vddc)
3967                 pi->max_vddc_in_table = pl->vddc;
3968
3969         /* patch up boot state */
3970         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
3971                 u16 vddc, vddci, mvdd;
3972                 radeon_atombios_get_default_voltages(rdev, &vddc, &vddci, &mvdd);
3973                 pl->mclk = rdev->clock.default_mclk;
3974                 pl->sclk = rdev->clock.default_sclk;
3975                 pl->vddc = vddc;
3976                 pl->vddci = vddci;
3977         }
3978
3979         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) ==
3980             ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE) {
3981                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk = pl->sclk;
3982                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk = pl->mclk;
3983                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc = pl->vddc;
3984                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci = pl->vddci;
3985         }
3986 }
3987
3988 static int ni_parse_power_table(struct radeon_device *rdev)
3989 {
3990         struct radeon_mode_info *mode_info = &rdev->mode_info;
3991         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
3992         union pplib_power_state *power_state;
3993         int i, j;
3994         union pplib_clock_info *clock_info;
3995         union power_info *power_info;
3996         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
3997         u16 data_offset;
3998         u8 frev, crev;
3999         struct ni_ps *ps;
4000
4001         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
4002                                    &frev, &crev, &data_offset))
4003                 return -EINVAL;
4004         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
4005
4006         rdev->pm.dpm.ps = kcalloc(power_info->pplib.ucNumStates,
4007                                   sizeof(struct radeon_ps),
4008                                   GFP_KERNEL);
4009         if (!rdev->pm.dpm.ps)
4010                 return -ENOMEM;
4011
4012         for (i = 0; i < power_info->pplib.ucNumStates; i++) {
4013                 power_state = (union pplib_power_state *)
4014                         (mode_info->atom_context->bios + data_offset +
4015                          le16_to_cpu(power_info->pplib.usStateArrayOffset) +
4016                          i * power_info->pplib.ucStateEntrySize);
4017                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
4018                         (mode_info->atom_context->bios + data_offset +
4019                          le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset) +
4020                          (power_state->v1.ucNonClockStateIndex *
4021                           power_info->pplib.ucNonClockSize));
4022                 if (power_info->pplib.ucStateEntrySize - 1) {
4023                         u8 *idx;
4024                         ps = kzalloc(sizeof(struct ni_ps), GFP_KERNEL);
4025                         if (ps == NULL) {
4026                                 kfree(rdev->pm.dpm.ps);
4027                                 return -ENOMEM;
4028                         }
4029                         rdev->pm.dpm.ps[i].ps_priv = ps;
4030                         ni_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
4031                                                          non_clock_info,
4032                                                          power_info->pplib.ucNonClockSize);
4033                         idx = (u8 *)&power_state->v1.ucClockStateIndices[0];
4034                         for (j = 0; j < (power_info->pplib.ucStateEntrySize - 1); j++) {
4035                                 clock_info = (union pplib_clock_info *)
4036                                         (mode_info->atom_context->bios + data_offset +
4037                                          le16_to_cpu(power_info->pplib.usClockInfoArrayOffset) +
4038                                          (idx[j] * power_info->pplib.ucClockInfoSize));
4039                                 ni_parse_pplib_clock_info(rdev,
4040                                                           &rdev->pm.dpm.ps[i], j,
4041                                                           clock_info);
4042                         }
4043                 }
4044         }
4045         rdev->pm.dpm.num_ps = power_info->pplib.ucNumStates;
4046         return 0;
4047 }
4048
4049 int ni_dpm_init(struct radeon_device *rdev)
4050 {
4051         struct rv7xx_power_info *pi;
4052         struct evergreen_power_info *eg_pi;
4053         struct ni_power_info *ni_pi;
4054         struct atom_clock_dividers dividers;
4055         int ret;
4056
4057         ni_pi = kzalloc(sizeof(struct ni_power_info), GFP_KERNEL);
4058         if (ni_pi == NULL)
4059                 return -ENOMEM;
4060         rdev->pm.dpm.priv = ni_pi;
4061         eg_pi = &ni_pi->eg;
4062         pi = &eg_pi->rv7xx;
4063
4064         rv770_get_max_vddc(rdev);
4065
4066         eg_pi->ulv.supported = false;
4067         pi->acpi_vddc = 0;
4068         eg_pi->acpi_vddci = 0;
4069         pi->min_vddc_in_table = 0;
4070         pi->max_vddc_in_table = 0;
4071
4072         ret = r600_get_platform_caps(rdev);
4073         if (ret)
4074                 return ret;
4075
4076         ret = ni_parse_power_table(rdev);
4077         if (ret)
4078                 return ret;
4079         ret = r600_parse_extended_power_table(rdev);
4080         if (ret)
4081                 return ret;
4082
4083         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
4084                 kcalloc(4,
4085                         sizeof(struct radeon_clock_voltage_dependency_entry),
4086                         GFP_KERNEL);
4087         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
4088                 r600_free_extended_power_table(rdev);
4089                 return -ENOMEM;
4090         }
4091         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
4092         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
4093         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
4094         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
4095         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
4096         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
4097         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
4098         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
4099         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
4100
4101         ni_patch_dependency_tables_based_on_leakage(rdev);
4102
4103         if (rdev->pm.dpm.voltage_response_time == 0)
4104                 rdev->pm.dpm.voltage_response_time = R600_VOLTAGERESPONSETIME_DFLT;
4105         if (rdev->pm.dpm.backbias_response_time == 0)
4106                 rdev->pm.dpm.backbias_response_time = R600_BACKBIASRESPONSETIME_DFLT;
4107
4108         ret = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
4109                                              0, false, &dividers);
4110         if (ret)
4111                 pi->ref_div = dividers.ref_div + 1;
4112         else
4113                 pi->ref_div = R600_REFERENCEDIVIDER_DFLT;
4114
4115         pi->rlp = RV770_RLP_DFLT;
4116         pi->rmp = RV770_RMP_DFLT;
4117         pi->lhp = RV770_LHP_DFLT;
4118         pi->lmp = RV770_LMP_DFLT;
4119
4120         eg_pi->ats[0].rlp = RV770_RLP_DFLT;
4121         eg_pi->ats[0].rmp = RV770_RMP_DFLT;
4122         eg_pi->ats[0].lhp = RV770_LHP_DFLT;
4123         eg_pi->ats[0].lmp = RV770_LMP_DFLT;
4124
4125         eg_pi->ats[1].rlp = BTC_RLP_UVD_DFLT;
4126         eg_pi->ats[1].rmp = BTC_RMP_UVD_DFLT;
4127         eg_pi->ats[1].lhp = BTC_LHP_UVD_DFLT;
4128         eg_pi->ats[1].lmp = BTC_LMP_UVD_DFLT;
4129
4130         eg_pi->smu_uvd_hs = true;
4131
4132         if (rdev->pdev->device == 0x6707) {
4133                 pi->mclk_strobe_mode_threshold = 55000;
4134                 pi->mclk_edc_enable_threshold = 55000;
4135                 eg_pi->mclk_edc_wr_enable_threshold = 55000;
4136         } else {
4137                 pi->mclk_strobe_mode_threshold = 40000;
4138                 pi->mclk_edc_enable_threshold = 40000;
4139                 eg_pi->mclk_edc_wr_enable_threshold = 40000;
4140         }
4141         ni_pi->mclk_rtt_mode_threshold = eg_pi->mclk_edc_wr_enable_threshold;
4142
4143         pi->voltage_control =
4144                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDC, 0);
4145
4146         pi->mvdd_control =
4147                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_MVDDC, 0);
4148
4149         eg_pi->vddci_control =
4150                 radeon_atom_is_voltage_gpio(rdev, SET_VOLTAGE_TYPE_ASIC_VDDCI, 0);
4151
4152         rv770_get_engine_memory_ss(rdev);
4153
4154         pi->asi = RV770_ASI_DFLT;
4155         pi->pasi = CYPRESS_HASI_DFLT;
4156         pi->vrc = CYPRESS_VRC_DFLT;
4157
4158         pi->power_gating = false;
4159
4160         pi->gfx_clock_gating = true;
4161
4162         pi->mg_clock_gating = true;
4163         pi->mgcgtssm = true;
4164         eg_pi->ls_clock_gating = false;
4165         eg_pi->sclk_deep_sleep = false;
4166
4167         pi->dynamic_pcie_gen2 = true;
4168
4169         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
4170                 pi->thermal_protection = true;
4171         else
4172                 pi->thermal_protection = false;
4173
4174         pi->display_gap = true;
4175
4176         pi->dcodt = true;
4177
4178         pi->ulps = true;
4179
4180         eg_pi->dynamic_ac_timing = true;
4181         eg_pi->abm = true;
4182         eg_pi->mcls = true;
4183         eg_pi->light_sleep = true;
4184         eg_pi->memory_transition = true;
4185 #if defined(CONFIG_ACPI)
4186         eg_pi->pcie_performance_request =
4187                 radeon_acpi_is_pcie_performance_request_supported(rdev);
4188 #else
4189         eg_pi->pcie_performance_request = false;
4190 #endif
4191
4192         eg_pi->dll_default_on = false;
4193
4194         eg_pi->sclk_deep_sleep = false;
4195
4196         pi->mclk_stutter_mode_threshold = 0;
4197
4198         pi->sram_end = SMC_RAM_END;
4199
4200         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 3;
4201         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
4202         rdev->pm.dpm.dyn_state.min_vddc_for_pcie_gen2 = 900;
4203         rdev->pm.dpm.dyn_state.valid_sclk_values.count = ARRAY_SIZE(btc_valid_sclk);
4204         rdev->pm.dpm.dyn_state.valid_sclk_values.values = btc_valid_sclk;
4205         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
4206         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
4207         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 12500;
4208
4209         ni_pi->cac_data.leakage_coefficients.at = 516;
4210         ni_pi->cac_data.leakage_coefficients.bt = 18;
4211         ni_pi->cac_data.leakage_coefficients.av = 51;
4212         ni_pi->cac_data.leakage_coefficients.bv = 2957;
4213
4214         switch (rdev->pdev->device) {
4215         case 0x6700:
4216         case 0x6701:
4217         case 0x6702:
4218         case 0x6703:
4219         case 0x6718:
4220                 ni_pi->cac_weights = &cac_weights_cayman_xt;
4221                 break;
4222         case 0x6705:
4223         case 0x6719:
4224         case 0x671D:
4225         case 0x671C:
4226         default:
4227                 ni_pi->cac_weights = &cac_weights_cayman_pro;
4228                 break;
4229         case 0x6704:
4230         case 0x6706:
4231         case 0x6707:
4232         case 0x6708:
4233         case 0x6709:
4234                 ni_pi->cac_weights = &cac_weights_cayman_le;
4235                 break;
4236         }
4237
4238         if (ni_pi->cac_weights->enable_power_containment_by_default) {
4239                 ni_pi->enable_power_containment = true;
4240                 ni_pi->enable_cac = true;
4241                 ni_pi->enable_sq_ramping = true;
4242         } else {
4243                 ni_pi->enable_power_containment = false;
4244                 ni_pi->enable_cac = false;
4245                 ni_pi->enable_sq_ramping = false;
4246         }
4247
4248         ni_pi->driver_calculate_cac_leakage = false;
4249         ni_pi->cac_configuration_required = true;
4250
4251         if (ni_pi->cac_configuration_required) {
4252                 ni_pi->support_cac_long_term_average = true;
4253                 ni_pi->lta_window_size = ni_pi->cac_weights->l2_lta_window_size;
4254                 ni_pi->lts_truncate = ni_pi->cac_weights->lts_truncate;
4255         } else {
4256                 ni_pi->support_cac_long_term_average = false;
4257                 ni_pi->lta_window_size = 0;
4258                 ni_pi->lts_truncate = 0;
4259         }
4260
4261         ni_pi->use_power_boost_limit = true;
4262
4263         /* make sure dc limits are valid */
4264         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
4265             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
4266                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
4267                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4268
4269         return 0;
4270 }
4271
4272 void ni_dpm_fini(struct radeon_device *rdev)
4273 {
4274         int i;
4275
4276         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
4277                 kfree(rdev->pm.dpm.ps[i].ps_priv);
4278         }
4279         kfree(rdev->pm.dpm.ps);
4280         kfree(rdev->pm.dpm.priv);
4281         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
4282         r600_free_extended_power_table(rdev);
4283 }
4284
4285 void ni_dpm_print_power_state(struct radeon_device *rdev,
4286                               struct radeon_ps *rps)
4287 {
4288         struct ni_ps *ps = ni_get_ps(rps);
4289         struct rv7xx_pl *pl;
4290         int i;
4291
4292         r600_dpm_print_class_info(rps->class, rps->class2);
4293         r600_dpm_print_cap_info(rps->caps);
4294         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4295         for (i = 0; i < ps->performance_level_count; i++) {
4296                 pl = &ps->performance_levels[i];
4297                 if (rdev->family >= CHIP_TAHITI)
4298                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u pcie gen: %u\n",
4299                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci, pl->pcie_gen + 1);
4300                 else
4301                         printk("\t\tpower level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4302                                i, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4303         }
4304         r600_dpm_print_ps_status(rdev, rps);
4305 }
4306
4307 void ni_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
4308                                                     struct seq_file *m)
4309 {
4310         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4311         struct radeon_ps *rps = &eg_pi->current_rps;
4312         struct ni_ps *ps = ni_get_ps(rps);
4313         struct rv7xx_pl *pl;
4314         u32 current_index =
4315                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4316                 CURRENT_STATE_INDEX_SHIFT;
4317
4318         if (current_index >= ps->performance_level_count) {
4319                 seq_printf(m, "invalid dpm profile %d\n", current_index);
4320         } else {
4321                 pl = &ps->performance_levels[current_index];
4322                 seq_printf(m, "uvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
4323                 seq_printf(m, "power level %d    sclk: %u mclk: %u vddc: %u vddci: %u\n",
4324                            current_index, pl->sclk, pl->mclk, pl->vddc, pl->vddci);
4325         }
4326 }
4327
4328 u32 ni_dpm_get_current_sclk(struct radeon_device *rdev)
4329 {
4330         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4331         struct radeon_ps *rps = &eg_pi->current_rps;
4332         struct ni_ps *ps = ni_get_ps(rps);
4333         struct rv7xx_pl *pl;
4334         u32 current_index =
4335                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4336                 CURRENT_STATE_INDEX_SHIFT;
4337
4338         if (current_index >= ps->performance_level_count) {
4339                 return 0;
4340         } else {
4341                 pl = &ps->performance_levels[current_index];
4342                 return pl->sclk;
4343         }
4344 }
4345
4346 u32 ni_dpm_get_current_mclk(struct radeon_device *rdev)
4347 {
4348         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4349         struct radeon_ps *rps = &eg_pi->current_rps;
4350         struct ni_ps *ps = ni_get_ps(rps);
4351         struct rv7xx_pl *pl;
4352         u32 current_index =
4353                 (RREG32(TARGET_AND_CURRENT_PROFILE_INDEX) & CURRENT_STATE_INDEX_MASK) >>
4354                 CURRENT_STATE_INDEX_SHIFT;
4355
4356         if (current_index >= ps->performance_level_count) {
4357                 return 0;
4358         } else {
4359                 pl = &ps->performance_levels[current_index];
4360                 return pl->mclk;
4361         }
4362 }
4363
4364 u32 ni_dpm_get_sclk(struct radeon_device *rdev, bool low)
4365 {
4366         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4367         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4368
4369         if (low)
4370                 return requested_state->performance_levels[0].sclk;
4371         else
4372                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
4373 }
4374
4375 u32 ni_dpm_get_mclk(struct radeon_device *rdev, bool low)
4376 {
4377         struct evergreen_power_info *eg_pi = evergreen_get_pi(rdev);
4378         struct ni_ps *requested_state = ni_get_ps(&eg_pi->requested_rps);
4379
4380         if (low)
4381                 return requested_state->performance_levels[0].mclk;
4382         else
4383                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
4384 }
4385