drivers/gpu/drm/radeon/ci_dpm.c

   1 /*
   2  * Copyright 2013 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  */
  23
  24 #include <linux/firmware.h>
  25 #include <linux/pci.h>
  26 #include <linux/seq_file.h>
  27
  28 #include "atom.h"
  29 #include "ci_dpm.h"
  30 #include "cik.h"
  31 #include "cikd.h"
  32 #include "r600_dpm.h"
  33 #include "radeon.h"
  34 #include "radeon_asic.h"
  35 #include "radeon_ucode.h"
  36 #include "si_dpm.h"
  37
  38 #define MC_CG_ARB_FREQ_F0           0x0a
  39 #define MC_CG_ARB_FREQ_F1           0x0b
  40 #define MC_CG_ARB_FREQ_F2           0x0c
  41 #define MC_CG_ARB_FREQ_F3           0x0d
  42
  43 #define SMC_RAM_END 0x40000
  44
  45 #define VOLTAGE_SCALE               4
  46 #define VOLTAGE_VID_OFFSET_SCALE1    625
  47 #define VOLTAGE_VID_OFFSET_SCALE2    100
  48
  49 static const struct ci_pt_defaults defaults_hawaii_xt = {
  50         1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0xB0000,
  51         { 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
  52         { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
  53 };
  54
  55 static const struct ci_pt_defaults defaults_hawaii_pro = {
  56         1, 0xF, 0xFD, 0x19, 5, 0x14, 0, 0x65062,
  57         { 0x2E,  0x00,  0x00,  0x88,  0x00,  0x00,  0x72,  0x60,  0x51,  0xA7,  0x79,  0x6B,  0x90,  0xBD,  0x79  },
  58         { 0x217, 0x217, 0x217, 0x242, 0x242, 0x242, 0x269, 0x269, 0x269, 0x2A1, 0x2A1, 0x2A1, 0x2C9, 0x2C9, 0x2C9 }
  59 };
  60
  61 static const struct ci_pt_defaults defaults_bonaire_xt = {
  62         1, 0xF, 0xFD, 0x19, 5, 45, 0, 0xB0000,
  63         { 0x79,  0x253, 0x25D, 0xAE,  0x72,  0x80,  0x83,  0x86,  0x6F,  0xC8,  0xC9,  0xC9,  0x2F,  0x4D,  0x61  },
  64         { 0x17C, 0x172, 0x180, 0x1BC, 0x1B3, 0x1BD, 0x206, 0x200, 0x203, 0x25D, 0x25A, 0x255, 0x2C3, 0x2C5, 0x2B4 }
  65 };
  66
  67 static const struct ci_pt_defaults defaults_saturn_xt = {
  68         1, 0xF, 0xFD, 0x19, 5, 55, 0, 0x70000,
  69         { 0x8C,  0x247, 0x249, 0xA6,  0x80,  0x81,  0x8B,  0x89,  0x86,  0xC9,  0xCA,  0xC9,  0x4D,  0x4D,  0x4D  },
  70         { 0x187, 0x187, 0x187, 0x1C7, 0x1C7, 0x1C7, 0x210, 0x210, 0x210, 0x266, 0x266, 0x266, 0x2C9, 0x2C9, 0x2C9 }
  71 };
  72
  73 static const struct ci_pt_config_reg didt_config_ci[] = {
  74         { 0x10, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  75         { 0x10, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  76         { 0x10, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  77         { 0x10, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  78         { 0x11, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  79         { 0x11, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  80         { 0x11, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  81         { 0x11, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  82         { 0x12, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  83         { 0x12, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  84         { 0x12, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  85         { 0x12, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  86         { 0x2, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
  87         { 0x2, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
  88         { 0x2, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
  89         { 0x1, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
  90         { 0x1, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
  91         { 0x0, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  92         { 0x30, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  93         { 0x30, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  94         { 0x30, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  95         { 0x30, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  96         { 0x31, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  97         { 0x31, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  98         { 0x31, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
  99         { 0x31, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 100         { 0x32, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 101         { 0x32, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 102         { 0x32, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 103         { 0x32, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 104         { 0x22, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 105         { 0x22, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 106         { 0x22, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 107         { 0x21, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 108         { 0x21, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 109         { 0x20, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 110         { 0x50, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 111         { 0x50, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 112         { 0x50, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 113         { 0x50, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 114         { 0x51, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 115         { 0x51, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 116         { 0x51, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 117         { 0x51, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 118         { 0x52, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 119         { 0x52, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 120         { 0x52, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 121         { 0x52, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 122         { 0x42, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 123         { 0x42, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 124         { 0x42, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 125         { 0x41, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 126         { 0x41, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 127         { 0x40, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 128         { 0x70, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 129         { 0x70, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 130         { 0x70, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 131         { 0x70, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 132         { 0x71, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 133         { 0x71, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 134         { 0x71, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 135         { 0x71, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 136         { 0x72, 0x000000ff, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 137         { 0x72, 0x0000ff00, 8, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 138         { 0x72, 0x00ff0000, 16, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 139         { 0x72, 0xff000000, 24, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 140         { 0x62, 0x00003fff, 0, 0x4, CISLANDS_CONFIGREG_DIDT_IND },
 141         { 0x62, 0x03ff0000, 16, 0x80, CISLANDS_CONFIGREG_DIDT_IND },
 142         { 0x62, 0x78000000, 27, 0x3, CISLANDS_CONFIGREG_DIDT_IND },
 143         { 0x61, 0x0000ffff, 0, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 144         { 0x61, 0xffff0000, 16, 0x3FFF, CISLANDS_CONFIGREG_DIDT_IND },
 145         { 0x60, 0x00000001, 0, 0x0, CISLANDS_CONFIGREG_DIDT_IND },
 146         { 0xFFFFFFFF }
 147 };
 148
 149 extern u8 rv770_get_memory_module_index(struct radeon_device *rdev);
 150 extern int ni_copy_and_switch_arb_sets(struct radeon_device *rdev,
 151                                        u32 arb_freq_src, u32 arb_freq_dest);
 152 static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev,
 153                                          struct atom_voltage_table_entry *voltage_table,
 154                                          u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd);
 155 static int ci_set_power_limit(struct radeon_device *rdev, u32 n);
 156 static int ci_set_overdrive_target_tdp(struct radeon_device *rdev,
 157                                        u32 target_tdp);
 158 static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate);
 159
 160 static PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg);
 161 static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
 162                                                       PPSMC_Msg msg, u32 parameter);
 163
 164 static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev);
 165 static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev);
 166
 167 static struct ci_power_info *ci_get_pi(struct radeon_device *rdev)
 168 {
 169         struct ci_power_info *pi = rdev->pm.dpm.priv;
 170
 171         return pi;
 172 }
 173
 174 static struct ci_ps *ci_get_ps(struct radeon_ps *rps)
 175 {
 176         struct ci_ps *ps = rps->ps_priv;
 177
 178         return ps;
 179 }
 180
 181 static void ci_initialize_powertune_defaults(struct radeon_device *rdev)
 182 {
 183         struct ci_power_info *pi = ci_get_pi(rdev);
 184
 185         switch (rdev->pdev->device) {
 186         case 0x6649:
 187         case 0x6650:
 188         case 0x6651:
 189         case 0x6658:
 190         case 0x665C:
 191         case 0x665D:
 192         default:
 193                 pi->powertune_defaults = &defaults_bonaire_xt;
 194                 break;
 195         case 0x6640:
 196         case 0x6641:
 197         case 0x6646:
 198         case 0x6647:
 199                 pi->powertune_defaults = &defaults_saturn_xt;
 200                 break;
 201         case 0x67B8:
 202         case 0x67B0:
 203                 pi->powertune_defaults = &defaults_hawaii_xt;
 204                 break;
 205         case 0x67BA:
 206         case 0x67B1:
 207                 pi->powertune_defaults = &defaults_hawaii_pro;
 208                 break;
 209         case 0x67A0:
 210         case 0x67A1:
 211         case 0x67A2:
 212         case 0x67A8:
 213         case 0x67A9:
 214         case 0x67AA:
 215         case 0x67B9:
 216         case 0x67BE:
 217                 pi->powertune_defaults = &defaults_bonaire_xt;
 218                 break;
 219         }
 220
 221         pi->dte_tj_offset = 0;
 222
 223         pi->caps_power_containment = true;
 224         pi->caps_cac = false;
 225         pi->caps_sq_ramping = false;
 226         pi->caps_db_ramping = false;
 227         pi->caps_td_ramping = false;
 228         pi->caps_tcp_ramping = false;
 229
 230         if (pi->caps_power_containment) {
 231                 pi->caps_cac = true;
 232                 if (rdev->family == CHIP_HAWAII)
 233                         pi->enable_bapm_feature = false;
 234                 else
 235                         pi->enable_bapm_feature = true;
 236                 pi->enable_tdc_limit_feature = true;
 237                 pi->enable_pkg_pwr_tracking_feature = true;
 238         }
 239 }
 240
 241 static u8 ci_convert_to_vid(u16 vddc)
 242 {
 243         return (6200 - (vddc * VOLTAGE_SCALE)) / 25;
 244 }
 245
 246 static int ci_populate_bapm_vddc_vid_sidd(struct radeon_device *rdev)
 247 {
 248         struct ci_power_info *pi = ci_get_pi(rdev);
 249         u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
 250         u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
 251         u8 *hi2_vid = pi->smc_powertune_table.BapmVddCVidHiSidd2;
 252         u32 i;
 253
 254         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries == NULL)
 255                 return -EINVAL;
 256         if (rdev->pm.dpm.dyn_state.cac_leakage_table.count > 8)
 257                 return -EINVAL;
 258         if (rdev->pm.dpm.dyn_state.cac_leakage_table.count !=
 259             rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count)
 260                 return -EINVAL;
 261
 262         for (i = 0; i < rdev->pm.dpm.dyn_state.cac_leakage_table.count; i++) {
 263                 if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
 264                         lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc1);
 265                         hi_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc2);
 266                         hi2_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc3);
 267                 } else {
 268                         lo_vid[i] = ci_convert_to_vid(rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].vddc);
 269                         hi_vid[i] = ci_convert_to_vid((u16)rdev->pm.dpm.dyn_state.cac_leakage_table.entries[i].leakage);
 270                 }
 271         }
 272         return 0;
 273 }
 274
 275 static int ci_populate_vddc_vid(struct radeon_device *rdev)
 276 {
 277         struct ci_power_info *pi = ci_get_pi(rdev);
 278         u8 *vid = pi->smc_powertune_table.VddCVid;
 279         u32 i;
 280
 281         if (pi->vddc_voltage_table.count > 8)
 282                 return -EINVAL;
 283
 284         for (i = 0; i < pi->vddc_voltage_table.count; i++)
 285                 vid[i] = ci_convert_to_vid(pi->vddc_voltage_table.entries[i].value);
 286
 287         return 0;
 288 }
 289
 290 static int ci_populate_svi_load_line(struct radeon_device *rdev)
 291 {
 292         struct ci_power_info *pi = ci_get_pi(rdev);
 293         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 294
 295         pi->smc_powertune_table.SviLoadLineEn = pt_defaults->svi_load_line_en;
 296         pi->smc_powertune_table.SviLoadLineVddC = pt_defaults->svi_load_line_vddc;
 297         pi->smc_powertune_table.SviLoadLineTrimVddC = 3;
 298         pi->smc_powertune_table.SviLoadLineOffsetVddC = 0;
 299
 300         return 0;
 301 }
 302
 303 static int ci_populate_tdc_limit(struct radeon_device *rdev)
 304 {
 305         struct ci_power_info *pi = ci_get_pi(rdev);
 306         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 307         u16 tdc_limit;
 308
 309         tdc_limit = rdev->pm.dpm.dyn_state.cac_tdp_table->tdc * 256;
 310         pi->smc_powertune_table.TDC_VDDC_PkgLimit = cpu_to_be16(tdc_limit);
 311         pi->smc_powertune_table.TDC_VDDC_ThrottleReleaseLimitPerc =
 312                 pt_defaults->tdc_vddc_throttle_release_limit_perc;
 313         pi->smc_powertune_table.TDC_MAWt = pt_defaults->tdc_mawt;
 314
 315         return 0;
 316 }
 317
 318 static int ci_populate_dw8(struct radeon_device *rdev)
 319 {
 320         struct ci_power_info *pi = ci_get_pi(rdev);
 321         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 322         int ret;
 323
 324         ret = ci_read_smc_sram_dword(rdev,
 325                                      SMU7_FIRMWARE_HEADER_LOCATION +
 326                                      offsetof(SMU7_Firmware_Header, PmFuseTable) +
 327                                      offsetof(SMU7_Discrete_PmFuses, TdcWaterfallCtl),
 328                                      (u32 *)&pi->smc_powertune_table.TdcWaterfallCtl,
 329                                      pi->sram_end);
 330         if (ret)
 331                 return -EINVAL;
 332         else
 333                 pi->smc_powertune_table.TdcWaterfallCtl = pt_defaults->tdc_waterfall_ctl;
 334
 335         return 0;
 336 }
 337
 338 static int ci_populate_fuzzy_fan(struct radeon_device *rdev)
 339 {
 340         struct ci_power_info *pi = ci_get_pi(rdev);
 341
 342         if ((rdev->pm.dpm.fan.fan_output_sensitivity & (1 << 15)) ||
 343             (rdev->pm.dpm.fan.fan_output_sensitivity == 0))
 344                 rdev->pm.dpm.fan.fan_output_sensitivity =
 345                         rdev->pm.dpm.fan.default_fan_output_sensitivity;
 346
 347         pi->smc_powertune_table.FuzzyFan_PwmSetDelta =
 348                 cpu_to_be16(rdev->pm.dpm.fan.fan_output_sensitivity);
 349
 350         return 0;
 351 }
 352
 353 static int ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(struct radeon_device *rdev)
 354 {
 355         struct ci_power_info *pi = ci_get_pi(rdev);
 356         u8 *hi_vid = pi->smc_powertune_table.BapmVddCVidHiSidd;
 357         u8 *lo_vid = pi->smc_powertune_table.BapmVddCVidLoSidd;
 358         int i, min, max;
 359
 360         min = max = hi_vid[0];
 361         for (i = 0; i < 8; i++) {
 362                 if (0 != hi_vid[i]) {
 363                         if (min > hi_vid[i])
 364                                 min = hi_vid[i];
 365                         if (max < hi_vid[i])
 366                                 max = hi_vid[i];
 367                 }
 368
 369                 if (0 != lo_vid[i]) {
 370                         if (min > lo_vid[i])
 371                                 min = lo_vid[i];
 372                         if (max < lo_vid[i])
 373                                 max = lo_vid[i];
 374                 }
 375         }
 376
 377         if ((min == 0) || (max == 0))
 378                 return -EINVAL;
 379         pi->smc_powertune_table.GnbLPMLMaxVid = (u8)max;
 380         pi->smc_powertune_table.GnbLPMLMinVid = (u8)min;
 381
 382         return 0;
 383 }
 384
 385 static int ci_populate_bapm_vddc_base_leakage_sidd(struct radeon_device *rdev)
 386 {
 387         struct ci_power_info *pi = ci_get_pi(rdev);
 388         u16 hi_sidd, lo_sidd;
 389         struct radeon_cac_tdp_table *cac_tdp_table =
 390                 rdev->pm.dpm.dyn_state.cac_tdp_table;
 391
 392         hi_sidd = cac_tdp_table->high_cac_leakage / 100 * 256;
 393         lo_sidd = cac_tdp_table->low_cac_leakage / 100 * 256;
 394
 395         pi->smc_powertune_table.BapmVddCBaseLeakageHiSidd = cpu_to_be16(hi_sidd);
 396         pi->smc_powertune_table.BapmVddCBaseLeakageLoSidd = cpu_to_be16(lo_sidd);
 397
 398         return 0;
 399 }
 400
 401 static int ci_populate_bapm_parameters_in_dpm_table(struct radeon_device *rdev)
 402 {
 403         struct ci_power_info *pi = ci_get_pi(rdev);
 404         const struct ci_pt_defaults *pt_defaults = pi->powertune_defaults;
 405         SMU7_Discrete_DpmTable  *dpm_table = &pi->smc_state_table;
 406         struct radeon_cac_tdp_table *cac_tdp_table =
 407                 rdev->pm.dpm.dyn_state.cac_tdp_table;
 408         struct radeon_ppm_table *ppm = rdev->pm.dpm.dyn_state.ppm_table;
 409         int i, j, k;
 410         const u16 *def1;
 411         const u16 *def2;
 412
 413         dpm_table->DefaultTdp = cac_tdp_table->tdp * 256;
 414         dpm_table->TargetTdp = cac_tdp_table->configurable_tdp * 256;
 415
 416         dpm_table->DTETjOffset = (u8)pi->dte_tj_offset;
 417         dpm_table->GpuTjMax =
 418                 (u8)(pi->thermal_temp_setting.temperature_high / 1000);
 419         dpm_table->GpuTjHyst = 8;
 420
 421         dpm_table->DTEAmbientTempBase = pt_defaults->dte_ambient_temp_base;
 422
 423         if (ppm) {
 424                 dpm_table->PPM_PkgPwrLimit = cpu_to_be16((u16)ppm->dgpu_tdp * 256 / 1000);
 425                 dpm_table->PPM_TemperatureLimit = cpu_to_be16((u16)ppm->tj_max * 256);
 426         } else {
 427                 dpm_table->PPM_PkgPwrLimit = cpu_to_be16(0);
 428                 dpm_table->PPM_TemperatureLimit = cpu_to_be16(0);
 429         }
 430
 431         dpm_table->BAPM_TEMP_GRADIENT = cpu_to_be32(pt_defaults->bapm_temp_gradient);
 432         def1 = pt_defaults->bapmti_r;
 433         def2 = pt_defaults->bapmti_rc;
 434
 435         for (i = 0; i < SMU7_DTE_ITERATIONS; i++) {
 436                 for (j = 0; j < SMU7_DTE_SOURCES; j++) {
 437                         for (k = 0; k < SMU7_DTE_SINKS; k++) {
 438                                 dpm_table->BAPMTI_R[i][j][k] = cpu_to_be16(*def1);
 439                                 dpm_table->BAPMTI_RC[i][j][k] = cpu_to_be16(*def2);
 440                                 def1++;
 441                                 def2++;
 442                         }
 443                 }
 444         }
 445
 446         return 0;
 447 }
 448
 449 static int ci_populate_pm_base(struct radeon_device *rdev)
 450 {
 451         struct ci_power_info *pi = ci_get_pi(rdev);
 452         u32 pm_fuse_table_offset;
 453         int ret;
 454
 455         if (pi->caps_power_containment) {
 456                 ret = ci_read_smc_sram_dword(rdev,
 457                                              SMU7_FIRMWARE_HEADER_LOCATION +
 458                                              offsetof(SMU7_Firmware_Header, PmFuseTable),
 459                                              &pm_fuse_table_offset, pi->sram_end);
 460                 if (ret)
 461                         return ret;
 462                 ret = ci_populate_bapm_vddc_vid_sidd(rdev);
 463                 if (ret)
 464                         return ret;
 465                 ret = ci_populate_vddc_vid(rdev);
 466                 if (ret)
 467                         return ret;
 468                 ret = ci_populate_svi_load_line(rdev);
 469                 if (ret)
 470                         return ret;
 471                 ret = ci_populate_tdc_limit(rdev);
 472                 if (ret)
 473                         return ret;
 474                 ret = ci_populate_dw8(rdev);
 475                 if (ret)
 476                         return ret;
 477                 ret = ci_populate_fuzzy_fan(rdev);
 478                 if (ret)
 479                         return ret;
 480                 ret = ci_min_max_v_gnbl_pm_lid_from_bapm_vddc(rdev);
 481                 if (ret)
 482                         return ret;
 483                 ret = ci_populate_bapm_vddc_base_leakage_sidd(rdev);
 484                 if (ret)
 485                         return ret;
 486                 ret = ci_copy_bytes_to_smc(rdev, pm_fuse_table_offset,
 487                                            (u8 *)&pi->smc_powertune_table,
 488                                            sizeof(SMU7_Discrete_PmFuses), pi->sram_end);
 489                 if (ret)
 490                         return ret;
 491         }
 492
 493         return 0;
 494 }
 495
 496 static void ci_do_enable_didt(struct radeon_device *rdev, const bool enable)
 497 {
 498         struct ci_power_info *pi = ci_get_pi(rdev);
 499         u32 data;
 500
 501         if (pi->caps_sq_ramping) {
 502                 data = RREG32_DIDT(DIDT_SQ_CTRL0);
 503                 if (enable)
 504                         data |= DIDT_CTRL_EN;
 505                 else
 506                         data &= ~DIDT_CTRL_EN;
 507                 WREG32_DIDT(DIDT_SQ_CTRL0, data);
 508         }
 509
 510         if (pi->caps_db_ramping) {
 511                 data = RREG32_DIDT(DIDT_DB_CTRL0);
 512                 if (enable)
 513                         data |= DIDT_CTRL_EN;
 514                 else
 515                         data &= ~DIDT_CTRL_EN;
 516                 WREG32_DIDT(DIDT_DB_CTRL0, data);
 517         }
 518
 519         if (pi->caps_td_ramping) {
 520                 data = RREG32_DIDT(DIDT_TD_CTRL0);
 521                 if (enable)
 522                         data |= DIDT_CTRL_EN;
 523                 else
 524                         data &= ~DIDT_CTRL_EN;
 525                 WREG32_DIDT(DIDT_TD_CTRL0, data);
 526         }
 527
 528         if (pi->caps_tcp_ramping) {
 529                 data = RREG32_DIDT(DIDT_TCP_CTRL0);
 530                 if (enable)
 531                         data |= DIDT_CTRL_EN;
 532                 else
 533                         data &= ~DIDT_CTRL_EN;
 534                 WREG32_DIDT(DIDT_TCP_CTRL0, data);
 535         }
 536 }
 537
 538 static int ci_program_pt_config_registers(struct radeon_device *rdev,
 539                                           const struct ci_pt_config_reg *cac_config_regs)
 540 {
 541         const struct ci_pt_config_reg *config_regs = cac_config_regs;
 542         u32 data;
 543         u32 cache = 0;
 544
 545         if (config_regs == NULL)
 546                 return -EINVAL;
 547
 548         while (config_regs->offset != 0xFFFFFFFF) {
 549                 if (config_regs->type == CISLANDS_CONFIGREG_CACHE) {
 550                         cache |= ((config_regs->value << config_regs->shift) & config_regs->mask);
 551                 } else {
 552                         switch (config_regs->type) {
 553                         case CISLANDS_CONFIGREG_SMC_IND:
 554                                 data = RREG32_SMC(config_regs->offset);
 555                                 break;
 556                         case CISLANDS_CONFIGREG_DIDT_IND:
 557                                 data = RREG32_DIDT(config_regs->offset);
 558                                 break;
 559                         default:
 560                                 data = RREG32(config_regs->offset << 2);
 561                                 break;
 562                         }
 563
 564                         data &= ~config_regs->mask;
 565                         data |= ((config_regs->value << config_regs->shift) & config_regs->mask);
 566                         data |= cache;
 567
 568                         switch (config_regs->type) {
 569                         case CISLANDS_CONFIGREG_SMC_IND:
 570                                 WREG32_SMC(config_regs->offset, data);
 571                                 break;
 572                         case CISLANDS_CONFIGREG_DIDT_IND:
 573                                 WREG32_DIDT(config_regs->offset, data);
 574                                 break;
 575                         default:
 576                                 WREG32(config_regs->offset << 2, data);
 577                                 break;
 578                         }
 579                         cache = 0;
 580                 }
 581                 config_regs++;
 582         }
 583         return 0;
 584 }
 585
 586 static int ci_enable_didt(struct radeon_device *rdev, bool enable)
 587 {
 588         struct ci_power_info *pi = ci_get_pi(rdev);
 589         int ret;
 590
 591         if (pi->caps_sq_ramping || pi->caps_db_ramping ||
 592             pi->caps_td_ramping || pi->caps_tcp_ramping) {
 593                 cik_enter_rlc_safe_mode(rdev);
 594
 595                 if (enable) {
 596                         ret = ci_program_pt_config_registers(rdev, didt_config_ci);
 597                         if (ret) {
 598                                 cik_exit_rlc_safe_mode(rdev);
 599                                 return ret;
 600                         }
 601                 }
 602
 603                 ci_do_enable_didt(rdev, enable);
 604
 605                 cik_exit_rlc_safe_mode(rdev);
 606         }
 607
 608         return 0;
 609 }
 610
 611 static int ci_enable_power_containment(struct radeon_device *rdev, bool enable)
 612 {
 613         struct ci_power_info *pi = ci_get_pi(rdev);
 614         PPSMC_Result smc_result;
 615         int ret = 0;
 616
 617         if (enable) {
 618                 pi->power_containment_features = 0;
 619                 if (pi->caps_power_containment) {
 620                         if (pi->enable_bapm_feature) {
 621                                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableDTE);
 622                                 if (smc_result != PPSMC_Result_OK)
 623                                         ret = -EINVAL;
 624                                 else
 625                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_BAPM;
 626                         }
 627
 628                         if (pi->enable_tdc_limit_feature) {
 629                                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitEnable);
 630                                 if (smc_result != PPSMC_Result_OK)
 631                                         ret = -EINVAL;
 632                                 else
 633                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_TDCLimit;
 634                         }
 635
 636                         if (pi->enable_pkg_pwr_tracking_feature) {
 637                                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitEnable);
 638                                 if (smc_result != PPSMC_Result_OK) {
 639                                         ret = -EINVAL;
 640                                 } else {
 641                                         struct radeon_cac_tdp_table *cac_tdp_table =
 642                                                 rdev->pm.dpm.dyn_state.cac_tdp_table;
 643                                         u32 default_pwr_limit =
 644                                                 (u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
 645
 646                                         pi->power_containment_features |= POWERCONTAINMENT_FEATURE_PkgPwrLimit;
 647
 648                                         ci_set_power_limit(rdev, default_pwr_limit);
 649                                 }
 650                         }
 651                 }
 652         } else {
 653                 if (pi->caps_power_containment && pi->power_containment_features) {
 654                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_TDCLimit)
 655                                 ci_send_msg_to_smc(rdev, PPSMC_MSG_TDCLimitDisable);
 656
 657                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_BAPM)
 658                                 ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableDTE);
 659
 660                         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit)
 661                                 ci_send_msg_to_smc(rdev, PPSMC_MSG_PkgPwrLimitDisable);
 662                         pi->power_containment_features = 0;
 663                 }
 664         }
 665
 666         return ret;
 667 }
 668
 669 static int ci_enable_smc_cac(struct radeon_device *rdev, bool enable)
 670 {
 671         struct ci_power_info *pi = ci_get_pi(rdev);
 672         PPSMC_Result smc_result;
 673         int ret = 0;
 674
 675         if (pi->caps_cac) {
 676                 if (enable) {
 677                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableCac);
 678                         if (smc_result != PPSMC_Result_OK) {
 679                                 ret = -EINVAL;
 680                                 pi->cac_enabled = false;
 681                         } else {
 682                                 pi->cac_enabled = true;
 683                         }
 684                 } else if (pi->cac_enabled) {
 685                         ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableCac);
 686                         pi->cac_enabled = false;
 687                 }
 688         }
 689
 690         return ret;
 691 }
 692
 693 static int ci_enable_thermal_based_sclk_dpm(struct radeon_device *rdev,
 694                                             bool enable)
 695 {
 696         struct ci_power_info *pi = ci_get_pi(rdev);
 697         PPSMC_Result smc_result = PPSMC_Result_OK;
 698
 699         if (pi->thermal_sclk_dpm_enabled) {
 700                 if (enable)
 701                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_ENABLE_THERMAL_DPM);
 702                 else
 703                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DISABLE_THERMAL_DPM);
 704         }
 705
 706         if (smc_result == PPSMC_Result_OK)
 707                 return 0;
 708         else
 709                 return -EINVAL;
 710 }
 711
 712 static int ci_power_control_set_level(struct radeon_device *rdev)
 713 {
 714         struct ci_power_info *pi = ci_get_pi(rdev);
 715         struct radeon_cac_tdp_table *cac_tdp_table =
 716                 rdev->pm.dpm.dyn_state.cac_tdp_table;
 717         s32 adjust_percent;
 718         s32 target_tdp;
 719         int ret = 0;
 720         bool adjust_polarity = false; /* ??? */
 721
 722         if (pi->caps_power_containment) {
 723                 adjust_percent = adjust_polarity ?
 724                         rdev->pm.dpm.tdp_adjustment : (-1 * rdev->pm.dpm.tdp_adjustment);
 725                 target_tdp = ((100 + adjust_percent) *
 726                               (s32)cac_tdp_table->configurable_tdp) / 100;
 727
 728                 ret = ci_set_overdrive_target_tdp(rdev, (u32)target_tdp);
 729         }
 730
 731         return ret;
 732 }
 733
 734 void ci_dpm_powergate_uvd(struct radeon_device *rdev, bool gate)
 735 {
 736         struct ci_power_info *pi = ci_get_pi(rdev);
 737
 738         if (pi->uvd_power_gated == gate)
 739                 return;
 740
 741         pi->uvd_power_gated = gate;
 742
 743         ci_update_uvd_dpm(rdev, gate);
 744 }
 745
 746 bool ci_dpm_vblank_too_short(struct radeon_device *rdev)
 747 {
 748         struct ci_power_info *pi = ci_get_pi(rdev);
 749         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
 750         u32 switch_limit = pi->mem_gddr5 ? 450 : 300;
 751
 752         /* disable mclk switching if the refresh is >120Hz, even if the
 753         * blanking period would allow it
 754         */
 755         if (r600_dpm_get_vrefresh(rdev) > 120)
 756                 return true;
 757
 758         if (vblank_time < switch_limit)
 759                 return true;
 760         else
 761                 return false;
 762
 763 }
 764
 765 static void ci_apply_state_adjust_rules(struct radeon_device *rdev,
 766                                         struct radeon_ps *rps)
 767 {
 768         struct ci_ps *ps = ci_get_ps(rps);
 769         struct ci_power_info *pi = ci_get_pi(rdev);
 770         struct radeon_clock_and_voltage_limits *max_limits;
 771         bool disable_mclk_switching;
 772         u32 sclk, mclk;
 773         int i;
 774
 775         if (rps->vce_active) {
 776                 rps->evclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].evclk;
 777                 rps->ecclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].ecclk;
 778         } else {
 779                 rps->evclk = 0;
 780                 rps->ecclk = 0;
 781         }
 782
 783         if ((rdev->pm.dpm.new_active_crtc_count > 1) ||
 784             ci_dpm_vblank_too_short(rdev))
 785                 disable_mclk_switching = true;
 786         else
 787                 disable_mclk_switching = false;
 788
 789         if ((rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) == ATOM_PPLIB_CLASSIFICATION_UI_BATTERY)
 790                 pi->battery_state = true;
 791         else
 792                 pi->battery_state = false;
 793
 794         if (rdev->pm.dpm.ac_power)
 795                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
 796         else
 797                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
 798
 799         if (rdev->pm.dpm.ac_power == false) {
 800                 for (i = 0; i < ps->performance_level_count; i++) {
 801                         if (ps->performance_levels[i].mclk > max_limits->mclk)
 802                                 ps->performance_levels[i].mclk = max_limits->mclk;
 803                         if (ps->performance_levels[i].sclk > max_limits->sclk)
 804                                 ps->performance_levels[i].sclk = max_limits->sclk;
 805                 }
 806         }
 807
 808         /* XXX validate the min clocks required for display */
 809
 810         if (disable_mclk_switching) {
 811                 mclk  = ps->performance_levels[ps->performance_level_count - 1].mclk;
 812                 sclk = ps->performance_levels[0].sclk;
 813         } else {
 814                 mclk = ps->performance_levels[0].mclk;
 815                 sclk = ps->performance_levels[0].sclk;
 816         }
 817
 818         if (rps->vce_active) {
 819                 if (sclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk)
 820                         sclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].sclk;
 821                 if (mclk < rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk)
 822                         mclk = rdev->pm.dpm.vce_states[rdev->pm.dpm.vce_level].mclk;
 823         }
 824
 825         ps->performance_levels[0].sclk = sclk;
 826         ps->performance_levels[0].mclk = mclk;
 827
 828         if (ps->performance_levels[1].sclk < ps->performance_levels[0].sclk)
 829                 ps->performance_levels[1].sclk = ps->performance_levels[0].sclk;
 830
 831         if (disable_mclk_switching) {
 832                 if (ps->performance_levels[0].mclk < ps->performance_levels[1].mclk)
 833                         ps->performance_levels[0].mclk = ps->performance_levels[1].mclk;
 834         } else {
 835                 if (ps->performance_levels[1].mclk < ps->performance_levels[0].mclk)
 836                         ps->performance_levels[1].mclk = ps->performance_levels[0].mclk;
 837         }
 838 }
 839
 840 static int ci_thermal_set_temperature_range(struct radeon_device *rdev,
 841                                             int min_temp, int max_temp)
 842 {
 843         int low_temp = 0 * 1000;
 844         int high_temp = 255 * 1000;
 845         u32 tmp;
 846
 847         if (low_temp < min_temp)
 848                 low_temp = min_temp;
 849         if (high_temp > max_temp)
 850                 high_temp = max_temp;
 851         if (high_temp < low_temp) {
 852                 DRM_ERROR("invalid thermal range: %d - %d\n", low_temp, high_temp);
 853                 return -EINVAL;
 854         }
 855
 856         tmp = RREG32_SMC(CG_THERMAL_INT);
 857         tmp &= ~(CI_DIG_THERM_INTH_MASK | CI_DIG_THERM_INTL_MASK);
 858         tmp |= CI_DIG_THERM_INTH(high_temp / 1000) |
 859                 CI_DIG_THERM_INTL(low_temp / 1000);
 860         WREG32_SMC(CG_THERMAL_INT, tmp);
 861
 862 #if 0
 863         /* XXX: need to figure out how to handle this properly */
 864         tmp = RREG32_SMC(CG_THERMAL_CTRL);
 865         tmp &= DIG_THERM_DPM_MASK;
 866         tmp |= DIG_THERM_DPM(high_temp / 1000);
 867         WREG32_SMC(CG_THERMAL_CTRL, tmp);
 868 #endif
 869
 870         rdev->pm.dpm.thermal.min_temp = low_temp;
 871         rdev->pm.dpm.thermal.max_temp = high_temp;
 872
 873         return 0;
 874 }
 875
 876 static int ci_thermal_enable_alert(struct radeon_device *rdev,
 877                                    bool enable)
 878 {
 879         u32 thermal_int = RREG32_SMC(CG_THERMAL_INT);
 880         PPSMC_Result result;
 881
 882         if (enable) {
 883                 thermal_int &= ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
 884                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
 885                 rdev->irq.dpm_thermal = false;
 886                 result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Enable);
 887                 if (result != PPSMC_Result_OK) {
 888                         DRM_DEBUG_KMS("Could not enable thermal interrupts.\n");
 889                         return -EINVAL;
 890                 }
 891         } else {
 892                 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
 893                 WREG32_SMC(CG_THERMAL_INT, thermal_int);
 894                 rdev->irq.dpm_thermal = true;
 895                 result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Thermal_Cntl_Disable);
 896                 if (result != PPSMC_Result_OK) {
 897                         DRM_DEBUG_KMS("Could not disable thermal interrupts.\n");
 898                         return -EINVAL;
 899                 }
 900         }
 901
 902         return 0;
 903 }
 904
 905 static void ci_fan_ctrl_set_static_mode(struct radeon_device *rdev, u32 mode)
 906 {
 907         struct ci_power_info *pi = ci_get_pi(rdev);
 908         u32 tmp;
 909
 910         if (pi->fan_ctrl_is_in_default_mode) {
 911                 tmp = (RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK) >> FDO_PWM_MODE_SHIFT;
 912                 pi->fan_ctrl_default_mode = tmp;
 913                 tmp = (RREG32_SMC(CG_FDO_CTRL2) & TMIN_MASK) >> TMIN_SHIFT;
 914                 pi->t_min = tmp;
 915                 pi->fan_ctrl_is_in_default_mode = false;
 916         }
 917
 918         tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK;
 919         tmp |= TMIN(0);
 920         WREG32_SMC(CG_FDO_CTRL2, tmp);
 921
 922         tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
 923         tmp |= FDO_PWM_MODE(mode);
 924         WREG32_SMC(CG_FDO_CTRL2, tmp);
 925 }
 926
 927 static int ci_thermal_setup_fan_table(struct radeon_device *rdev)
 928 {
 929         struct ci_power_info *pi = ci_get_pi(rdev);
 930         SMU7_Discrete_FanTable fan_table = { FDO_MODE_HARDWARE };
 931         u32 duty100;
 932         u32 t_diff1, t_diff2, pwm_diff1, pwm_diff2;
 933         u16 fdo_min, slope1, slope2;
 934         u32 reference_clock, tmp;
 935         int ret;
 936         u64 tmp64;
 937
 938         if (!pi->fan_table_start) {
 939                 rdev->pm.dpm.fan.ucode_fan_control = false;
 940                 return 0;
 941         }
 942
 943         duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
 944
 945         if (duty100 == 0) {
 946                 rdev->pm.dpm.fan.ucode_fan_control = false;
 947                 return 0;
 948         }
 949
 950         tmp64 = (u64)rdev->pm.dpm.fan.pwm_min * duty100;
 951         do_div(tmp64, 10000);
 952         fdo_min = (u16)tmp64;
 953
 954         t_diff1 = rdev->pm.dpm.fan.t_med - rdev->pm.dpm.fan.t_min;
 955         t_diff2 = rdev->pm.dpm.fan.t_high - rdev->pm.dpm.fan.t_med;
 956
 957         pwm_diff1 = rdev->pm.dpm.fan.pwm_med - rdev->pm.dpm.fan.pwm_min;
 958         pwm_diff2 = rdev->pm.dpm.fan.pwm_high - rdev->pm.dpm.fan.pwm_med;
 959
 960         slope1 = (u16)((50 + ((16 * duty100 * pwm_diff1) / t_diff1)) / 100);
 961         slope2 = (u16)((50 + ((16 * duty100 * pwm_diff2) / t_diff2)) / 100);
 962
 963         fan_table.TempMin = cpu_to_be16((50 + rdev->pm.dpm.fan.t_min) / 100);
 964         fan_table.TempMed = cpu_to_be16((50 + rdev->pm.dpm.fan.t_med) / 100);
 965         fan_table.TempMax = cpu_to_be16((50 + rdev->pm.dpm.fan.t_max) / 100);
 966
 967         fan_table.Slope1 = cpu_to_be16(slope1);
 968         fan_table.Slope2 = cpu_to_be16(slope2);
 969
 970         fan_table.FdoMin = cpu_to_be16(fdo_min);
 971
 972         fan_table.HystDown = cpu_to_be16(rdev->pm.dpm.fan.t_hyst);
 973
 974         fan_table.HystUp = cpu_to_be16(1);
 975
 976         fan_table.HystSlope = cpu_to_be16(1);
 977
 978         fan_table.TempRespLim = cpu_to_be16(5);
 979
 980         reference_clock = radeon_get_xclk(rdev);
 981
 982         fan_table.RefreshPeriod = cpu_to_be32((rdev->pm.dpm.fan.cycle_delay *
 983                                                reference_clock) / 1600);
 984
 985         fan_table.FdoMax = cpu_to_be16((u16)duty100);
 986
 987         tmp = (RREG32_SMC(CG_MULT_THERMAL_CTRL) & TEMP_SEL_MASK) >> TEMP_SEL_SHIFT;
 988         fan_table.TempSrc = (uint8_t)tmp;
 989
 990         ret = ci_copy_bytes_to_smc(rdev,
 991                                    pi->fan_table_start,
 992                                    (u8 *)(&fan_table),
 993                                    sizeof(fan_table),
 994                                    pi->sram_end);
 995
 996         if (ret) {
 997                 DRM_ERROR("Failed to load fan table to the SMC.");
 998                 rdev->pm.dpm.fan.ucode_fan_control = false;
 999         }
1000
1001         return 0;
1002 }
1003
1004 static int ci_fan_ctrl_start_smc_fan_control(struct radeon_device *rdev)
1005 {
1006         struct ci_power_info *pi = ci_get_pi(rdev);
1007         PPSMC_Result ret;
1008
1009         if (pi->caps_od_fuzzy_fan_control_support) {
1010                 ret = ci_send_msg_to_smc_with_parameter(rdev,
1011                                                         PPSMC_StartFanControl,
1012                                                         FAN_CONTROL_FUZZY);
1013                 if (ret != PPSMC_Result_OK)
1014                         return -EINVAL;
1015                 ret = ci_send_msg_to_smc_with_parameter(rdev,
1016                                                         PPSMC_MSG_SetFanPwmMax,
1017                                                         rdev->pm.dpm.fan.default_max_fan_pwm);
1018                 if (ret != PPSMC_Result_OK)
1019                         return -EINVAL;
1020         } else {
1021                 ret = ci_send_msg_to_smc_with_parameter(rdev,
1022                                                         PPSMC_StartFanControl,
1023                                                         FAN_CONTROL_TABLE);
1024                 if (ret != PPSMC_Result_OK)
1025                         return -EINVAL;
1026         }
1027
1028         pi->fan_is_controlled_by_smc = true;
1029         return 0;
1030 }
1031
1032 static int ci_fan_ctrl_stop_smc_fan_control(struct radeon_device *rdev)
1033 {
1034         PPSMC_Result ret;
1035         struct ci_power_info *pi = ci_get_pi(rdev);
1036
1037         ret = ci_send_msg_to_smc(rdev, PPSMC_StopFanControl);
1038         if (ret == PPSMC_Result_OK) {
1039                 pi->fan_is_controlled_by_smc = false;
1040                 return 0;
1041         } else
1042                 return -EINVAL;
1043 }
1044
1045 int ci_fan_ctrl_get_fan_speed_percent(struct radeon_device *rdev,
1046                                              u32 *speed)
1047 {
1048         u32 duty, duty100;
1049         u64 tmp64;
1050
1051         if (rdev->pm.no_fan)
1052                 return -ENOENT;
1053
1054         duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
1055         duty = (RREG32_SMC(CG_THERMAL_STATUS) & FDO_PWM_DUTY_MASK) >> FDO_PWM_DUTY_SHIFT;
1056
1057         if (duty100 == 0)
1058                 return -EINVAL;
1059
1060         tmp64 = (u64)duty * 100;
1061         do_div(tmp64, duty100);
1062         *speed = (u32)tmp64;
1063
1064         if (*speed > 100)
1065                 *speed = 100;
1066
1067         return 0;
1068 }
1069
1070 int ci_fan_ctrl_set_fan_speed_percent(struct radeon_device *rdev,
1071                                              u32 speed)
1072 {
1073         u32 tmp;
1074         u32 duty, duty100;
1075         u64 tmp64;
1076         struct ci_power_info *pi = ci_get_pi(rdev);
1077
1078         if (rdev->pm.no_fan)
1079                 return -ENOENT;
1080
1081         if (pi->fan_is_controlled_by_smc)
1082                 return -EINVAL;
1083
1084         if (speed > 100)
1085                 return -EINVAL;
1086
1087         duty100 = (RREG32_SMC(CG_FDO_CTRL1) & FMAX_DUTY100_MASK) >> FMAX_DUTY100_SHIFT;
1088
1089         if (duty100 == 0)
1090                 return -EINVAL;
1091
1092         tmp64 = (u64)speed * duty100;
1093         do_div(tmp64, 100);
1094         duty = (u32)tmp64;
1095
1096         tmp = RREG32_SMC(CG_FDO_CTRL0) & ~FDO_STATIC_DUTY_MASK;
1097         tmp |= FDO_STATIC_DUTY(duty);
1098         WREG32_SMC(CG_FDO_CTRL0, tmp);
1099
1100         return 0;
1101 }
1102
1103 void ci_fan_ctrl_set_mode(struct radeon_device *rdev, u32 mode)
1104 {
1105         if (mode) {
1106                 /* stop auto-manage */
1107                 if (rdev->pm.dpm.fan.ucode_fan_control)
1108                         ci_fan_ctrl_stop_smc_fan_control(rdev);
1109                 ci_fan_ctrl_set_static_mode(rdev, mode);
1110         } else {
1111                 /* restart auto-manage */
1112                 if (rdev->pm.dpm.fan.ucode_fan_control)
1113                         ci_thermal_start_smc_fan_control(rdev);
1114                 else
1115                         ci_fan_ctrl_set_default_mode(rdev);
1116         }
1117 }
1118
1119 u32 ci_fan_ctrl_get_mode(struct radeon_device *rdev)
1120 {
1121         struct ci_power_info *pi = ci_get_pi(rdev);
1122         u32 tmp;
1123
1124         if (pi->fan_is_controlled_by_smc)
1125                 return 0;
1126
1127         tmp = RREG32_SMC(CG_FDO_CTRL2) & FDO_PWM_MODE_MASK;
1128         return (tmp >> FDO_PWM_MODE_SHIFT);
1129 }
1130
1131 #if 0
1132 static int ci_fan_ctrl_get_fan_speed_rpm(struct radeon_device *rdev,
1133                                          u32 *speed)
1134 {
1135         u32 tach_period;
1136         u32 xclk = radeon_get_xclk(rdev);
1137
1138         if (rdev->pm.no_fan)
1139                 return -ENOENT;
1140
1141         if (rdev->pm.fan_pulses_per_revolution == 0)
1142                 return -ENOENT;
1143
1144         tach_period = (RREG32_SMC(CG_TACH_STATUS) & TACH_PERIOD_MASK) >> TACH_PERIOD_SHIFT;
1145         if (tach_period == 0)
1146                 return -ENOENT;
1147
1148         *speed = 60 * xclk * 10000 / tach_period;
1149
1150         return 0;
1151 }
1152
1153 static int ci_fan_ctrl_set_fan_speed_rpm(struct radeon_device *rdev,
1154                                          u32 speed)
1155 {
1156         u32 tach_period, tmp;
1157         u32 xclk = radeon_get_xclk(rdev);
1158
1159         if (rdev->pm.no_fan)
1160                 return -ENOENT;
1161
1162         if (rdev->pm.fan_pulses_per_revolution == 0)
1163                 return -ENOENT;
1164
1165         if ((speed < rdev->pm.fan_min_rpm) ||
1166             (speed > rdev->pm.fan_max_rpm))
1167                 return -EINVAL;
1168
1169         if (rdev->pm.dpm.fan.ucode_fan_control)
1170                 ci_fan_ctrl_stop_smc_fan_control(rdev);
1171
1172         tach_period = 60 * xclk * 10000 / (8 * speed);
1173         tmp = RREG32_SMC(CG_TACH_CTRL) & ~TARGET_PERIOD_MASK;
1174         tmp |= TARGET_PERIOD(tach_period);
1175         WREG32_SMC(CG_TACH_CTRL, tmp);
1176
1177         ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC_RPM);
1178
1179         return 0;
1180 }
1181 #endif
1182
1183 static void ci_fan_ctrl_set_default_mode(struct radeon_device *rdev)
1184 {
1185         struct ci_power_info *pi = ci_get_pi(rdev);
1186         u32 tmp;
1187
1188         if (!pi->fan_ctrl_is_in_default_mode) {
1189                 tmp = RREG32_SMC(CG_FDO_CTRL2) & ~FDO_PWM_MODE_MASK;
1190                 tmp |= FDO_PWM_MODE(pi->fan_ctrl_default_mode);
1191                 WREG32_SMC(CG_FDO_CTRL2, tmp);
1192
1193                 tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TMIN_MASK;
1194                 tmp |= TMIN(pi->t_min);
1195                 WREG32_SMC(CG_FDO_CTRL2, tmp);
1196                 pi->fan_ctrl_is_in_default_mode = true;
1197         }
1198 }
1199
1200 static void ci_thermal_start_smc_fan_control(struct radeon_device *rdev)
1201 {
1202         if (rdev->pm.dpm.fan.ucode_fan_control) {
1203                 ci_fan_ctrl_start_smc_fan_control(rdev);
1204                 ci_fan_ctrl_set_static_mode(rdev, FDO_PWM_MODE_STATIC);
1205         }
1206 }
1207
1208 static void ci_thermal_initialize(struct radeon_device *rdev)
1209 {
1210         u32 tmp;
1211
1212         if (rdev->pm.fan_pulses_per_revolution) {
1213                 tmp = RREG32_SMC(CG_TACH_CTRL) & ~EDGE_PER_REV_MASK;
1214                 tmp |= EDGE_PER_REV(rdev->pm.fan_pulses_per_revolution - 1);
1215                 WREG32_SMC(CG_TACH_CTRL, tmp);
1216         }
1217
1218         tmp = RREG32_SMC(CG_FDO_CTRL2) & ~TACH_PWM_RESP_RATE_MASK;
1219         tmp |= TACH_PWM_RESP_RATE(0x28);
1220         WREG32_SMC(CG_FDO_CTRL2, tmp);
1221 }
1222
1223 static int ci_thermal_start_thermal_controller(struct radeon_device *rdev)
1224 {
1225         int ret;
1226
1227         ci_thermal_initialize(rdev);
1228         ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
1229         if (ret)
1230                 return ret;
1231         ret = ci_thermal_enable_alert(rdev, true);
1232         if (ret)
1233                 return ret;
1234         if (rdev->pm.dpm.fan.ucode_fan_control) {
1235                 ret = ci_thermal_setup_fan_table(rdev);
1236                 if (ret)
1237                         return ret;
1238                 ci_thermal_start_smc_fan_control(rdev);
1239         }
1240
1241         return 0;
1242 }
1243
1244 static void ci_thermal_stop_thermal_controller(struct radeon_device *rdev)
1245 {
1246         if (!rdev->pm.no_fan)
1247                 ci_fan_ctrl_set_default_mode(rdev);
1248 }
1249
1250 #if 0
1251 static int ci_read_smc_soft_register(struct radeon_device *rdev,
1252                                      u16 reg_offset, u32 *value)
1253 {
1254         struct ci_power_info *pi = ci_get_pi(rdev);
1255
1256         return ci_read_smc_sram_dword(rdev,
1257                                       pi->soft_regs_start + reg_offset,
1258                                       value, pi->sram_end);
1259 }
1260 #endif
1261
1262 static int ci_write_smc_soft_register(struct radeon_device *rdev,
1263                                       u16 reg_offset, u32 value)
1264 {
1265         struct ci_power_info *pi = ci_get_pi(rdev);
1266
1267         return ci_write_smc_sram_dword(rdev,
1268                                        pi->soft_regs_start + reg_offset,
1269                                        value, pi->sram_end);
1270 }
1271
1272 static void ci_init_fps_limits(struct radeon_device *rdev)
1273 {
1274         struct ci_power_info *pi = ci_get_pi(rdev);
1275         SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
1276
1277         if (pi->caps_fps) {
1278                 u16 tmp;
1279
1280                 tmp = 45;
1281                 table->FpsHighT = cpu_to_be16(tmp);
1282
1283                 tmp = 30;
1284                 table->FpsLowT = cpu_to_be16(tmp);
1285         }
1286 }
1287
1288 static int ci_update_sclk_t(struct radeon_device *rdev)
1289 {
1290         struct ci_power_info *pi = ci_get_pi(rdev);
1291         int ret = 0;
1292         u32 low_sclk_interrupt_t = 0;
1293
1294         if (pi->caps_sclk_throttle_low_notification) {
1295                 low_sclk_interrupt_t = cpu_to_be32(pi->low_sclk_interrupt_t);
1296
1297                 ret = ci_copy_bytes_to_smc(rdev,
1298                                            pi->dpm_table_start +
1299                                            offsetof(SMU7_Discrete_DpmTable, LowSclkInterruptT),
1300                                            (u8 *)&low_sclk_interrupt_t,
1301                                            sizeof(u32), pi->sram_end);
1302
1303         }
1304
1305         return ret;
1306 }
1307
1308 static void ci_get_leakage_voltages(struct radeon_device *rdev)
1309 {
1310         struct ci_power_info *pi = ci_get_pi(rdev);
1311         u16 leakage_id, virtual_voltage_id;
1312         u16 vddc, vddci;
1313         int i;
1314
1315         pi->vddc_leakage.count = 0;
1316         pi->vddci_leakage.count = 0;
1317
1318         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_EVV) {
1319                 for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1320                         virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1321                         if (radeon_atom_get_voltage_evv(rdev, virtual_voltage_id, &vddc) != 0)
1322                                 continue;
1323                         if (vddc != 0 && vddc != virtual_voltage_id) {
1324                                 pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1325                                 pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1326                                 pi->vddc_leakage.count++;
1327                         }
1328                 }
1329         } else if (radeon_atom_get_leakage_id_from_vbios(rdev, &leakage_id) == 0) {
1330                 for (i = 0; i < CISLANDS_MAX_LEAKAGE_COUNT; i++) {
1331                         virtual_voltage_id = ATOM_VIRTUAL_VOLTAGE_ID0 + i;
1332                         if (radeon_atom_get_leakage_vddc_based_on_leakage_params(rdev, &vddc, &vddci,
1333                                                                                  virtual_voltage_id,
1334                                                                                  leakage_id) == 0) {
1335                                 if (vddc != 0 && vddc != virtual_voltage_id) {
1336                                         pi->vddc_leakage.actual_voltage[pi->vddc_leakage.count] = vddc;
1337                                         pi->vddc_leakage.leakage_id[pi->vddc_leakage.count] = virtual_voltage_id;
1338                                         pi->vddc_leakage.count++;
1339                                 }
1340                                 if (vddci != 0 && vddci != virtual_voltage_id) {
1341                                         pi->vddci_leakage.actual_voltage[pi->vddci_leakage.count] = vddci;
1342                                         pi->vddci_leakage.leakage_id[pi->vddci_leakage.count] = virtual_voltage_id;
1343                                         pi->vddci_leakage.count++;
1344                                 }
1345                         }
1346                 }
1347         }
1348 }
1349
1350 static void ci_set_dpm_event_sources(struct radeon_device *rdev, u32 sources)
1351 {
1352         struct ci_power_info *pi = ci_get_pi(rdev);
1353         bool want_thermal_protection;
1354         u32 tmp;
1355
1356         switch (sources) {
1357         case 0:
1358         default:
1359                 want_thermal_protection = false;
1360                 break;
1361         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL):
1362                 want_thermal_protection = true;
1363                 break;
1364         case (1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL):
1365                 want_thermal_protection = true;
1366                 break;
1367         case ((1 << RADEON_DPM_AUTO_THROTTLE_SRC_EXTERNAL) |
1368               (1 << RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL)):
1369                 want_thermal_protection = true;
1370                 break;
1371         }
1372
1373         if (want_thermal_protection) {
1374                 tmp = RREG32_SMC(GENERAL_PWRMGT);
1375                 if (pi->thermal_protection)
1376                         tmp &= ~THERMAL_PROTECTION_DIS;
1377                 else
1378                         tmp |= THERMAL_PROTECTION_DIS;
1379                 WREG32_SMC(GENERAL_PWRMGT, tmp);
1380         } else {
1381                 tmp = RREG32_SMC(GENERAL_PWRMGT);
1382                 tmp |= THERMAL_PROTECTION_DIS;
1383                 WREG32_SMC(GENERAL_PWRMGT, tmp);
1384         }
1385 }
1386
1387 static void ci_enable_auto_throttle_source(struct radeon_device *rdev,
1388                                            enum radeon_dpm_auto_throttle_src source,
1389                                            bool enable)
1390 {
1391         struct ci_power_info *pi = ci_get_pi(rdev);
1392
1393         if (enable) {
1394                 if (!(pi->active_auto_throttle_sources & (1 << source))) {
1395                         pi->active_auto_throttle_sources |= 1 << source;
1396                         ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1397                 }
1398         } else {
1399                 if (pi->active_auto_throttle_sources & (1 << source)) {
1400                         pi->active_auto_throttle_sources &= ~(1 << source);
1401                         ci_set_dpm_event_sources(rdev, pi->active_auto_throttle_sources);
1402                 }
1403         }
1404 }
1405
1406 static void ci_enable_vr_hot_gpio_interrupt(struct radeon_device *rdev)
1407 {
1408         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_REGULATOR_HOT)
1409                 ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableVRHotGPIOInterrupt);
1410 }
1411
1412 static int ci_unfreeze_sclk_mclk_dpm(struct radeon_device *rdev)
1413 {
1414         struct ci_power_info *pi = ci_get_pi(rdev);
1415         PPSMC_Result smc_result;
1416
1417         if (!pi->need_update_smu7_dpm_table)
1418                 return 0;
1419
1420         if ((!pi->sclk_dpm_key_disabled) &&
1421             (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1422                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_UnfreezeLevel);
1423                 if (smc_result != PPSMC_Result_OK)
1424                         return -EINVAL;
1425         }
1426
1427         if ((!pi->mclk_dpm_key_disabled) &&
1428             (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1429                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_UnfreezeLevel);
1430                 if (smc_result != PPSMC_Result_OK)
1431                         return -EINVAL;
1432         }
1433
1434         pi->need_update_smu7_dpm_table = 0;
1435         return 0;
1436 }
1437
1438 static int ci_enable_sclk_mclk_dpm(struct radeon_device *rdev, bool enable)
1439 {
1440         struct ci_power_info *pi = ci_get_pi(rdev);
1441         PPSMC_Result smc_result;
1442
1443         if (enable) {
1444                 if (!pi->sclk_dpm_key_disabled) {
1445                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Enable);
1446                         if (smc_result != PPSMC_Result_OK)
1447                                 return -EINVAL;
1448                 }
1449
1450                 if (!pi->mclk_dpm_key_disabled) {
1451                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Enable);
1452                         if (smc_result != PPSMC_Result_OK)
1453                                 return -EINVAL;
1454
1455                         WREG32_P(MC_SEQ_CNTL_3, CAC_EN, ~CAC_EN);
1456
1457                         WREG32_SMC(LCAC_MC0_CNTL, 0x05);
1458                         WREG32_SMC(LCAC_MC1_CNTL, 0x05);
1459                         WREG32_SMC(LCAC_CPL_CNTL, 0x100005);
1460
1461                         udelay(10);
1462
1463                         WREG32_SMC(LCAC_MC0_CNTL, 0x400005);
1464                         WREG32_SMC(LCAC_MC1_CNTL, 0x400005);
1465                         WREG32_SMC(LCAC_CPL_CNTL, 0x500005);
1466                 }
1467         } else {
1468                 if (!pi->sclk_dpm_key_disabled) {
1469                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_DPM_Disable);
1470                         if (smc_result != PPSMC_Result_OK)
1471                                 return -EINVAL;
1472                 }
1473
1474                 if (!pi->mclk_dpm_key_disabled) {
1475                         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_Disable);
1476                         if (smc_result != PPSMC_Result_OK)
1477                                 return -EINVAL;
1478                 }
1479         }
1480
1481         return 0;
1482 }
1483
1484 static int ci_start_dpm(struct radeon_device *rdev)
1485 {
1486         struct ci_power_info *pi = ci_get_pi(rdev);
1487         PPSMC_Result smc_result;
1488         int ret;
1489         u32 tmp;
1490
1491         tmp = RREG32_SMC(GENERAL_PWRMGT);
1492         tmp |= GLOBAL_PWRMGT_EN;
1493         WREG32_SMC(GENERAL_PWRMGT, tmp);
1494
1495         tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1496         tmp |= DYNAMIC_PM_EN;
1497         WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1498
1499         ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VoltageChangeTimeout), 0x1000);
1500
1501         WREG32_P(BIF_LNCNT_RESET, 0, ~RESET_LNCNT_EN);
1502
1503         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Enable);
1504         if (smc_result != PPSMC_Result_OK)
1505                 return -EINVAL;
1506
1507         ret = ci_enable_sclk_mclk_dpm(rdev, true);
1508         if (ret)
1509                 return ret;
1510
1511         if (!pi->pcie_dpm_key_disabled) {
1512                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Enable);
1513                 if (smc_result != PPSMC_Result_OK)
1514                         return -EINVAL;
1515         }
1516
1517         return 0;
1518 }
1519
1520 static int ci_freeze_sclk_mclk_dpm(struct radeon_device *rdev)
1521 {
1522         struct ci_power_info *pi = ci_get_pi(rdev);
1523         PPSMC_Result smc_result;
1524
1525         if (!pi->need_update_smu7_dpm_table)
1526                 return 0;
1527
1528         if ((!pi->sclk_dpm_key_disabled) &&
1529             (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK))) {
1530                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_SCLKDPM_FreezeLevel);
1531                 if (smc_result != PPSMC_Result_OK)
1532                         return -EINVAL;
1533         }
1534
1535         if ((!pi->mclk_dpm_key_disabled) &&
1536             (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)) {
1537                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_MCLKDPM_FreezeLevel);
1538                 if (smc_result != PPSMC_Result_OK)
1539                         return -EINVAL;
1540         }
1541
1542         return 0;
1543 }
1544
1545 static int ci_stop_dpm(struct radeon_device *rdev)
1546 {
1547         struct ci_power_info *pi = ci_get_pi(rdev);
1548         PPSMC_Result smc_result;
1549         int ret;
1550         u32 tmp;
1551
1552         tmp = RREG32_SMC(GENERAL_PWRMGT);
1553         tmp &= ~GLOBAL_PWRMGT_EN;
1554         WREG32_SMC(GENERAL_PWRMGT, tmp);
1555
1556         tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1557         tmp &= ~DYNAMIC_PM_EN;
1558         WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1559
1560         if (!pi->pcie_dpm_key_disabled) {
1561                 smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_PCIeDPM_Disable);
1562                 if (smc_result != PPSMC_Result_OK)
1563                         return -EINVAL;
1564         }
1565
1566         ret = ci_enable_sclk_mclk_dpm(rdev, false);
1567         if (ret)
1568                 return ret;
1569
1570         smc_result = ci_send_msg_to_smc(rdev, PPSMC_MSG_Voltage_Cntl_Disable);
1571         if (smc_result != PPSMC_Result_OK)
1572                 return -EINVAL;
1573
1574         return 0;
1575 }
1576
1577 static void ci_enable_sclk_control(struct radeon_device *rdev, bool enable)
1578 {
1579         u32 tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
1580
1581         if (enable)
1582                 tmp &= ~SCLK_PWRMGT_OFF;
1583         else
1584                 tmp |= SCLK_PWRMGT_OFF;
1585         WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
1586 }
1587
1588 #if 0
1589 static int ci_notify_hw_of_power_source(struct radeon_device *rdev,
1590                                         bool ac_power)
1591 {
1592         struct ci_power_info *pi = ci_get_pi(rdev);
1593         struct radeon_cac_tdp_table *cac_tdp_table =
1594                 rdev->pm.dpm.dyn_state.cac_tdp_table;
1595         u32 power_limit;
1596
1597         if (ac_power)
1598                 power_limit = (u32)(cac_tdp_table->maximum_power_delivery_limit * 256);
1599         else
1600                 power_limit = (u32)(cac_tdp_table->battery_power_limit * 256);
1601
1602         ci_set_power_limit(rdev, power_limit);
1603
1604         if (pi->caps_automatic_dc_transition) {
1605                 if (ac_power)
1606                         ci_send_msg_to_smc(rdev, PPSMC_MSG_RunningOnAC);
1607                 else
1608                         ci_send_msg_to_smc(rdev, PPSMC_MSG_Remove_DC_Clamp);
1609         }
1610
1611         return 0;
1612 }
1613 #endif
1614
1615 static PPSMC_Result ci_send_msg_to_smc(struct radeon_device *rdev, PPSMC_Msg msg)
1616 {
1617         u32 tmp;
1618         int i;
1619
1620         if (!ci_is_smc_running(rdev))
1621                 return PPSMC_Result_Failed;
1622
1623         WREG32(SMC_MESSAGE_0, msg);
1624
1625         for (i = 0; i < rdev->usec_timeout; i++) {
1626                 tmp = RREG32(SMC_RESP_0);
1627                 if (tmp != 0)
1628                         break;
1629                 udelay(1);
1630         }
1631         tmp = RREG32(SMC_RESP_0);
1632
1633         return (PPSMC_Result)tmp;
1634 }
1635
1636 static PPSMC_Result ci_send_msg_to_smc_with_parameter(struct radeon_device *rdev,
1637                                                       PPSMC_Msg msg, u32 parameter)
1638 {
1639         WREG32(SMC_MSG_ARG_0, parameter);
1640         return ci_send_msg_to_smc(rdev, msg);
1641 }
1642
1643 static PPSMC_Result ci_send_msg_to_smc_return_parameter(struct radeon_device *rdev,
1644                                                         PPSMC_Msg msg, u32 *parameter)
1645 {
1646         PPSMC_Result smc_result;
1647
1648         smc_result = ci_send_msg_to_smc(rdev, msg);
1649
1650         if ((smc_result == PPSMC_Result_OK) && parameter)
1651                 *parameter = RREG32(SMC_MSG_ARG_0);
1652
1653         return smc_result;
1654 }
1655
1656 static int ci_dpm_force_state_sclk(struct radeon_device *rdev, u32 n)
1657 {
1658         struct ci_power_info *pi = ci_get_pi(rdev);
1659
1660         if (!pi->sclk_dpm_key_disabled) {
1661                 PPSMC_Result smc_result =
1662                         ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_SCLKDPM_SetEnabledMask, 1 << n);
1663                 if (smc_result != PPSMC_Result_OK)
1664                         return -EINVAL;
1665         }
1666
1667         return 0;
1668 }
1669
1670 static int ci_dpm_force_state_mclk(struct radeon_device *rdev, u32 n)
1671 {
1672         struct ci_power_info *pi = ci_get_pi(rdev);
1673
1674         if (!pi->mclk_dpm_key_disabled) {
1675                 PPSMC_Result smc_result =
1676                         ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_MCLKDPM_SetEnabledMask, 1 << n);
1677                 if (smc_result != PPSMC_Result_OK)
1678                         return -EINVAL;
1679         }
1680
1681         return 0;
1682 }
1683
1684 static int ci_dpm_force_state_pcie(struct radeon_device *rdev, u32 n)
1685 {
1686         struct ci_power_info *pi = ci_get_pi(rdev);
1687
1688         if (!pi->pcie_dpm_key_disabled) {
1689                 PPSMC_Result smc_result =
1690                         ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PCIeDPM_ForceLevel, n);
1691                 if (smc_result != PPSMC_Result_OK)
1692                         return -EINVAL;
1693         }
1694
1695         return 0;
1696 }
1697
1698 static int ci_set_power_limit(struct radeon_device *rdev, u32 n)
1699 {
1700         struct ci_power_info *pi = ci_get_pi(rdev);
1701
1702         if (pi->power_containment_features & POWERCONTAINMENT_FEATURE_PkgPwrLimit) {
1703                 PPSMC_Result smc_result =
1704                         ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_PkgPwrSetLimit, n);
1705                 if (smc_result != PPSMC_Result_OK)
1706                         return -EINVAL;
1707         }
1708
1709         return 0;
1710 }
1711
1712 static int ci_set_overdrive_target_tdp(struct radeon_device *rdev,
1713                                        u32 target_tdp)
1714 {
1715         PPSMC_Result smc_result =
1716                 ci_send_msg_to_smc_with_parameter(rdev, PPSMC_MSG_OverDriveSetTargetTdp, target_tdp);
1717         if (smc_result != PPSMC_Result_OK)
1718                 return -EINVAL;
1719         return 0;
1720 }
1721
1722 #if 0
1723 static int ci_set_boot_state(struct radeon_device *rdev)
1724 {
1725         return ci_enable_sclk_mclk_dpm(rdev, false);
1726 }
1727 #endif
1728
1729 static u32 ci_get_average_sclk_freq(struct radeon_device *rdev)
1730 {
1731         u32 sclk_freq;
1732         PPSMC_Result smc_result =
1733                 ci_send_msg_to_smc_return_parameter(rdev,
1734                                                     PPSMC_MSG_API_GetSclkFrequency,
1735                                                     &sclk_freq);
1736         if (smc_result != PPSMC_Result_OK)
1737                 sclk_freq = 0;
1738
1739         return sclk_freq;
1740 }
1741
1742 static u32 ci_get_average_mclk_freq(struct radeon_device *rdev)
1743 {
1744         u32 mclk_freq;
1745         PPSMC_Result smc_result =
1746                 ci_send_msg_to_smc_return_parameter(rdev,
1747                                                     PPSMC_MSG_API_GetMclkFrequency,
1748                                                     &mclk_freq);
1749         if (smc_result != PPSMC_Result_OK)
1750                 mclk_freq = 0;
1751
1752         return mclk_freq;
1753 }
1754
1755 static void ci_dpm_start_smc(struct radeon_device *rdev)
1756 {
1757         int i;
1758
1759         ci_program_jump_on_start(rdev);
1760         ci_start_smc_clock(rdev);
1761         ci_start_smc(rdev);
1762         for (i = 0; i < rdev->usec_timeout; i++) {
1763                 if (RREG32_SMC(FIRMWARE_FLAGS) & INTERRUPTS_ENABLED)
1764                         break;
1765         }
1766 }
1767
1768 static void ci_dpm_stop_smc(struct radeon_device *rdev)
1769 {
1770         ci_reset_smc(rdev);
1771         ci_stop_smc_clock(rdev);
1772 }
1773
1774 static int ci_process_firmware_header(struct radeon_device *rdev)
1775 {
1776         struct ci_power_info *pi = ci_get_pi(rdev);
1777         u32 tmp;
1778         int ret;
1779
1780         ret = ci_read_smc_sram_dword(rdev,
1781                                      SMU7_FIRMWARE_HEADER_LOCATION +
1782                                      offsetof(SMU7_Firmware_Header, DpmTable),
1783                                      &tmp, pi->sram_end);
1784         if (ret)
1785                 return ret;
1786
1787         pi->dpm_table_start = tmp;
1788
1789         ret = ci_read_smc_sram_dword(rdev,
1790                                      SMU7_FIRMWARE_HEADER_LOCATION +
1791                                      offsetof(SMU7_Firmware_Header, SoftRegisters),
1792                                      &tmp, pi->sram_end);
1793         if (ret)
1794                 return ret;
1795
1796         pi->soft_regs_start = tmp;
1797
1798         ret = ci_read_smc_sram_dword(rdev,
1799                                      SMU7_FIRMWARE_HEADER_LOCATION +
1800                                      offsetof(SMU7_Firmware_Header, mcRegisterTable),
1801                                      &tmp, pi->sram_end);
1802         if (ret)
1803                 return ret;
1804
1805         pi->mc_reg_table_start = tmp;
1806
1807         ret = ci_read_smc_sram_dword(rdev,
1808                                      SMU7_FIRMWARE_HEADER_LOCATION +
1809                                      offsetof(SMU7_Firmware_Header, FanTable),
1810                                      &tmp, pi->sram_end);
1811         if (ret)
1812                 return ret;
1813
1814         pi->fan_table_start = tmp;
1815
1816         ret = ci_read_smc_sram_dword(rdev,
1817                                      SMU7_FIRMWARE_HEADER_LOCATION +
1818                                      offsetof(SMU7_Firmware_Header, mcArbDramTimingTable),
1819                                      &tmp, pi->sram_end);
1820         if (ret)
1821                 return ret;
1822
1823         pi->arb_table_start = tmp;
1824
1825         return 0;
1826 }
1827
1828 static void ci_read_clock_registers(struct radeon_device *rdev)
1829 {
1830         struct ci_power_info *pi = ci_get_pi(rdev);
1831
1832         pi->clock_registers.cg_spll_func_cntl =
1833                 RREG32_SMC(CG_SPLL_FUNC_CNTL);
1834         pi->clock_registers.cg_spll_func_cntl_2 =
1835                 RREG32_SMC(CG_SPLL_FUNC_CNTL_2);
1836         pi->clock_registers.cg_spll_func_cntl_3 =
1837                 RREG32_SMC(CG_SPLL_FUNC_CNTL_3);
1838         pi->clock_registers.cg_spll_func_cntl_4 =
1839                 RREG32_SMC(CG_SPLL_FUNC_CNTL_4);
1840         pi->clock_registers.cg_spll_spread_spectrum =
1841                 RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM);
1842         pi->clock_registers.cg_spll_spread_spectrum_2 =
1843                 RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM_2);
1844         pi->clock_registers.dll_cntl = RREG32(DLL_CNTL);
1845         pi->clock_registers.mclk_pwrmgt_cntl = RREG32(MCLK_PWRMGT_CNTL);
1846         pi->clock_registers.mpll_ad_func_cntl = RREG32(MPLL_AD_FUNC_CNTL);
1847         pi->clock_registers.mpll_dq_func_cntl = RREG32(MPLL_DQ_FUNC_CNTL);
1848         pi->clock_registers.mpll_func_cntl = RREG32(MPLL_FUNC_CNTL);
1849         pi->clock_registers.mpll_func_cntl_1 = RREG32(MPLL_FUNC_CNTL_1);
1850         pi->clock_registers.mpll_func_cntl_2 = RREG32(MPLL_FUNC_CNTL_2);
1851         pi->clock_registers.mpll_ss1 = RREG32(MPLL_SS1);
1852         pi->clock_registers.mpll_ss2 = RREG32(MPLL_SS2);
1853 }
1854
1855 static void ci_init_sclk_t(struct radeon_device *rdev)
1856 {
1857         struct ci_power_info *pi = ci_get_pi(rdev);
1858
1859         pi->low_sclk_interrupt_t = 0;
1860 }
1861
1862 static void ci_enable_thermal_protection(struct radeon_device *rdev,
1863                                          bool enable)
1864 {
1865         u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
1866
1867         if (enable)
1868                 tmp &= ~THERMAL_PROTECTION_DIS;
1869         else
1870                 tmp |= THERMAL_PROTECTION_DIS;
1871         WREG32_SMC(GENERAL_PWRMGT, tmp);
1872 }
1873
1874 static void ci_enable_acpi_power_management(struct radeon_device *rdev)
1875 {
1876         u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
1877
1878         tmp |= STATIC_PM_EN;
1879
1880         WREG32_SMC(GENERAL_PWRMGT, tmp);
1881 }
1882
1883 #if 0
1884 static int ci_enter_ulp_state(struct radeon_device *rdev)
1885 {
1886
1887         WREG32(SMC_MESSAGE_0, PPSMC_MSG_SwitchToMinimumPower);
1888
1889         udelay(25000);
1890
1891         return 0;
1892 }
1893
1894 static int ci_exit_ulp_state(struct radeon_device *rdev)
1895 {
1896         int i;
1897
1898         WREG32(SMC_MESSAGE_0, PPSMC_MSG_ResumeFromMinimumPower);
1899
1900         udelay(7000);
1901
1902         for (i = 0; i < rdev->usec_timeout; i++) {
1903                 if (RREG32(SMC_RESP_0) == 1)
1904                         break;
1905                 udelay(1000);
1906         }
1907
1908         return 0;
1909 }
1910 #endif
1911
1912 static int ci_notify_smc_display_change(struct radeon_device *rdev,
1913                                         bool has_display)
1914 {
1915         PPSMC_Msg msg = has_display ? PPSMC_MSG_HasDisplay : PPSMC_MSG_NoDisplay;
1916
1917         return (ci_send_msg_to_smc(rdev, msg) == PPSMC_Result_OK) ?  0 : -EINVAL;
1918 }
1919
1920 static int ci_enable_ds_master_switch(struct radeon_device *rdev,
1921                                       bool enable)
1922 {
1923         struct ci_power_info *pi = ci_get_pi(rdev);
1924
1925         if (enable) {
1926                 if (pi->caps_sclk_ds) {
1927                         if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_ON) != PPSMC_Result_OK)
1928                                 return -EINVAL;
1929                 } else {
1930                         if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
1931                                 return -EINVAL;
1932                 }
1933         } else {
1934                 if (pi->caps_sclk_ds) {
1935                         if (ci_send_msg_to_smc(rdev, PPSMC_MSG_MASTER_DeepSleep_OFF) != PPSMC_Result_OK)
1936                                 return -EINVAL;
1937                 }
1938         }
1939
1940         return 0;
1941 }
1942
1943 static void ci_program_display_gap(struct radeon_device *rdev)
1944 {
1945         u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL);
1946         u32 pre_vbi_time_in_us;
1947         u32 frame_time_in_us;
1948         u32 ref_clock = rdev->clock.spll.reference_freq;
1949         u32 refresh_rate = r600_dpm_get_vrefresh(rdev);
1950         u32 vblank_time = r600_dpm_get_vblank_time(rdev);
1951
1952         tmp &= ~DISP_GAP_MASK;
1953         if (rdev->pm.dpm.new_active_crtc_count > 0)
1954                 tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_VBLANK_OR_WM);
1955         else
1956                 tmp |= DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE);
1957         WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp);
1958
1959         if (refresh_rate == 0)
1960                 refresh_rate = 60;
1961         if (vblank_time == 0xffffffff)
1962                 vblank_time = 500;
1963         frame_time_in_us = 1000000 / refresh_rate;
1964         pre_vbi_time_in_us =
1965                 frame_time_in_us - 200 - vblank_time;
1966         tmp = pre_vbi_time_in_us * (ref_clock / 100);
1967
1968         WREG32_SMC(CG_DISPLAY_GAP_CNTL2, tmp);
1969         ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, PreVBlankGap), 0x64);
1970         ci_write_smc_soft_register(rdev, offsetof(SMU7_SoftRegisters, VBlankTimeout), (frame_time_in_us - pre_vbi_time_in_us));
1971
1972
1973         ci_notify_smc_display_change(rdev, (rdev->pm.dpm.new_active_crtc_count == 1));
1974
1975 }
1976
1977 static void ci_enable_spread_spectrum(struct radeon_device *rdev, bool enable)
1978 {
1979         struct ci_power_info *pi = ci_get_pi(rdev);
1980         u32 tmp;
1981
1982         if (enable) {
1983                 if (pi->caps_sclk_ss_support) {
1984                         tmp = RREG32_SMC(GENERAL_PWRMGT);
1985                         tmp |= DYN_SPREAD_SPECTRUM_EN;
1986                         WREG32_SMC(GENERAL_PWRMGT, tmp);
1987                 }
1988         } else {
1989                 tmp = RREG32_SMC(CG_SPLL_SPREAD_SPECTRUM);
1990                 tmp &= ~SSEN;
1991                 WREG32_SMC(CG_SPLL_SPREAD_SPECTRUM, tmp);
1992
1993                 tmp = RREG32_SMC(GENERAL_PWRMGT);
1994                 tmp &= ~DYN_SPREAD_SPECTRUM_EN;
1995                 WREG32_SMC(GENERAL_PWRMGT, tmp);
1996         }
1997 }
1998
1999 static void ci_program_sstp(struct radeon_device *rdev)
2000 {
2001         WREG32_SMC(CG_SSP, (SSTU(R600_SSTU_DFLT) | SST(R600_SST_DFLT)));
2002 }
2003
2004 static void ci_enable_display_gap(struct radeon_device *rdev)
2005 {
2006         u32 tmp = RREG32_SMC(CG_DISPLAY_GAP_CNTL);
2007
2008         tmp &= ~(DISP_GAP_MASK | DISP_GAP_MCHG_MASK);
2009         tmp |= (DISP_GAP(R600_PM_DISPLAY_GAP_IGNORE) |
2010                 DISP_GAP_MCHG(R600_PM_DISPLAY_GAP_VBLANK));
2011
2012         WREG32_SMC(CG_DISPLAY_GAP_CNTL, tmp);
2013 }
2014
2015 static void ci_program_vc(struct radeon_device *rdev)
2016 {
2017         u32 tmp;
2018
2019         tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
2020         tmp &= ~(RESET_SCLK_CNT | RESET_BUSY_CNT);
2021         WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
2022
2023         WREG32_SMC(CG_FTV_0, CISLANDS_VRC_DFLT0);
2024         WREG32_SMC(CG_FTV_1, CISLANDS_VRC_DFLT1);
2025         WREG32_SMC(CG_FTV_2, CISLANDS_VRC_DFLT2);
2026         WREG32_SMC(CG_FTV_3, CISLANDS_VRC_DFLT3);
2027         WREG32_SMC(CG_FTV_4, CISLANDS_VRC_DFLT4);
2028         WREG32_SMC(CG_FTV_5, CISLANDS_VRC_DFLT5);
2029         WREG32_SMC(CG_FTV_6, CISLANDS_VRC_DFLT6);
2030         WREG32_SMC(CG_FTV_7, CISLANDS_VRC_DFLT7);
2031 }
2032
2033 static void ci_clear_vc(struct radeon_device *rdev)
2034 {
2035         u32 tmp;
2036
2037         tmp = RREG32_SMC(SCLK_PWRMGT_CNTL);
2038         tmp |= (RESET_SCLK_CNT | RESET_BUSY_CNT);
2039         WREG32_SMC(SCLK_PWRMGT_CNTL, tmp);
2040
2041         WREG32_SMC(CG_FTV_0, 0);
2042         WREG32_SMC(CG_FTV_1, 0);
2043         WREG32_SMC(CG_FTV_2, 0);
2044         WREG32_SMC(CG_FTV_3, 0);
2045         WREG32_SMC(CG_FTV_4, 0);
2046         WREG32_SMC(CG_FTV_5, 0);
2047         WREG32_SMC(CG_FTV_6, 0);
2048         WREG32_SMC(CG_FTV_7, 0);
2049 }
2050
2051 static int ci_upload_firmware(struct radeon_device *rdev)
2052 {
2053         struct ci_power_info *pi = ci_get_pi(rdev);
2054         int i;
2055
2056         for (i = 0; i < rdev->usec_timeout; i++) {
2057                 if (RREG32_SMC(RCU_UC_EVENTS) & BOOT_SEQ_DONE)
2058                         break;
2059         }
2060         WREG32_SMC(SMC_SYSCON_MISC_CNTL, 1);
2061
2062         ci_stop_smc_clock(rdev);
2063         ci_reset_smc(rdev);
2064
2065         return ci_load_smc_ucode(rdev, pi->sram_end);
2066
2067 }
2068
2069 static int ci_get_svi2_voltage_table(struct radeon_device *rdev,
2070                                      struct radeon_clock_voltage_dependency_table *voltage_dependency_table,
2071                                      struct atom_voltage_table *voltage_table)
2072 {
2073         u32 i;
2074
2075         if (voltage_dependency_table == NULL)
2076                 return -EINVAL;
2077
2078         voltage_table->mask_low = 0;
2079         voltage_table->phase_delay = 0;
2080
2081         voltage_table->count = voltage_dependency_table->count;
2082         for (i = 0; i < voltage_table->count; i++) {
2083                 voltage_table->entries[i].value = voltage_dependency_table->entries[i].v;
2084                 voltage_table->entries[i].smio_low = 0;
2085         }
2086
2087         return 0;
2088 }
2089
2090 static int ci_construct_voltage_tables(struct radeon_device *rdev)
2091 {
2092         struct ci_power_info *pi = ci_get_pi(rdev);
2093         int ret;
2094
2095         if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2096                 ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDC,
2097                                                     VOLTAGE_OBJ_GPIO_LUT,
2098                                                     &pi->vddc_voltage_table);
2099                 if (ret)
2100                         return ret;
2101         } else if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2102                 ret = ci_get_svi2_voltage_table(rdev,
2103                                                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2104                                                 &pi->vddc_voltage_table);
2105                 if (ret)
2106                         return ret;
2107         }
2108
2109         if (pi->vddc_voltage_table.count > SMU7_MAX_LEVELS_VDDC)
2110                 si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDC,
2111                                                          &pi->vddc_voltage_table);
2112
2113         if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2114                 ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_VDDCI,
2115                                                     VOLTAGE_OBJ_GPIO_LUT,
2116                                                     &pi->vddci_voltage_table);
2117                 if (ret)
2118                         return ret;
2119         } else if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2120                 ret = ci_get_svi2_voltage_table(rdev,
2121                                                 &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
2122                                                 &pi->vddci_voltage_table);
2123                 if (ret)
2124                         return ret;
2125         }
2126
2127         if (pi->vddci_voltage_table.count > SMU7_MAX_LEVELS_VDDCI)
2128                 si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_VDDCI,
2129                                                          &pi->vddci_voltage_table);
2130
2131         if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO) {
2132                 ret = radeon_atom_get_voltage_table(rdev, VOLTAGE_TYPE_MVDDC,
2133                                                     VOLTAGE_OBJ_GPIO_LUT,
2134                                                     &pi->mvdd_voltage_table);
2135                 if (ret)
2136                         return ret;
2137         } else if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
2138                 ret = ci_get_svi2_voltage_table(rdev,
2139                                                 &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
2140                                                 &pi->mvdd_voltage_table);
2141                 if (ret)
2142                         return ret;
2143         }
2144
2145         if (pi->mvdd_voltage_table.count > SMU7_MAX_LEVELS_MVDD)
2146                 si_trim_voltage_table_to_fit_state_table(rdev, SMU7_MAX_LEVELS_MVDD,
2147                                                          &pi->mvdd_voltage_table);
2148
2149         return 0;
2150 }
2151
2152 static void ci_populate_smc_voltage_table(struct radeon_device *rdev,
2153                                           struct atom_voltage_table_entry *voltage_table,
2154                                           SMU7_Discrete_VoltageLevel *smc_voltage_table)
2155 {
2156         int ret;
2157
2158         ret = ci_get_std_voltage_value_sidd(rdev, voltage_table,
2159                                             &smc_voltage_table->StdVoltageHiSidd,
2160                                             &smc_voltage_table->StdVoltageLoSidd);
2161
2162         if (ret) {
2163                 smc_voltage_table->StdVoltageHiSidd = voltage_table->value * VOLTAGE_SCALE;
2164                 smc_voltage_table->StdVoltageLoSidd = voltage_table->value * VOLTAGE_SCALE;
2165         }
2166
2167         smc_voltage_table->Voltage = cpu_to_be16(voltage_table->value * VOLTAGE_SCALE);
2168         smc_voltage_table->StdVoltageHiSidd =
2169                 cpu_to_be16(smc_voltage_table->StdVoltageHiSidd);
2170         smc_voltage_table->StdVoltageLoSidd =
2171                 cpu_to_be16(smc_voltage_table->StdVoltageLoSidd);
2172 }
2173
2174 static int ci_populate_smc_vddc_table(struct radeon_device *rdev,
2175                                       SMU7_Discrete_DpmTable *table)
2176 {
2177         struct ci_power_info *pi = ci_get_pi(rdev);
2178         unsigned int count;
2179
2180         table->VddcLevelCount = pi->vddc_voltage_table.count;
2181         for (count = 0; count < table->VddcLevelCount; count++) {
2182                 ci_populate_smc_voltage_table(rdev,
2183                                               &pi->vddc_voltage_table.entries[count],
2184                                               &table->VddcLevel[count]);
2185
2186                 if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2187                         table->VddcLevel[count].Smio |=
2188                                 pi->vddc_voltage_table.entries[count].smio_low;
2189                 else
2190                         table->VddcLevel[count].Smio = 0;
2191         }
2192         table->VddcLevelCount = cpu_to_be32(table->VddcLevelCount);
2193
2194         return 0;
2195 }
2196
2197 static int ci_populate_smc_vddci_table(struct radeon_device *rdev,
2198                                        SMU7_Discrete_DpmTable *table)
2199 {
2200         unsigned int count;
2201         struct ci_power_info *pi = ci_get_pi(rdev);
2202
2203         table->VddciLevelCount = pi->vddci_voltage_table.count;
2204         for (count = 0; count < table->VddciLevelCount; count++) {
2205                 ci_populate_smc_voltage_table(rdev,
2206                                               &pi->vddci_voltage_table.entries[count],
2207                                               &table->VddciLevel[count]);
2208
2209                 if (pi->vddci_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2210                         table->VddciLevel[count].Smio |=
2211                                 pi->vddci_voltage_table.entries[count].smio_low;
2212                 else
2213                         table->VddciLevel[count].Smio = 0;
2214         }
2215         table->VddciLevelCount = cpu_to_be32(table->VddciLevelCount);
2216
2217         return 0;
2218 }
2219
2220 static int ci_populate_smc_mvdd_table(struct radeon_device *rdev,
2221                                       SMU7_Discrete_DpmTable *table)
2222 {
2223         struct ci_power_info *pi = ci_get_pi(rdev);
2224         unsigned int count;
2225
2226         table->MvddLevelCount = pi->mvdd_voltage_table.count;
2227         for (count = 0; count < table->MvddLevelCount; count++) {
2228                 ci_populate_smc_voltage_table(rdev,
2229                                               &pi->mvdd_voltage_table.entries[count],
2230                                               &table->MvddLevel[count]);
2231
2232                 if (pi->mvdd_control == CISLANDS_VOLTAGE_CONTROL_BY_GPIO)
2233                         table->MvddLevel[count].Smio |=
2234                                 pi->mvdd_voltage_table.entries[count].smio_low;
2235                 else
2236                         table->MvddLevel[count].Smio = 0;
2237         }
2238         table->MvddLevelCount = cpu_to_be32(table->MvddLevelCount);
2239
2240         return 0;
2241 }
2242
2243 static int ci_populate_smc_voltage_tables(struct radeon_device *rdev,
2244                                           SMU7_Discrete_DpmTable *table)
2245 {
2246         int ret;
2247
2248         ret = ci_populate_smc_vddc_table(rdev, table);
2249         if (ret)
2250                 return ret;
2251
2252         ret = ci_populate_smc_vddci_table(rdev, table);
2253         if (ret)
2254                 return ret;
2255
2256         ret = ci_populate_smc_mvdd_table(rdev, table);
2257         if (ret)
2258                 return ret;
2259
2260         return 0;
2261 }
2262
2263 static int ci_populate_mvdd_value(struct radeon_device *rdev, u32 mclk,
2264                                   SMU7_Discrete_VoltageLevel *voltage)
2265 {
2266         struct ci_power_info *pi = ci_get_pi(rdev);
2267         u32 i = 0;
2268
2269         if (pi->mvdd_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
2270                 for (i = 0; i < rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count; i++) {
2271                         if (mclk <= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries[i].clk) {
2272                                 voltage->Voltage = pi->mvdd_voltage_table.entries[i].value;
2273                                 break;
2274                         }
2275                 }
2276
2277                 if (i >= rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.count)
2278                         return -EINVAL;
2279         }
2280
2281         return -EINVAL;
2282 }
2283
2284 static int ci_get_std_voltage_value_sidd(struct radeon_device *rdev,
2285                                          struct atom_voltage_table_entry *voltage_table,
2286                                          u16 *std_voltage_hi_sidd, u16 *std_voltage_lo_sidd)
2287 {
2288         u16 v_index, idx;
2289         bool voltage_found = false;
2290         *std_voltage_hi_sidd = voltage_table->value * VOLTAGE_SCALE;
2291         *std_voltage_lo_sidd = voltage_table->value * VOLTAGE_SCALE;
2292
2293         if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries == NULL)
2294                 return -EINVAL;
2295
2296         if (rdev->pm.dpm.dyn_state.cac_leakage_table.entries) {
2297                 for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2298                         if (voltage_table->value ==
2299                             rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2300                                 voltage_found = true;
2301                                 if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
2302                                         idx = v_index;
2303                                 else
2304                                         idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2305                                 *std_voltage_lo_sidd =
2306                                         rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2307                                 *std_voltage_hi_sidd =
2308                                         rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2309                                 break;
2310                         }
2311                 }
2312
2313                 if (!voltage_found) {
2314                         for (v_index = 0; (u32)v_index < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; v_index++) {
2315                                 if (voltage_table->value <=
2316                                     rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[v_index].v) {
2317                                         voltage_found = true;
2318                                         if ((u32)v_index < rdev->pm.dpm.dyn_state.cac_leakage_table.count)
2319                                                 idx = v_index;
2320                                         else
2321                                                 idx = rdev->pm.dpm.dyn_state.cac_leakage_table.count - 1;
2322                                         *std_voltage_lo_sidd =
2323                                                 rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].vddc * VOLTAGE_SCALE;
2324                                         *std_voltage_hi_sidd =
2325                                                 rdev->pm.dpm.dyn_state.cac_leakage_table.entries[idx].leakage * VOLTAGE_SCALE;
2326                                         break;
2327                                 }
2328                         }
2329                 }
2330         }
2331
2332         return 0;
2333 }
2334
2335 static void ci_populate_phase_value_based_on_sclk(struct radeon_device *rdev,
2336                                                   const struct radeon_phase_shedding_limits_table *limits,
2337                                                   u32 sclk,
2338                                                   u32 *phase_shedding)
2339 {
2340         unsigned int i;
2341
2342         *phase_shedding = 1;
2343
2344         for (i = 0; i < limits->count; i++) {
2345                 if (sclk < limits->entries[i].sclk) {
2346                         *phase_shedding = i;
2347                         break;
2348                 }
2349         }
2350 }
2351
2352 static void ci_populate_phase_value_based_on_mclk(struct radeon_device *rdev,
2353                                                   const struct radeon_phase_shedding_limits_table *limits,
2354                                                   u32 mclk,
2355                                                   u32 *phase_shedding)
2356 {
2357         unsigned int i;
2358
2359         *phase_shedding = 1;
2360
2361         for (i = 0; i < limits->count; i++) {
2362                 if (mclk < limits->entries[i].mclk) {
2363                         *phase_shedding = i;
2364                         break;
2365                 }
2366         }
2367 }
2368
2369 static int ci_init_arb_table_index(struct radeon_device *rdev)
2370 {
2371         struct ci_power_info *pi = ci_get_pi(rdev);
2372         u32 tmp;
2373         int ret;
2374
2375         ret = ci_read_smc_sram_dword(rdev, pi->arb_table_start,
2376                                      &tmp, pi->sram_end);
2377         if (ret)
2378                 return ret;
2379
2380         tmp &= 0x00FFFFFF;
2381         tmp |= MC_CG_ARB_FREQ_F1 << 24;
2382
2383         return ci_write_smc_sram_dword(rdev, pi->arb_table_start,
2384                                        tmp, pi->sram_end);
2385 }
2386
2387 static int ci_get_dependency_volt_by_clk(struct radeon_device *rdev,
2388                                          struct radeon_clock_voltage_dependency_table *allowed_clock_voltage_table,
2389                                          u32 clock, u32 *voltage)
2390 {
2391         u32 i = 0;
2392
2393         if (allowed_clock_voltage_table->count == 0)
2394                 return -EINVAL;
2395
2396         for (i = 0; i < allowed_clock_voltage_table->count; i++) {
2397                 if (allowed_clock_voltage_table->entries[i].clk >= clock) {
2398                         *voltage = allowed_clock_voltage_table->entries[i].v;
2399                         return 0;
2400                 }
2401         }
2402
2403         *voltage = allowed_clock_voltage_table->entries[i-1].v;
2404
2405         return 0;
2406 }
2407
2408 static u8 ci_get_sleep_divider_id_from_clock(struct radeon_device *rdev,
2409                                              u32 sclk, u32 min_sclk_in_sr)
2410 {
2411         u32 i;
2412         u32 tmp;
2413         u32 min = (min_sclk_in_sr > CISLAND_MINIMUM_ENGINE_CLOCK) ?
2414                 min_sclk_in_sr : CISLAND_MINIMUM_ENGINE_CLOCK;
2415
2416         if (sclk < min)
2417                 return 0;
2418
2419         for (i = CISLAND_MAX_DEEPSLEEP_DIVIDER_ID;  ; i--) {
2420                 tmp = sclk / (1 << i);
2421                 if (tmp >= min || i == 0)
2422                         break;
2423         }
2424
2425         return (u8)i;
2426 }
2427
2428 static int ci_initial_switch_from_arb_f0_to_f1(struct radeon_device *rdev)
2429 {
2430         return ni_copy_and_switch_arb_sets(rdev, MC_CG_ARB_FREQ_F0, MC_CG_ARB_FREQ_F1);
2431 }
2432
2433 static int ci_reset_to_default(struct radeon_device *rdev)
2434 {
2435         return (ci_send_msg_to_smc(rdev, PPSMC_MSG_ResetToDefaults) == PPSMC_Result_OK) ?
2436                 0 : -EINVAL;
2437 }
2438
2439 static int ci_force_switch_to_arb_f0(struct radeon_device *rdev)
2440 {
2441         u32 tmp;
2442
2443         tmp = (RREG32_SMC(SMC_SCRATCH9) & 0x0000ff00) >> 8;
2444
2445         if (tmp == MC_CG_ARB_FREQ_F0)
2446                 return 0;
2447
2448         return ni_copy_and_switch_arb_sets(rdev, tmp, MC_CG_ARB_FREQ_F0);
2449 }
2450
2451 static void ci_register_patching_mc_arb(struct radeon_device *rdev,
2452                                         const u32 engine_clock,
2453                                         const u32 memory_clock,
2454                                         u32 *dram_timimg2)
2455 {
2456         bool patch;
2457         u32 tmp, tmp2;
2458
2459         tmp = RREG32(MC_SEQ_MISC0);
2460         patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
2461
2462         if (patch &&
2463             ((rdev->pdev->device == 0x67B0) ||
2464              (rdev->pdev->device == 0x67B1))) {
2465                 if ((memory_clock > 100000) && (memory_clock <= 125000)) {
2466                         tmp2 = (((0x31 * engine_clock) / 125000) - 1) & 0xff;
2467                         *dram_timimg2 &= ~0x00ff0000;
2468                         *dram_timimg2 |= tmp2 << 16;
2469                 } else if ((memory_clock > 125000) && (memory_clock <= 137500)) {
2470                         tmp2 = (((0x36 * engine_clock) / 137500) - 1) & 0xff;
2471                         *dram_timimg2 &= ~0x00ff0000;
2472                         *dram_timimg2 |= tmp2 << 16;
2473                 }
2474         }
2475 }
2476
2477
2478 static int ci_populate_memory_timing_parameters(struct radeon_device *rdev,
2479                                                 u32 sclk,
2480                                                 u32 mclk,
2481                                                 SMU7_Discrete_MCArbDramTimingTableEntry *arb_regs)
2482 {
2483         u32 dram_timing;
2484         u32 dram_timing2;
2485         u32 burst_time;
2486
2487         radeon_atom_set_engine_dram_timings(rdev, sclk, mclk);
2488
2489         dram_timing  = RREG32(MC_ARB_DRAM_TIMING);
2490         dram_timing2 = RREG32(MC_ARB_DRAM_TIMING2);
2491         burst_time = RREG32(MC_ARB_BURST_TIME) & STATE0_MASK;
2492
2493         ci_register_patching_mc_arb(rdev, sclk, mclk, &dram_timing2);
2494
2495         arb_regs->McArbDramTiming  = cpu_to_be32(dram_timing);
2496         arb_regs->McArbDramTiming2 = cpu_to_be32(dram_timing2);
2497         arb_regs->McArbBurstTime = (u8)burst_time;
2498
2499         return 0;
2500 }
2501
2502 static int ci_do_program_memory_timing_parameters(struct radeon_device *rdev)
2503 {
2504         struct ci_power_info *pi = ci_get_pi(rdev);
2505         SMU7_Discrete_MCArbDramTimingTable arb_regs;
2506         u32 i, j;
2507         int ret =  0;
2508
2509         memset(&arb_regs, 0, sizeof(SMU7_Discrete_MCArbDramTimingTable));
2510
2511         for (i = 0; i < pi->dpm_table.sclk_table.count; i++) {
2512                 for (j = 0; j < pi->dpm_table.mclk_table.count; j++) {
2513                         ret = ci_populate_memory_timing_parameters(rdev,
2514                                                                    pi->dpm_table.sclk_table.dpm_levels[i].value,
2515                                                                    pi->dpm_table.mclk_table.dpm_levels[j].value,
2516                                                                    &arb_regs.entries[i][j]);
2517                         if (ret)
2518                                 break;
2519                 }
2520         }
2521
2522         if (ret == 0)
2523                 ret = ci_copy_bytes_to_smc(rdev,
2524                                            pi->arb_table_start,
2525                                            (u8 *)&arb_regs,
2526                                            sizeof(SMU7_Discrete_MCArbDramTimingTable),
2527                                            pi->sram_end);
2528
2529         return ret;
2530 }
2531
2532 static int ci_program_memory_timing_parameters(struct radeon_device *rdev)
2533 {
2534         struct ci_power_info *pi = ci_get_pi(rdev);
2535
2536         if (pi->need_update_smu7_dpm_table == 0)
2537                 return 0;
2538
2539         return ci_do_program_memory_timing_parameters(rdev);
2540 }
2541
2542 static void ci_populate_smc_initial_state(struct radeon_device *rdev,
2543                                           struct radeon_ps *radeon_boot_state)
2544 {
2545         struct ci_ps *boot_state = ci_get_ps(radeon_boot_state);
2546         struct ci_power_info *pi = ci_get_pi(rdev);
2547         u32 level = 0;
2548
2549         for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.count; level++) {
2550                 if (rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[level].clk >=
2551                     boot_state->performance_levels[0].sclk) {
2552                         pi->smc_state_table.GraphicsBootLevel = level;
2553                         break;
2554                 }
2555         }
2556
2557         for (level = 0; level < rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.count; level++) {
2558                 if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries[level].clk >=
2559                     boot_state->performance_levels[0].mclk) {
2560                         pi->smc_state_table.MemoryBootLevel = level;
2561                         break;
2562                 }
2563         }
2564 }
2565
2566 static u32 ci_get_dpm_level_enable_mask_value(struct ci_single_dpm_table *dpm_table)
2567 {
2568         u32 i;
2569         u32 mask_value = 0;
2570
2571         for (i = dpm_table->count; i > 0; i--) {
2572                 mask_value = mask_value << 1;
2573                 if (dpm_table->dpm_levels[i-1].enabled)
2574                         mask_value |= 0x1;
2575                 else
2576                         mask_value &= 0xFFFFFFFE;
2577         }
2578
2579         return mask_value;
2580 }
2581
2582 static void ci_populate_smc_link_level(struct radeon_device *rdev,
2583                                        SMU7_Discrete_DpmTable *table)
2584 {
2585         struct ci_power_info *pi = ci_get_pi(rdev);
2586         struct ci_dpm_table *dpm_table = &pi->dpm_table;
2587         u32 i;
2588
2589         for (i = 0; i < dpm_table->pcie_speed_table.count; i++) {
2590                 table->LinkLevel[i].PcieGenSpeed =
2591                         (u8)dpm_table->pcie_speed_table.dpm_levels[i].value;
2592                 table->LinkLevel[i].PcieLaneCount =
2593                         r600_encode_pci_lane_width(dpm_table->pcie_speed_table.dpm_levels[i].param1);
2594                 table->LinkLevel[i].EnabledForActivity = 1;
2595                 table->LinkLevel[i].DownT = cpu_to_be32(5);
2596                 table->LinkLevel[i].UpT = cpu_to_be32(30);
2597         }
2598
2599         pi->smc_state_table.LinkLevelCount = (u8)dpm_table->pcie_speed_table.count;
2600         pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
2601                 ci_get_dpm_level_enable_mask_value(&dpm_table->pcie_speed_table);
2602 }
2603
2604 static int ci_populate_smc_uvd_level(struct radeon_device *rdev,
2605                                      SMU7_Discrete_DpmTable *table)
2606 {
2607         u32 count;
2608         struct atom_clock_dividers dividers;
2609         int ret = -EINVAL;
2610
2611         table->UvdLevelCount =
2612                 rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count;
2613
2614         for (count = 0; count < table->UvdLevelCount; count++) {
2615                 table->UvdLevel[count].VclkFrequency =
2616                         rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].vclk;
2617                 table->UvdLevel[count].DclkFrequency =
2618                         rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].dclk;
2619                 table->UvdLevel[count].MinVddc =
2620                         rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2621                 table->UvdLevel[count].MinVddcPhases = 1;
2622
2623                 ret = radeon_atom_get_clock_dividers(rdev,
2624                                                      COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2625                                                      table->UvdLevel[count].VclkFrequency, false, &dividers);
2626                 if (ret)
2627                         return ret;
2628
2629                 table->UvdLevel[count].VclkDivider = (u8)dividers.post_divider;
2630
2631                 ret = radeon_atom_get_clock_dividers(rdev,
2632                                                      COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2633                                                      table->UvdLevel[count].DclkFrequency, false, &dividers);
2634                 if (ret)
2635                         return ret;
2636
2637                 table->UvdLevel[count].DclkDivider = (u8)dividers.post_divider;
2638
2639                 table->UvdLevel[count].VclkFrequency = cpu_to_be32(table->UvdLevel[count].VclkFrequency);
2640                 table->UvdLevel[count].DclkFrequency = cpu_to_be32(table->UvdLevel[count].DclkFrequency);
2641                 table->UvdLevel[count].MinVddc = cpu_to_be16(table->UvdLevel[count].MinVddc);
2642         }
2643
2644         return ret;
2645 }
2646
2647 static int ci_populate_smc_vce_level(struct radeon_device *rdev,
2648                                      SMU7_Discrete_DpmTable *table)
2649 {
2650         u32 count;
2651         struct atom_clock_dividers dividers;
2652         int ret = -EINVAL;
2653
2654         table->VceLevelCount =
2655                 rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count;
2656
2657         for (count = 0; count < table->VceLevelCount; count++) {
2658                 table->VceLevel[count].Frequency =
2659                         rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].evclk;
2660                 table->VceLevel[count].MinVoltage =
2661                         (u16)rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2662                 table->VceLevel[count].MinPhases = 1;
2663
2664                 ret = radeon_atom_get_clock_dividers(rdev,
2665                                                      COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2666                                                      table->VceLevel[count].Frequency, false, &dividers);
2667                 if (ret)
2668                         return ret;
2669
2670                 table->VceLevel[count].Divider = (u8)dividers.post_divider;
2671
2672                 table->VceLevel[count].Frequency = cpu_to_be32(table->VceLevel[count].Frequency);
2673                 table->VceLevel[count].MinVoltage = cpu_to_be16(table->VceLevel[count].MinVoltage);
2674         }
2675
2676         return ret;
2677
2678 }
2679
2680 static int ci_populate_smc_acp_level(struct radeon_device *rdev,
2681                                      SMU7_Discrete_DpmTable *table)
2682 {
2683         u32 count;
2684         struct atom_clock_dividers dividers;
2685         int ret = -EINVAL;
2686
2687         table->AcpLevelCount = (u8)
2688                 (rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count);
2689
2690         for (count = 0; count < table->AcpLevelCount; count++) {
2691                 table->AcpLevel[count].Frequency =
2692                         rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].clk;
2693                 table->AcpLevel[count].MinVoltage =
2694                         rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[count].v;
2695                 table->AcpLevel[count].MinPhases = 1;
2696
2697                 ret = radeon_atom_get_clock_dividers(rdev,
2698                                                      COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2699                                                      table->AcpLevel[count].Frequency, false, &dividers);
2700                 if (ret)
2701                         return ret;
2702
2703                 table->AcpLevel[count].Divider = (u8)dividers.post_divider;
2704
2705                 table->AcpLevel[count].Frequency = cpu_to_be32(table->AcpLevel[count].Frequency);
2706                 table->AcpLevel[count].MinVoltage = cpu_to_be16(table->AcpLevel[count].MinVoltage);
2707         }
2708
2709         return ret;
2710 }
2711
2712 static int ci_populate_smc_samu_level(struct radeon_device *rdev,
2713                                       SMU7_Discrete_DpmTable *table)
2714 {
2715         u32 count;
2716         struct atom_clock_dividers dividers;
2717         int ret = -EINVAL;
2718
2719         table->SamuLevelCount =
2720                 rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count;
2721
2722         for (count = 0; count < table->SamuLevelCount; count++) {
2723                 table->SamuLevel[count].Frequency =
2724                         rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].clk;
2725                 table->SamuLevel[count].MinVoltage =
2726                         rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[count].v * VOLTAGE_SCALE;
2727                 table->SamuLevel[count].MinPhases = 1;
2728
2729                 ret = radeon_atom_get_clock_dividers(rdev,
2730                                                      COMPUTE_GPUCLK_INPUT_FLAG_DEFAULT_GPUCLK,
2731                                                      table->SamuLevel[count].Frequency, false, &dividers);
2732                 if (ret)
2733                         return ret;
2734
2735                 table->SamuLevel[count].Divider = (u8)dividers.post_divider;
2736
2737                 table->SamuLevel[count].Frequency = cpu_to_be32(table->SamuLevel[count].Frequency);
2738                 table->SamuLevel[count].MinVoltage = cpu_to_be16(table->SamuLevel[count].MinVoltage);
2739         }
2740
2741         return ret;
2742 }
2743
2744 static int ci_calculate_mclk_params(struct radeon_device *rdev,
2745                                     u32 memory_clock,
2746                                     SMU7_Discrete_MemoryLevel *mclk,
2747                                     bool strobe_mode,
2748                                     bool dll_state_on)
2749 {
2750         struct ci_power_info *pi = ci_get_pi(rdev);
2751         u32  dll_cntl = pi->clock_registers.dll_cntl;
2752         u32  mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
2753         u32  mpll_ad_func_cntl = pi->clock_registers.mpll_ad_func_cntl;
2754         u32  mpll_dq_func_cntl = pi->clock_registers.mpll_dq_func_cntl;
2755         u32  mpll_func_cntl = pi->clock_registers.mpll_func_cntl;
2756         u32  mpll_func_cntl_1 = pi->clock_registers.mpll_func_cntl_1;
2757         u32  mpll_func_cntl_2 = pi->clock_registers.mpll_func_cntl_2;
2758         u32  mpll_ss1 = pi->clock_registers.mpll_ss1;
2759         u32  mpll_ss2 = pi->clock_registers.mpll_ss2;
2760         struct atom_mpll_param mpll_param;
2761         int ret;
2762
2763         ret = radeon_atom_get_memory_pll_dividers(rdev, memory_clock, strobe_mode, &mpll_param);
2764         if (ret)
2765                 return ret;
2766
2767         mpll_func_cntl &= ~BWCTRL_MASK;
2768         mpll_func_cntl |= BWCTRL(mpll_param.bwcntl);
2769
2770         mpll_func_cntl_1 &= ~(CLKF_MASK | CLKFRAC_MASK | VCO_MODE_MASK);
2771         mpll_func_cntl_1 |= CLKF(mpll_param.clkf) |
2772                 CLKFRAC(mpll_param.clkfrac) | VCO_MODE(mpll_param.vco_mode);
2773
2774         mpll_ad_func_cntl &= ~YCLK_POST_DIV_MASK;
2775         mpll_ad_func_cntl |= YCLK_POST_DIV(mpll_param.post_div);
2776
2777         if (pi->mem_gddr5) {
2778                 mpll_dq_func_cntl &= ~(YCLK_SEL_MASK | YCLK_POST_DIV_MASK);
2779                 mpll_dq_func_cntl |= YCLK_SEL(mpll_param.yclk_sel) |
2780                         YCLK_POST_DIV(mpll_param.post_div);
2781         }
2782
2783         if (pi->caps_mclk_ss_support) {
2784                 struct radeon_atom_ss ss;
2785                 u32 freq_nom;
2786                 u32 tmp;
2787                 u32 reference_clock = rdev->clock.mpll.reference_freq;
2788
2789                 if (mpll_param.qdr == 1)
2790                         freq_nom = memory_clock * 4 * (1 << mpll_param.post_div);
2791                 else
2792                         freq_nom = memory_clock * 2 * (1 << mpll_param.post_div);
2793
2794                 tmp = (freq_nom / reference_clock);
2795                 tmp = tmp * tmp;
2796                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
2797                                                      ASIC_INTERNAL_MEMORY_SS, freq_nom)) {
2798                         u32 clks = reference_clock * 5 / ss.rate;
2799                         u32 clkv = (u32)((((131 * ss.percentage * ss.rate) / 100) * tmp) / freq_nom);
2800
2801                         mpll_ss1 &= ~CLKV_MASK;
2802                         mpll_ss1 |= CLKV(clkv);
2803
2804                         mpll_ss2 &= ~CLKS_MASK;
2805                         mpll_ss2 |= CLKS(clks);
2806                 }
2807         }
2808
2809         mclk_pwrmgt_cntl &= ~DLL_SPEED_MASK;
2810         mclk_pwrmgt_cntl |= DLL_SPEED(mpll_param.dll_speed);
2811
2812         if (dll_state_on)
2813                 mclk_pwrmgt_cntl |= MRDCK0_PDNB | MRDCK1_PDNB;
2814         else
2815                 mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
2816
2817         mclk->MclkFrequency = memory_clock;
2818         mclk->MpllFuncCntl = mpll_func_cntl;
2819         mclk->MpllFuncCntl_1 = mpll_func_cntl_1;
2820         mclk->MpllFuncCntl_2 = mpll_func_cntl_2;
2821         mclk->MpllAdFuncCntl = mpll_ad_func_cntl;
2822         mclk->MpllDqFuncCntl = mpll_dq_func_cntl;
2823         mclk->MclkPwrmgtCntl = mclk_pwrmgt_cntl;
2824         mclk->DllCntl = dll_cntl;
2825         mclk->MpllSs1 = mpll_ss1;
2826         mclk->MpllSs2 = mpll_ss2;
2827
2828         return 0;
2829 }
2830
2831 static int ci_populate_single_memory_level(struct radeon_device *rdev,
2832                                            u32 memory_clock,
2833                                            SMU7_Discrete_MemoryLevel *memory_level)
2834 {
2835         struct ci_power_info *pi = ci_get_pi(rdev);
2836         int ret;
2837         bool dll_state_on;
2838
2839         if (rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk.entries) {
2840                 ret = ci_get_dependency_volt_by_clk(rdev,
2841                                                     &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk,
2842                                                     memory_clock, &memory_level->MinVddc);
2843                 if (ret)
2844                         return ret;
2845         }
2846
2847         if (rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk.entries) {
2848                 ret = ci_get_dependency_volt_by_clk(rdev,
2849                                                     &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk,
2850                                                     memory_clock, &memory_level->MinVddci);
2851                 if (ret)
2852                         return ret;
2853         }
2854
2855         if (rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk.entries) {
2856                 ret = ci_get_dependency_volt_by_clk(rdev,
2857                                                     &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk,
2858                                                     memory_clock, &memory_level->MinMvdd);
2859                 if (ret)
2860                         return ret;
2861         }
2862
2863         memory_level->MinVddcPhases = 1;
2864
2865         if (pi->vddc_phase_shed_control)
2866                 ci_populate_phase_value_based_on_mclk(rdev,
2867                                                       &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
2868                                                       memory_clock,
2869                                                       &memory_level->MinVddcPhases);
2870
2871         memory_level->EnabledForThrottle = 1;
2872         memory_level->UpH = 0;
2873         memory_level->DownH = 100;
2874         memory_level->VoltageDownH = 0;
2875         memory_level->ActivityLevel = (u16)pi->mclk_activity_target;
2876
2877         memory_level->StutterEnable = false;
2878         memory_level->StrobeEnable = false;
2879         memory_level->EdcReadEnable = false;
2880         memory_level->EdcWriteEnable = false;
2881         memory_level->RttEnable = false;
2882
2883         memory_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
2884
2885         if (pi->mclk_stutter_mode_threshold &&
2886             (memory_clock <= pi->mclk_stutter_mode_threshold) &&
2887             (pi->uvd_enabled == false) &&
2888             (RREG32(DPG_PIPE_STUTTER_CONTROL) & STUTTER_ENABLE) &&
2889             (rdev->pm.dpm.new_active_crtc_count <= 2))
2890                 memory_level->StutterEnable = true;
2891
2892         if (pi->mclk_strobe_mode_threshold &&
2893             (memory_clock <= pi->mclk_strobe_mode_threshold))
2894                 memory_level->StrobeEnable = 1;
2895
2896         if (pi->mem_gddr5) {
2897                 memory_level->StrobeRatio =
2898                         si_get_mclk_frequency_ratio(memory_clock, memory_level->StrobeEnable);
2899                 if (pi->mclk_edc_enable_threshold &&
2900                     (memory_clock > pi->mclk_edc_enable_threshold))
2901                         memory_level->EdcReadEnable = true;
2902
2903                 if (pi->mclk_edc_wr_enable_threshold &&
2904                     (memory_clock > pi->mclk_edc_wr_enable_threshold))
2905                         memory_level->EdcWriteEnable = true;
2906
2907                 if (memory_level->StrobeEnable) {
2908                         if (si_get_mclk_frequency_ratio(memory_clock, true) >=
2909                             ((RREG32(MC_SEQ_MISC7) >> 16) & 0xf))
2910                                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2911                         else
2912                                 dll_state_on = ((RREG32(MC_SEQ_MISC6) >> 1) & 0x1) ? true : false;
2913                 } else {
2914                         dll_state_on = pi->dll_default_on;
2915                 }
2916         } else {
2917                 memory_level->StrobeRatio = si_get_ddr3_mclk_frequency_ratio(memory_clock);
2918                 dll_state_on = ((RREG32(MC_SEQ_MISC5) >> 1) & 0x1) ? true : false;
2919         }
2920
2921         ret = ci_calculate_mclk_params(rdev, memory_clock, memory_level, memory_level->StrobeEnable, dll_state_on);
2922         if (ret)
2923                 return ret;
2924
2925         memory_level->MinVddc = cpu_to_be32(memory_level->MinVddc * VOLTAGE_SCALE);
2926         memory_level->MinVddcPhases = cpu_to_be32(memory_level->MinVddcPhases);
2927         memory_level->MinVddci = cpu_to_be32(memory_level->MinVddci * VOLTAGE_SCALE);
2928         memory_level->MinMvdd = cpu_to_be32(memory_level->MinMvdd * VOLTAGE_SCALE);
2929
2930         memory_level->MclkFrequency = cpu_to_be32(memory_level->MclkFrequency);
2931         memory_level->ActivityLevel = cpu_to_be16(memory_level->ActivityLevel);
2932         memory_level->MpllFuncCntl = cpu_to_be32(memory_level->MpllFuncCntl);
2933         memory_level->MpllFuncCntl_1 = cpu_to_be32(memory_level->MpllFuncCntl_1);
2934         memory_level->MpllFuncCntl_2 = cpu_to_be32(memory_level->MpllFuncCntl_2);
2935         memory_level->MpllAdFuncCntl = cpu_to_be32(memory_level->MpllAdFuncCntl);
2936         memory_level->MpllDqFuncCntl = cpu_to_be32(memory_level->MpllDqFuncCntl);
2937         memory_level->MclkPwrmgtCntl = cpu_to_be32(memory_level->MclkPwrmgtCntl);
2938         memory_level->DllCntl = cpu_to_be32(memory_level->DllCntl);
2939         memory_level->MpllSs1 = cpu_to_be32(memory_level->MpllSs1);
2940         memory_level->MpllSs2 = cpu_to_be32(memory_level->MpllSs2);
2941
2942         return 0;
2943 }
2944
2945 static int ci_populate_smc_acpi_level(struct radeon_device *rdev,
2946                                       SMU7_Discrete_DpmTable *table)
2947 {
2948         struct ci_power_info *pi = ci_get_pi(rdev);
2949         struct atom_clock_dividers dividers;
2950         SMU7_Discrete_VoltageLevel voltage_level;
2951         u32 spll_func_cntl = pi->clock_registers.cg_spll_func_cntl;
2952         u32 spll_func_cntl_2 = pi->clock_registers.cg_spll_func_cntl_2;
2953         u32 dll_cntl = pi->clock_registers.dll_cntl;
2954         u32 mclk_pwrmgt_cntl = pi->clock_registers.mclk_pwrmgt_cntl;
2955         int ret;
2956
2957         table->ACPILevel.Flags &= ~PPSMC_SWSTATE_FLAG_DC;
2958
2959         if (pi->acpi_vddc)
2960                 table->ACPILevel.MinVddc = cpu_to_be32(pi->acpi_vddc * VOLTAGE_SCALE);
2961         else
2962                 table->ACPILevel.MinVddc = cpu_to_be32(pi->min_vddc_in_pp_table * VOLTAGE_SCALE);
2963
2964         table->ACPILevel.MinVddcPhases = pi->vddc_phase_shed_control ? 0 : 1;
2965
2966         table->ACPILevel.SclkFrequency = rdev->clock.spll.reference_freq;
2967
2968         ret = radeon_atom_get_clock_dividers(rdev,
2969                                              COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
2970                                              table->ACPILevel.SclkFrequency, false, &dividers);
2971         if (ret)
2972                 return ret;
2973
2974         table->ACPILevel.SclkDid = (u8)dividers.post_divider;
2975         table->ACPILevel.DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
2976         table->ACPILevel.DeepSleepDivId = 0;
2977
2978         spll_func_cntl &= ~SPLL_PWRON;
2979         spll_func_cntl |= SPLL_RESET;
2980
2981         spll_func_cntl_2 &= ~SCLK_MUX_SEL_MASK;
2982         spll_func_cntl_2 |= SCLK_MUX_SEL(4);
2983
2984         table->ACPILevel.CgSpllFuncCntl = spll_func_cntl;
2985         table->ACPILevel.CgSpllFuncCntl2 = spll_func_cntl_2;
2986         table->ACPILevel.CgSpllFuncCntl3 = pi->clock_registers.cg_spll_func_cntl_3;
2987         table->ACPILevel.CgSpllFuncCntl4 = pi->clock_registers.cg_spll_func_cntl_4;
2988         table->ACPILevel.SpllSpreadSpectrum = pi->clock_registers.cg_spll_spread_spectrum;
2989         table->ACPILevel.SpllSpreadSpectrum2 = pi->clock_registers.cg_spll_spread_spectrum_2;
2990         table->ACPILevel.CcPwrDynRm = 0;
2991         table->ACPILevel.CcPwrDynRm1 = 0;
2992
2993         table->ACPILevel.Flags = cpu_to_be32(table->ACPILevel.Flags);
2994         table->ACPILevel.MinVddcPhases = cpu_to_be32(table->ACPILevel.MinVddcPhases);
2995         table->ACPILevel.SclkFrequency = cpu_to_be32(table->ACPILevel.SclkFrequency);
2996         table->ACPILevel.CgSpllFuncCntl = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl);
2997         table->ACPILevel.CgSpllFuncCntl2 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl2);
2998         table->ACPILevel.CgSpllFuncCntl3 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl3);
2999         table->ACPILevel.CgSpllFuncCntl4 = cpu_to_be32(table->ACPILevel.CgSpllFuncCntl4);
3000         table->ACPILevel.SpllSpreadSpectrum = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum);
3001         table->ACPILevel.SpllSpreadSpectrum2 = cpu_to_be32(table->ACPILevel.SpllSpreadSpectrum2);
3002         table->ACPILevel.CcPwrDynRm = cpu_to_be32(table->ACPILevel.CcPwrDynRm);
3003         table->ACPILevel.CcPwrDynRm1 = cpu_to_be32(table->ACPILevel.CcPwrDynRm1);
3004
3005         table->MemoryACPILevel.MinVddc = table->ACPILevel.MinVddc;
3006         table->MemoryACPILevel.MinVddcPhases = table->ACPILevel.MinVddcPhases;
3007
3008         if (pi->vddci_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
3009                 if (pi->acpi_vddci)
3010                         table->MemoryACPILevel.MinVddci =
3011                                 cpu_to_be32(pi->acpi_vddci * VOLTAGE_SCALE);
3012                 else
3013                         table->MemoryACPILevel.MinVddci =
3014                                 cpu_to_be32(pi->min_vddci_in_pp_table * VOLTAGE_SCALE);
3015         }
3016
3017         if (ci_populate_mvdd_value(rdev, 0, &voltage_level))
3018                 table->MemoryACPILevel.MinMvdd = 0;
3019         else
3020                 table->MemoryACPILevel.MinMvdd =
3021                         cpu_to_be32(voltage_level.Voltage * VOLTAGE_SCALE);
3022
3023         mclk_pwrmgt_cntl |= MRDCK0_RESET | MRDCK1_RESET;
3024         mclk_pwrmgt_cntl &= ~(MRDCK0_PDNB | MRDCK1_PDNB);
3025
3026         dll_cntl &= ~(MRDCK0_BYPASS | MRDCK1_BYPASS);
3027
3028         table->MemoryACPILevel.DllCntl = cpu_to_be32(dll_cntl);
3029         table->MemoryACPILevel.MclkPwrmgtCntl = cpu_to_be32(mclk_pwrmgt_cntl);
3030         table->MemoryACPILevel.MpllAdFuncCntl =
3031                 cpu_to_be32(pi->clock_registers.mpll_ad_func_cntl);
3032         table->MemoryACPILevel.MpllDqFuncCntl =
3033                 cpu_to_be32(pi->clock_registers.mpll_dq_func_cntl);
3034         table->MemoryACPILevel.MpllFuncCntl =
3035                 cpu_to_be32(pi->clock_registers.mpll_func_cntl);
3036         table->MemoryACPILevel.MpllFuncCntl_1 =
3037                 cpu_to_be32(pi->clock_registers.mpll_func_cntl_1);
3038         table->MemoryACPILevel.MpllFuncCntl_2 =
3039                 cpu_to_be32(pi->clock_registers.mpll_func_cntl_2);
3040         table->MemoryACPILevel.MpllSs1 = cpu_to_be32(pi->clock_registers.mpll_ss1);
3041         table->MemoryACPILevel.MpllSs2 = cpu_to_be32(pi->clock_registers.mpll_ss2);
3042
3043         table->MemoryACPILevel.EnabledForThrottle = 0;
3044         table->MemoryACPILevel.EnabledForActivity = 0;
3045         table->MemoryACPILevel.UpH = 0;
3046         table->MemoryACPILevel.DownH = 100;
3047         table->MemoryACPILevel.VoltageDownH = 0;
3048         table->MemoryACPILevel.ActivityLevel =
3049                 cpu_to_be16((u16)pi->mclk_activity_target);
3050
3051         table->MemoryACPILevel.StutterEnable = false;
3052         table->MemoryACPILevel.StrobeEnable = false;
3053         table->MemoryACPILevel.EdcReadEnable = false;
3054         table->MemoryACPILevel.EdcWriteEnable = false;
3055         table->MemoryACPILevel.RttEnable = false;
3056
3057         return 0;
3058 }
3059
3060
3061 static int ci_enable_ulv(struct radeon_device *rdev, bool enable)
3062 {
3063         struct ci_power_info *pi = ci_get_pi(rdev);
3064         struct ci_ulv_parm *ulv = &pi->ulv;
3065
3066         if (ulv->supported) {
3067                 if (enable)
3068                         return (ci_send_msg_to_smc(rdev, PPSMC_MSG_EnableULV) == PPSMC_Result_OK) ?
3069                                 0 : -EINVAL;
3070                 else
3071                         return (ci_send_msg_to_smc(rdev, PPSMC_MSG_DisableULV) == PPSMC_Result_OK) ?
3072                                 0 : -EINVAL;
3073         }
3074
3075         return 0;
3076 }
3077
3078 static int ci_populate_ulv_level(struct radeon_device *rdev,
3079                                  SMU7_Discrete_Ulv *state)
3080 {
3081         struct ci_power_info *pi = ci_get_pi(rdev);
3082         u16 ulv_voltage = rdev->pm.dpm.backbias_response_time;
3083
3084         state->CcPwrDynRm = 0;
3085         state->CcPwrDynRm1 = 0;
3086
3087         if (ulv_voltage == 0) {
3088                 pi->ulv.supported = false;
3089                 return 0;
3090         }
3091
3092         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_BY_SVID2) {
3093                 if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3094                         state->VddcOffset = 0;
3095                 else
3096                         state->VddcOffset =
3097                                 rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage;
3098         } else {
3099                 if (ulv_voltage > rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v)
3100                         state->VddcOffsetVid = 0;
3101                 else
3102                         state->VddcOffsetVid = (u8)
3103                                 ((rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk.entries[0].v - ulv_voltage) *
3104                                  VOLTAGE_VID_OFFSET_SCALE2 / VOLTAGE_VID_OFFSET_SCALE1);
3105         }
3106         state->VddcPhase = pi->vddc_phase_shed_control ? 0 : 1;
3107
3108         state->CcPwrDynRm = cpu_to_be32(state->CcPwrDynRm);
3109         state->CcPwrDynRm1 = cpu_to_be32(state->CcPwrDynRm1);
3110         state->VddcOffset = cpu_to_be16(state->VddcOffset);
3111
3112         return 0;
3113 }
3114
3115 static int ci_calculate_sclk_params(struct radeon_device *rdev,
3116                                     u32 engine_clock,
3117                                     SMU7_Discrete_GraphicsLevel *sclk)
3118 {
3119         struct ci_power_info *pi = ci_get_pi(rdev);
3120         struct atom_clock_dividers dividers;
3121         u32 spll_func_cntl_3 = pi->clock_registers.cg_spll_func_cntl_3;
3122         u32 spll_func_cntl_4 = pi->clock_registers.cg_spll_func_cntl_4;
3123         u32 cg_spll_spread_spectrum = pi->clock_registers.cg_spll_spread_spectrum;
3124         u32 cg_spll_spread_spectrum_2 = pi->clock_registers.cg_spll_spread_spectrum_2;
3125         u32 reference_clock = rdev->clock.spll.reference_freq;
3126         u32 reference_divider;
3127         u32 fbdiv;
3128         int ret;
3129
3130         ret = radeon_atom_get_clock_dividers(rdev,
3131                                              COMPUTE_GPUCLK_INPUT_FLAG_SCLK,
3132                                              engine_clock, false, &dividers);
3133         if (ret)
3134                 return ret;
3135
3136         reference_divider = 1 + dividers.ref_div;
3137         fbdiv = dividers.fb_div & 0x3FFFFFF;
3138
3139         spll_func_cntl_3 &= ~SPLL_FB_DIV_MASK;
3140         spll_func_cntl_3 |= SPLL_FB_DIV(fbdiv);
3141         spll_func_cntl_3 |= SPLL_DITHEN;
3142
3143         if (pi->caps_sclk_ss_support) {
3144                 struct radeon_atom_ss ss;
3145                 u32 vco_freq = engine_clock * dividers.post_div;
3146
3147                 if (radeon_atombios_get_asic_ss_info(rdev, &ss,
3148                                                      ASIC_INTERNAL_ENGINE_SS, vco_freq)) {
3149                         u32 clk_s = reference_clock * 5 / (reference_divider * ss.rate);
3150                         u32 clk_v = 4 * ss.percentage * fbdiv / (clk_s * 10000);
3151
3152                         cg_spll_spread_spectrum &= ~CLK_S_MASK;
3153                         cg_spll_spread_spectrum |= CLK_S(clk_s);
3154                         cg_spll_spread_spectrum |= SSEN;
3155
3156                         cg_spll_spread_spectrum_2 &= ~CLK_V_MASK;
3157                         cg_spll_spread_spectrum_2 |= CLK_V(clk_v);
3158                 }
3159         }
3160
3161         sclk->SclkFrequency = engine_clock;
3162         sclk->CgSpllFuncCntl3 = spll_func_cntl_3;
3163         sclk->CgSpllFuncCntl4 = spll_func_cntl_4;
3164         sclk->SpllSpreadSpectrum = cg_spll_spread_spectrum;
3165         sclk->SpllSpreadSpectrum2  = cg_spll_spread_spectrum_2;
3166         sclk->SclkDid = (u8)dividers.post_divider;
3167
3168         return 0;
3169 }
3170
3171 static int ci_populate_single_graphic_level(struct radeon_device *rdev,
3172                                             u32 engine_clock,
3173                                             u16 sclk_activity_level_t,
3174                                             SMU7_Discrete_GraphicsLevel *graphic_level)
3175 {
3176         struct ci_power_info *pi = ci_get_pi(rdev);
3177         int ret;
3178
3179         ret = ci_calculate_sclk_params(rdev, engine_clock, graphic_level);
3180         if (ret)
3181                 return ret;
3182
3183         ret = ci_get_dependency_volt_by_clk(rdev,
3184                                             &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk,
3185                                             engine_clock, &graphic_level->MinVddc);
3186         if (ret)
3187                 return ret;
3188
3189         graphic_level->SclkFrequency = engine_clock;
3190
3191         graphic_level->Flags =  0;
3192         graphic_level->MinVddcPhases = 1;
3193
3194         if (pi->vddc_phase_shed_control)
3195                 ci_populate_phase_value_based_on_sclk(rdev,
3196                                                       &rdev->pm.dpm.dyn_state.phase_shedding_limits_table,
3197                                                       engine_clock,
3198                                                       &graphic_level->MinVddcPhases);
3199
3200         graphic_level->ActivityLevel = sclk_activity_level_t;
3201
3202         graphic_level->CcPwrDynRm = 0;
3203         graphic_level->CcPwrDynRm1 = 0;
3204         graphic_level->EnabledForThrottle = 1;
3205         graphic_level->UpH = 0;
3206         graphic_level->DownH = 0;
3207         graphic_level->VoltageDownH = 0;
3208         graphic_level->PowerThrottle = 0;
3209
3210         if (pi->caps_sclk_ds)
3211                 graphic_level->DeepSleepDivId = ci_get_sleep_divider_id_from_clock(rdev,
3212                                                                                    engine_clock,
3213                                                                                    CISLAND_MINIMUM_ENGINE_CLOCK);
3214
3215         graphic_level->DisplayWatermark = PPSMC_DISPLAY_WATERMARK_LOW;
3216
3217         graphic_level->Flags = cpu_to_be32(graphic_level->Flags);
3218         graphic_level->MinVddc = cpu_to_be32(graphic_level->MinVddc * VOLTAGE_SCALE);
3219         graphic_level->MinVddcPhases = cpu_to_be32(graphic_level->MinVddcPhases);
3220         graphic_level->SclkFrequency = cpu_to_be32(graphic_level->SclkFrequency);
3221         graphic_level->ActivityLevel = cpu_to_be16(graphic_level->ActivityLevel);
3222         graphic_level->CgSpllFuncCntl3 = cpu_to_be32(graphic_level->CgSpllFuncCntl3);
3223         graphic_level->CgSpllFuncCntl4 = cpu_to_be32(graphic_level->CgSpllFuncCntl4);
3224         graphic_level->SpllSpreadSpectrum = cpu_to_be32(graphic_level->SpllSpreadSpectrum);
3225         graphic_level->SpllSpreadSpectrum2 = cpu_to_be32(graphic_level->SpllSpreadSpectrum2);
3226         graphic_level->CcPwrDynRm = cpu_to_be32(graphic_level->CcPwrDynRm);
3227         graphic_level->CcPwrDynRm1 = cpu_to_be32(graphic_level->CcPwrDynRm1);
3228
3229         return 0;
3230 }
3231
3232 static int ci_populate_all_graphic_levels(struct radeon_device *rdev)
3233 {
3234         struct ci_power_info *pi = ci_get_pi(rdev);
3235         struct ci_dpm_table *dpm_table = &pi->dpm_table;
3236         u32 level_array_address = pi->dpm_table_start +
3237                 offsetof(SMU7_Discrete_DpmTable, GraphicsLevel);
3238         u32 level_array_size = sizeof(SMU7_Discrete_GraphicsLevel) *
3239                 SMU7_MAX_LEVELS_GRAPHICS;
3240         SMU7_Discrete_GraphicsLevel *levels = pi->smc_state_table.GraphicsLevel;
3241         u32 i, ret;
3242
3243         memset(levels, 0, level_array_size);
3244
3245         for (i = 0; i < dpm_table->sclk_table.count; i++) {
3246                 ret = ci_populate_single_graphic_level(rdev,
3247                                                        dpm_table->sclk_table.dpm_levels[i].value,
3248                                                        (u16)pi->activity_target[i],
3249                                                        &pi->smc_state_table.GraphicsLevel[i]);
3250                 if (ret)
3251                         return ret;
3252                 if (i > 1)
3253                         pi->smc_state_table.GraphicsLevel[i].DeepSleepDivId = 0;
3254                 if (i == (dpm_table->sclk_table.count - 1))
3255                         pi->smc_state_table.GraphicsLevel[i].DisplayWatermark =
3256                                 PPSMC_DISPLAY_WATERMARK_HIGH;
3257         }
3258         pi->smc_state_table.GraphicsLevel[0].EnabledForActivity = 1;
3259
3260         pi->smc_state_table.GraphicsDpmLevelCount = (u8)dpm_table->sclk_table.count;
3261         pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
3262                 ci_get_dpm_level_enable_mask_value(&dpm_table->sclk_table);
3263
3264         ret = ci_copy_bytes_to_smc(rdev, level_array_address,
3265                                    (u8 *)levels, level_array_size,
3266                                    pi->sram_end);
3267         if (ret)
3268                 return ret;
3269
3270         return 0;
3271 }
3272
3273 static int ci_populate_ulv_state(struct radeon_device *rdev,
3274                                  SMU7_Discrete_Ulv *ulv_level)
3275 {
3276         return ci_populate_ulv_level(rdev, ulv_level);
3277 }
3278
3279 static int ci_populate_all_memory_levels(struct radeon_device *rdev)
3280 {
3281         struct ci_power_info *pi = ci_get_pi(rdev);
3282         struct ci_dpm_table *dpm_table = &pi->dpm_table;
3283         u32 level_array_address = pi->dpm_table_start +
3284                 offsetof(SMU7_Discrete_DpmTable, MemoryLevel);
3285         u32 level_array_size = sizeof(SMU7_Discrete_MemoryLevel) *
3286                 SMU7_MAX_LEVELS_MEMORY;
3287         SMU7_Discrete_MemoryLevel *levels = pi->smc_state_table.MemoryLevel;
3288         u32 i, ret;
3289
3290         memset(levels, 0, level_array_size);
3291
3292         for (i = 0; i < dpm_table->mclk_table.count; i++) {
3293                 if (dpm_table->mclk_table.dpm_levels[i].value == 0)
3294                         return -EINVAL;
3295                 ret = ci_populate_single_memory_level(rdev,
3296                                                       dpm_table->mclk_table.dpm_levels[i].value,
3297                                                       &pi->smc_state_table.MemoryLevel[i]);
3298                 if (ret)
3299                         return ret;
3300         }
3301
3302         pi->smc_state_table.MemoryLevel[0].EnabledForActivity = 1;
3303
3304         if ((dpm_table->mclk_table.count >= 2) &&
3305             ((rdev->pdev->device == 0x67B0) || (rdev->pdev->device == 0x67B1))) {
3306                 pi->smc_state_table.MemoryLevel[1].MinVddc =
3307                         pi->smc_state_table.MemoryLevel[0].MinVddc;
3308                 pi->smc_state_table.MemoryLevel[1].MinVddcPhases =
3309                         pi->smc_state_table.MemoryLevel[0].MinVddcPhases;
3310         }
3311
3312         pi->smc_state_table.MemoryLevel[0].ActivityLevel = cpu_to_be16(0x1F);
3313
3314         pi->smc_state_table.MemoryDpmLevelCount = (u8)dpm_table->mclk_table.count;
3315         pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
3316                 ci_get_dpm_level_enable_mask_value(&dpm_table->mclk_table);
3317
3318         pi->smc_state_table.MemoryLevel[dpm_table->mclk_table.count - 1].DisplayWatermark =
3319                 PPSMC_DISPLAY_WATERMARK_HIGH;
3320
3321         ret = ci_copy_bytes_to_smc(rdev, level_array_address,
3322                                    (u8 *)levels, level_array_size,
3323                                    pi->sram_end);
3324         if (ret)
3325                 return ret;
3326
3327         return 0;
3328 }
3329
3330 static void ci_reset_single_dpm_table(struct radeon_device *rdev,
3331                                       struct ci_single_dpm_table *dpm_table,
3332                                       u32 count)
3333 {
3334         u32 i;
3335
3336         dpm_table->count = count;
3337         for (i = 0; i < MAX_REGULAR_DPM_NUMBER; i++)
3338                 dpm_table->dpm_levels[i].enabled = false;
3339 }
3340
3341 static void ci_setup_pcie_table_entry(struct ci_single_dpm_table *dpm_table,
3342                                       u32 index, u32 pcie_gen, u32 pcie_lanes)
3343 {
3344         dpm_table->dpm_levels[index].value = pcie_gen;
3345         dpm_table->dpm_levels[index].param1 = pcie_lanes;
3346         dpm_table->dpm_levels[index].enabled = true;
3347 }
3348
3349 static int ci_setup_default_pcie_tables(struct radeon_device *rdev)
3350 {
3351         struct ci_power_info *pi = ci_get_pi(rdev);
3352
3353         if (!pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels)
3354                 return -EINVAL;
3355
3356         if (pi->use_pcie_performance_levels && !pi->use_pcie_powersaving_levels) {
3357                 pi->pcie_gen_powersaving = pi->pcie_gen_performance;
3358                 pi->pcie_lane_powersaving = pi->pcie_lane_performance;
3359         } else if (!pi->use_pcie_performance_levels && pi->use_pcie_powersaving_levels) {
3360                 pi->pcie_gen_performance = pi->pcie_gen_powersaving;
3361                 pi->pcie_lane_performance = pi->pcie_lane_powersaving;
3362         }
3363
3364         ci_reset_single_dpm_table(rdev,
3365                                   &pi->dpm_table.pcie_speed_table,
3366                                   SMU7_MAX_LEVELS_LINK);
3367
3368         if (rdev->family == CHIP_BONAIRE)
3369                 ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3370                                           pi->pcie_gen_powersaving.min,
3371                                           pi->pcie_lane_powersaving.max);
3372         else
3373                 ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 0,
3374                                           pi->pcie_gen_powersaving.min,
3375                                           pi->pcie_lane_powersaving.min);
3376         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 1,
3377                                   pi->pcie_gen_performance.min,
3378                                   pi->pcie_lane_performance.min);
3379         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 2,
3380                                   pi->pcie_gen_powersaving.min,
3381                                   pi->pcie_lane_powersaving.max);
3382         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 3,
3383                                   pi->pcie_gen_performance.min,
3384                                   pi->pcie_lane_performance.max);
3385         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 4,
3386                                   pi->pcie_gen_powersaving.max,
3387                                   pi->pcie_lane_powersaving.max);
3388         ci_setup_pcie_table_entry(&pi->dpm_table.pcie_speed_table, 5,
3389                                   pi->pcie_gen_performance.max,
3390                                   pi->pcie_lane_performance.max);
3391
3392         pi->dpm_table.pcie_speed_table.count = 6;
3393
3394         return 0;
3395 }
3396
3397 static int ci_setup_default_dpm_tables(struct radeon_device *rdev)
3398 {
3399         struct ci_power_info *pi = ci_get_pi(rdev);
3400         struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table =
3401                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3402         struct radeon_clock_voltage_dependency_table *allowed_mclk_table =
3403                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
3404         struct radeon_cac_leakage_table *std_voltage_table =
3405                 &rdev->pm.dpm.dyn_state.cac_leakage_table;
3406         u32 i;
3407
3408         if (allowed_sclk_vddc_table == NULL)
3409                 return -EINVAL;
3410         if (allowed_sclk_vddc_table->count < 1)
3411                 return -EINVAL;
3412         if (allowed_mclk_table == NULL)
3413                 return -EINVAL;
3414         if (allowed_mclk_table->count < 1)
3415                 return -EINVAL;
3416
3417         memset(&pi->dpm_table, 0, sizeof(struct ci_dpm_table));
3418
3419         ci_reset_single_dpm_table(rdev,
3420                                   &pi->dpm_table.sclk_table,
3421                                   SMU7_MAX_LEVELS_GRAPHICS);
3422         ci_reset_single_dpm_table(rdev,
3423                                   &pi->dpm_table.mclk_table,
3424                                   SMU7_MAX_LEVELS_MEMORY);
3425         ci_reset_single_dpm_table(rdev,
3426                                   &pi->dpm_table.vddc_table,
3427                                   SMU7_MAX_LEVELS_VDDC);
3428         ci_reset_single_dpm_table(rdev,
3429                                   &pi->dpm_table.vddci_table,
3430                                   SMU7_MAX_LEVELS_VDDCI);
3431         ci_reset_single_dpm_table(rdev,
3432                                   &pi->dpm_table.mvdd_table,
3433                                   SMU7_MAX_LEVELS_MVDD);
3434
3435         pi->dpm_table.sclk_table.count = 0;
3436         for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3437                 if ((i == 0) ||
3438                     (pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count-1].value !=
3439                      allowed_sclk_vddc_table->entries[i].clk)) {
3440                         pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].value =
3441                                 allowed_sclk_vddc_table->entries[i].clk;
3442                         pi->dpm_table.sclk_table.dpm_levels[pi->dpm_table.sclk_table.count].enabled =
3443                                 (i == 0) ? true : false;
3444                         pi->dpm_table.sclk_table.count++;
3445                 }
3446         }
3447
3448         pi->dpm_table.mclk_table.count = 0;
3449         for (i = 0; i < allowed_mclk_table->count; i++) {
3450                 if ((i == 0) ||
3451                     (pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count-1].value !=
3452                      allowed_mclk_table->entries[i].clk)) {
3453                         pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].value =
3454                                 allowed_mclk_table->entries[i].clk;
3455                         pi->dpm_table.mclk_table.dpm_levels[pi->dpm_table.mclk_table.count].enabled =
3456                                 (i == 0) ? true : false;
3457                         pi->dpm_table.mclk_table.count++;
3458                 }
3459         }
3460
3461         for (i = 0; i < allowed_sclk_vddc_table->count; i++) {
3462                 pi->dpm_table.vddc_table.dpm_levels[i].value =
3463                         allowed_sclk_vddc_table->entries[i].v;
3464                 pi->dpm_table.vddc_table.dpm_levels[i].param1 =
3465                         std_voltage_table->entries[i].leakage;
3466                 pi->dpm_table.vddc_table.dpm_levels[i].enabled = true;
3467         }
3468         pi->dpm_table.vddc_table.count = allowed_sclk_vddc_table->count;
3469
3470         allowed_mclk_table = &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
3471         if (allowed_mclk_table) {
3472                 for (i = 0; i < allowed_mclk_table->count; i++) {
3473                         pi->dpm_table.vddci_table.dpm_levels[i].value =
3474                                 allowed_mclk_table->entries[i].v;
3475                         pi->dpm_table.vddci_table.dpm_levels[i].enabled = true;
3476                 }
3477                 pi->dpm_table.vddci_table.count = allowed_mclk_table->count;
3478         }
3479
3480         allowed_mclk_table = &rdev->pm.dpm.dyn_state.mvdd_dependency_on_mclk;
3481         if (allowed_mclk_table) {
3482                 for (i = 0; i < allowed_mclk_table->count; i++) {
3483                         pi->dpm_table.mvdd_table.dpm_levels[i].value =
3484                                 allowed_mclk_table->entries[i].v;
3485                         pi->dpm_table.mvdd_table.dpm_levels[i].enabled = true;
3486                 }
3487                 pi->dpm_table.mvdd_table.count = allowed_mclk_table->count;
3488         }
3489
3490         ci_setup_default_pcie_tables(rdev);
3491
3492         return 0;
3493 }
3494
3495 static int ci_find_boot_level(struct ci_single_dpm_table *table,
3496                               u32 value, u32 *boot_level)
3497 {
3498         u32 i;
3499         int ret = -EINVAL;
3500
3501         for (i = 0; i < table->count; i++) {
3502                 if (value == table->dpm_levels[i].value) {
3503                         *boot_level = i;
3504                         ret = 0;
3505                 }
3506         }
3507
3508         return ret;
3509 }
3510
3511 static int ci_init_smc_table(struct radeon_device *rdev)
3512 {
3513         struct ci_power_info *pi = ci_get_pi(rdev);
3514         struct ci_ulv_parm *ulv = &pi->ulv;
3515         struct radeon_ps *radeon_boot_state = rdev->pm.dpm.boot_ps;
3516         SMU7_Discrete_DpmTable *table = &pi->smc_state_table;
3517         int ret;
3518
3519         ret = ci_setup_default_dpm_tables(rdev);
3520         if (ret)
3521                 return ret;
3522
3523         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE)
3524                 ci_populate_smc_voltage_tables(rdev, table);
3525
3526         ci_init_fps_limits(rdev);
3527
3528         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_HARDWAREDC)
3529                 table->SystemFlags |= PPSMC_SYSTEMFLAG_GPIO_DC;
3530
3531         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_STEPVDDC)
3532                 table->SystemFlags |= PPSMC_SYSTEMFLAG_STEPVDDC;
3533
3534         if (pi->mem_gddr5)
3535                 table->SystemFlags |= PPSMC_SYSTEMFLAG_GDDR5;
3536
3537         if (ulv->supported) {
3538                 ret = ci_populate_ulv_state(rdev, &pi->smc_state_table.Ulv);
3539                 if (ret)
3540                         return ret;
3541                 WREG32_SMC(CG_ULV_PARAMETER, ulv->cg_ulv_parameter);
3542         }
3543
3544         ret = ci_populate_all_graphic_levels(rdev);
3545         if (ret)
3546                 return ret;
3547
3548         ret = ci_populate_all_memory_levels(rdev);
3549         if (ret)
3550                 return ret;
3551
3552         ci_populate_smc_link_level(rdev, table);
3553
3554         ret = ci_populate_smc_acpi_level(rdev, table);
3555         if (ret)
3556                 return ret;
3557
3558         ret = ci_populate_smc_vce_level(rdev, table);
3559         if (ret)
3560                 return ret;
3561
3562         ret = ci_populate_smc_acp_level(rdev, table);
3563         if (ret)
3564                 return ret;
3565
3566         ret = ci_populate_smc_samu_level(rdev, table);
3567         if (ret)
3568                 return ret;
3569
3570         ret = ci_do_program_memory_timing_parameters(rdev);
3571         if (ret)
3572                 return ret;
3573
3574         ret = ci_populate_smc_uvd_level(rdev, table);
3575         if (ret)
3576                 return ret;
3577
3578         table->UvdBootLevel  = 0;
3579         table->VceBootLevel  = 0;
3580         table->AcpBootLevel  = 0;
3581         table->SamuBootLevel  = 0;
3582         table->GraphicsBootLevel  = 0;
3583         table->MemoryBootLevel  = 0;
3584
3585         ret = ci_find_boot_level(&pi->dpm_table.sclk_table,
3586                                  pi->vbios_boot_state.sclk_bootup_value,
3587                                  (u32 *)&pi->smc_state_table.GraphicsBootLevel);
3588
3589         ret = ci_find_boot_level(&pi->dpm_table.mclk_table,
3590                                  pi->vbios_boot_state.mclk_bootup_value,
3591                                  (u32 *)&pi->smc_state_table.MemoryBootLevel);
3592
3593         table->BootVddc = pi->vbios_boot_state.vddc_bootup_value;
3594         table->BootVddci = pi->vbios_boot_state.vddci_bootup_value;
3595         table->BootMVdd = pi->vbios_boot_state.mvdd_bootup_value;
3596
3597         ci_populate_smc_initial_state(rdev, radeon_boot_state);
3598
3599         ret = ci_populate_bapm_parameters_in_dpm_table(rdev);
3600         if (ret)
3601                 return ret;
3602
3603         table->UVDInterval = 1;
3604         table->VCEInterval = 1;
3605         table->ACPInterval = 1;
3606         table->SAMUInterval = 1;
3607         table->GraphicsVoltageChangeEnable = 1;
3608         table->GraphicsThermThrottleEnable = 1;
3609         table->GraphicsInterval = 1;
3610         table->VoltageInterval = 1;
3611         table->ThermalInterval = 1;
3612         table->TemperatureLimitHigh = (u16)((pi->thermal_temp_setting.temperature_high *
3613                                              CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3614         table->TemperatureLimitLow = (u16)((pi->thermal_temp_setting.temperature_low *
3615                                             CISLANDS_Q88_FORMAT_CONVERSION_UNIT) / 1000);
3616         table->MemoryVoltageChangeEnable = 1;
3617         table->MemoryInterval = 1;
3618         table->VoltageResponseTime = 0;
3619         table->VddcVddciDelta = 4000;
3620         table->PhaseResponseTime = 0;
3621         table->MemoryThermThrottleEnable = 1;
3622         table->PCIeBootLinkLevel = pi->dpm_table.pcie_speed_table.count - 1;
3623         table->PCIeGenInterval = 1;
3624         if (pi->voltage_control == CISLANDS_VOLTAGE_CONTROL_BY_SVID2)
3625                 table->SVI2Enable  = 1;
3626         else
3627                 table->SVI2Enable  = 0;
3628
3629         table->ThermGpio = 17;
3630         table->SclkStepSize = 0x4000;
3631
3632         table->SystemFlags = cpu_to_be32(table->SystemFlags);
3633         table->SmioMaskVddcVid = cpu_to_be32(table->SmioMaskVddcVid);
3634         table->SmioMaskVddcPhase = cpu_to_be32(table->SmioMaskVddcPhase);
3635         table->SmioMaskVddciVid = cpu_to_be32(table->SmioMaskVddciVid);
3636         table->SmioMaskMvddVid = cpu_to_be32(table->SmioMaskMvddVid);
3637         table->SclkStepSize = cpu_to_be32(table->SclkStepSize);
3638         table->TemperatureLimitHigh = cpu_to_be16(table->TemperatureLimitHigh);
3639         table->TemperatureLimitLow = cpu_to_be16(table->TemperatureLimitLow);
3640         table->VddcVddciDelta = cpu_to_be16(table->VddcVddciDelta);
3641         table->VoltageResponseTime = cpu_to_be16(table->VoltageResponseTime);
3642         table->PhaseResponseTime = cpu_to_be16(table->PhaseResponseTime);
3643         table->BootVddc = cpu_to_be16(table->BootVddc * VOLTAGE_SCALE);
3644         table->BootVddci = cpu_to_be16(table->BootVddci * VOLTAGE_SCALE);
3645         table->BootMVdd = cpu_to_be16(table->BootMVdd * VOLTAGE_SCALE);
3646
3647         ret = ci_copy_bytes_to_smc(rdev,
3648                                    pi->dpm_table_start +
3649                                    offsetof(SMU7_Discrete_DpmTable, SystemFlags),
3650                                    (u8 *)&table->SystemFlags,
3651                                    sizeof(SMU7_Discrete_DpmTable) - 3 * sizeof(SMU7_PIDController),
3652                                    pi->sram_end);
3653         if (ret)
3654                 return ret;
3655
3656         return 0;
3657 }
3658
3659 static void ci_trim_single_dpm_states(struct radeon_device *rdev,
3660                                       struct ci_single_dpm_table *dpm_table,
3661                                       u32 low_limit, u32 high_limit)
3662 {
3663         u32 i;
3664
3665         for (i = 0; i < dpm_table->count; i++) {
3666                 if ((dpm_table->dpm_levels[i].value < low_limit) ||
3667                     (dpm_table->dpm_levels[i].value > high_limit))
3668                         dpm_table->dpm_levels[i].enabled = false;
3669                 else
3670                         dpm_table->dpm_levels[i].enabled = true;
3671         }
3672 }
3673
3674 static void ci_trim_pcie_dpm_states(struct radeon_device *rdev,
3675                                     u32 speed_low, u32 lanes_low,
3676                                     u32 speed_high, u32 lanes_high)
3677 {
3678         struct ci_power_info *pi = ci_get_pi(rdev);
3679         struct ci_single_dpm_table *pcie_table = &pi->dpm_table.pcie_speed_table;
3680         u32 i, j;
3681
3682         for (i = 0; i < pcie_table->count; i++) {
3683                 if ((pcie_table->dpm_levels[i].value < speed_low) ||
3684                     (pcie_table->dpm_levels[i].param1 < lanes_low) ||
3685                     (pcie_table->dpm_levels[i].value > speed_high) ||
3686                     (pcie_table->dpm_levels[i].param1 > lanes_high))
3687                         pcie_table->dpm_levels[i].enabled = false;
3688                 else
3689                         pcie_table->dpm_levels[i].enabled = true;
3690         }
3691
3692         for (i = 0; i < pcie_table->count; i++) {
3693                 if (pcie_table->dpm_levels[i].enabled) {
3694                         for (j = i + 1; j < pcie_table->count; j++) {
3695                                 if (pcie_table->dpm_levels[j].enabled) {
3696                                         if ((pcie_table->dpm_levels[i].value == pcie_table->dpm_levels[j].value) &&
3697                                             (pcie_table->dpm_levels[i].param1 == pcie_table->dpm_levels[j].param1))
3698                                                 pcie_table->dpm_levels[j].enabled = false;
3699                                 }
3700                         }
3701                 }
3702         }
3703 }
3704
3705 static int ci_trim_dpm_states(struct radeon_device *rdev,
3706                               struct radeon_ps *radeon_state)
3707 {
3708         struct ci_ps *state = ci_get_ps(radeon_state);
3709         struct ci_power_info *pi = ci_get_pi(rdev);
3710         u32 high_limit_count;
3711
3712         if (state->performance_level_count < 1)
3713                 return -EINVAL;
3714
3715         if (state->performance_level_count == 1)
3716                 high_limit_count = 0;
3717         else
3718                 high_limit_count = 1;
3719
3720         ci_trim_single_dpm_states(rdev,
3721                                   &pi->dpm_table.sclk_table,
3722                                   state->performance_levels[0].sclk,
3723                                   state->performance_levels[high_limit_count].sclk);
3724
3725         ci_trim_single_dpm_states(rdev,
3726                                   &pi->dpm_table.mclk_table,
3727                                   state->performance_levels[0].mclk,
3728                                   state->performance_levels[high_limit_count].mclk);
3729
3730         ci_trim_pcie_dpm_states(rdev,
3731                                 state->performance_levels[0].pcie_gen,
3732                                 state->performance_levels[0].pcie_lane,
3733                                 state->performance_levels[high_limit_count].pcie_gen,
3734                                 state->performance_levels[high_limit_count].pcie_lane);
3735
3736         return 0;
3737 }
3738
3739 static int ci_apply_disp_minimum_voltage_request(struct radeon_device *rdev)
3740 {
3741         struct radeon_clock_voltage_dependency_table *disp_voltage_table =
3742                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk;
3743         struct radeon_clock_voltage_dependency_table *vddc_table =
3744                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
3745         u32 requested_voltage = 0;
3746         u32 i;
3747
3748         if (disp_voltage_table == NULL)
3749                 return -EINVAL;
3750         if (!disp_voltage_table->count)
3751                 return -EINVAL;
3752
3753         for (i = 0; i < disp_voltage_table->count; i++) {
3754                 if (rdev->clock.current_dispclk == disp_voltage_table->entries[i].clk)
3755                         requested_voltage = disp_voltage_table->entries[i].v;
3756         }
3757
3758         for (i = 0; i < vddc_table->count; i++) {
3759                 if (requested_voltage <= vddc_table->entries[i].v) {
3760                         requested_voltage = vddc_table->entries[i].v;
3761                         return (ci_send_msg_to_smc_with_parameter(rdev,
3762                                                                   PPSMC_MSG_VddC_Request,
3763                                                                   requested_voltage * VOLTAGE_SCALE) == PPSMC_Result_OK) ?
3764                                 0 : -EINVAL;
3765                 }
3766         }
3767
3768         return -EINVAL;
3769 }
3770
3771 static int ci_upload_dpm_level_enable_mask(struct radeon_device *rdev)
3772 {
3773         struct ci_power_info *pi = ci_get_pi(rdev);
3774         PPSMC_Result result;
3775
3776         ci_apply_disp_minimum_voltage_request(rdev);
3777
3778         if (!pi->sclk_dpm_key_disabled) {
3779                 if (pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
3780                         result = ci_send_msg_to_smc_with_parameter(rdev,
3781                                                                    PPSMC_MSG_SCLKDPM_SetEnabledMask,
3782                                                                    pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
3783                         if (result != PPSMC_Result_OK)
3784                                 return -EINVAL;
3785                 }
3786         }
3787
3788         if (!pi->mclk_dpm_key_disabled) {
3789                 if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
3790                         result = ci_send_msg_to_smc_with_parameter(rdev,
3791                                                                    PPSMC_MSG_MCLKDPM_SetEnabledMask,
3792                                                                    pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3793                         if (result != PPSMC_Result_OK)
3794                                 return -EINVAL;
3795                 }
3796         }
3797 #if 0
3798         if (!pi->pcie_dpm_key_disabled) {
3799                 if (pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
3800                         result = ci_send_msg_to_smc_with_parameter(rdev,
3801                                                                    PPSMC_MSG_PCIeDPM_SetEnabledMask,
3802                                                                    pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
3803                         if (result != PPSMC_Result_OK)
3804                                 return -EINVAL;
3805                 }
3806         }
3807 #endif
3808         return 0;
3809 }
3810
3811 static void ci_find_dpm_states_clocks_in_dpm_table(struct radeon_device *rdev,
3812                                                    struct radeon_ps *radeon_state)
3813 {
3814         struct ci_power_info *pi = ci_get_pi(rdev);
3815         struct ci_ps *state = ci_get_ps(radeon_state);
3816         struct ci_single_dpm_table *sclk_table = &pi->dpm_table.sclk_table;
3817         u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
3818         struct ci_single_dpm_table *mclk_table = &pi->dpm_table.mclk_table;
3819         u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
3820         u32 i;
3821
3822         pi->need_update_smu7_dpm_table = 0;
3823
3824         for (i = 0; i < sclk_table->count; i++) {
3825                 if (sclk == sclk_table->dpm_levels[i].value)
3826                         break;
3827         }
3828
3829         if (i >= sclk_table->count) {
3830                 pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_SCLK;
3831         } else {
3832                 /* XXX The current code always reprogrammed the sclk levels,
3833                  * but we don't currently handle disp sclk requirements
3834                  * so just skip it.
3835                  */
3836                 if (CISLAND_MINIMUM_ENGINE_CLOCK != CISLAND_MINIMUM_ENGINE_CLOCK)
3837                         pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_SCLK;
3838         }
3839
3840         for (i = 0; i < mclk_table->count; i++) {
3841                 if (mclk == mclk_table->dpm_levels[i].value)
3842                         break;
3843         }
3844
3845         if (i >= mclk_table->count)
3846                 pi->need_update_smu7_dpm_table |= DPMTABLE_OD_UPDATE_MCLK;
3847
3848         if (rdev->pm.dpm.current_active_crtc_count !=
3849             rdev->pm.dpm.new_active_crtc_count)
3850                 pi->need_update_smu7_dpm_table |= DPMTABLE_UPDATE_MCLK;
3851 }
3852
3853 static int ci_populate_and_upload_sclk_mclk_dpm_levels(struct radeon_device *rdev,
3854                                                        struct radeon_ps *radeon_state)
3855 {
3856         struct ci_power_info *pi = ci_get_pi(rdev);
3857         struct ci_ps *state = ci_get_ps(radeon_state);
3858         u32 sclk = state->performance_levels[state->performance_level_count-1].sclk;
3859         u32 mclk = state->performance_levels[state->performance_level_count-1].mclk;
3860         struct ci_dpm_table *dpm_table = &pi->dpm_table;
3861         int ret;
3862
3863         if (!pi->need_update_smu7_dpm_table)
3864                 return 0;
3865
3866         if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_SCLK)
3867                 dpm_table->sclk_table.dpm_levels[dpm_table->sclk_table.count-1].value = sclk;
3868
3869         if (pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK)
3870                 dpm_table->mclk_table.dpm_levels[dpm_table->mclk_table.count-1].value = mclk;
3871
3872         if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_SCLK | DPMTABLE_UPDATE_SCLK)) {
3873                 ret = ci_populate_all_graphic_levels(rdev);
3874                 if (ret)
3875                         return ret;
3876         }
3877
3878         if (pi->need_update_smu7_dpm_table & (DPMTABLE_OD_UPDATE_MCLK | DPMTABLE_UPDATE_MCLK)) {
3879                 ret = ci_populate_all_memory_levels(rdev);
3880                 if (ret)
3881                         return ret;
3882         }
3883
3884         return 0;
3885 }
3886
3887 static int ci_enable_uvd_dpm(struct radeon_device *rdev, bool enable)
3888 {
3889         struct ci_power_info *pi = ci_get_pi(rdev);
3890         const struct radeon_clock_and_voltage_limits *max_limits;
3891         int i;
3892
3893         if (rdev->pm.dpm.ac_power)
3894                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3895         else
3896                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3897
3898         if (enable) {
3899                 pi->dpm_level_enable_mask.uvd_dpm_enable_mask = 0;
3900
3901                 for (i = rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3902                         if (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3903                                 pi->dpm_level_enable_mask.uvd_dpm_enable_mask |= 1 << i;
3904
3905                                 if (!pi->caps_uvd_dpm)
3906                                         break;
3907                         }
3908                 }
3909
3910                 ci_send_msg_to_smc_with_parameter(rdev,
3911                                                   PPSMC_MSG_UVDDPM_SetEnabledMask,
3912                                                   pi->dpm_level_enable_mask.uvd_dpm_enable_mask);
3913
3914                 if (pi->last_mclk_dpm_enable_mask & 0x1) {
3915                         pi->uvd_enabled = true;
3916                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
3917                         ci_send_msg_to_smc_with_parameter(rdev,
3918                                                           PPSMC_MSG_MCLKDPM_SetEnabledMask,
3919                                                           pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3920                 }
3921         } else {
3922                 if (pi->last_mclk_dpm_enable_mask & 0x1) {
3923                         pi->uvd_enabled = false;
3924                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask |= 1;
3925                         ci_send_msg_to_smc_with_parameter(rdev,
3926                                                           PPSMC_MSG_MCLKDPM_SetEnabledMask,
3927                                                           pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
3928                 }
3929         }
3930
3931         return (ci_send_msg_to_smc(rdev, enable ?
3932                                    PPSMC_MSG_UVDDPM_Enable : PPSMC_MSG_UVDDPM_Disable) == PPSMC_Result_OK) ?
3933                 0 : -EINVAL;
3934 }
3935
3936 static int ci_enable_vce_dpm(struct radeon_device *rdev, bool enable)
3937 {
3938         struct ci_power_info *pi = ci_get_pi(rdev);
3939         const struct radeon_clock_and_voltage_limits *max_limits;
3940         int i;
3941
3942         if (rdev->pm.dpm.ac_power)
3943                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3944         else
3945                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3946
3947         if (enable) {
3948                 pi->dpm_level_enable_mask.vce_dpm_enable_mask = 0;
3949                 for (i = rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3950                         if (rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3951                                 pi->dpm_level_enable_mask.vce_dpm_enable_mask |= 1 << i;
3952
3953                                 if (!pi->caps_vce_dpm)
3954                                         break;
3955                         }
3956                 }
3957
3958                 ci_send_msg_to_smc_with_parameter(rdev,
3959                                                   PPSMC_MSG_VCEDPM_SetEnabledMask,
3960                                                   pi->dpm_level_enable_mask.vce_dpm_enable_mask);
3961         }
3962
3963         return (ci_send_msg_to_smc(rdev, enable ?
3964                                    PPSMC_MSG_VCEDPM_Enable : PPSMC_MSG_VCEDPM_Disable) == PPSMC_Result_OK) ?
3965                 0 : -EINVAL;
3966 }
3967
3968 #if 0
3969 static int ci_enable_samu_dpm(struct radeon_device *rdev, bool enable)
3970 {
3971         struct ci_power_info *pi = ci_get_pi(rdev);
3972         const struct radeon_clock_and_voltage_limits *max_limits;
3973         int i;
3974
3975         if (rdev->pm.dpm.ac_power)
3976                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
3977         else
3978                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
3979
3980         if (enable) {
3981                 pi->dpm_level_enable_mask.samu_dpm_enable_mask = 0;
3982                 for (i = rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
3983                         if (rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
3984                                 pi->dpm_level_enable_mask.samu_dpm_enable_mask |= 1 << i;
3985
3986                                 if (!pi->caps_samu_dpm)
3987                                         break;
3988                         }
3989                 }
3990
3991                 ci_send_msg_to_smc_with_parameter(rdev,
3992                                                   PPSMC_MSG_SAMUDPM_SetEnabledMask,
3993                                                   pi->dpm_level_enable_mask.samu_dpm_enable_mask);
3994         }
3995         return (ci_send_msg_to_smc(rdev, enable ?
3996                                    PPSMC_MSG_SAMUDPM_Enable : PPSMC_MSG_SAMUDPM_Disable) == PPSMC_Result_OK) ?
3997                 0 : -EINVAL;
3998 }
3999
4000 static int ci_enable_acp_dpm(struct radeon_device *rdev, bool enable)
4001 {
4002         struct ci_power_info *pi = ci_get_pi(rdev);
4003         const struct radeon_clock_and_voltage_limits *max_limits;
4004         int i;
4005
4006         if (rdev->pm.dpm.ac_power)
4007                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
4008         else
4009                 max_limits = &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc;
4010
4011         if (enable) {
4012                 pi->dpm_level_enable_mask.acp_dpm_enable_mask = 0;
4013                 for (i = rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.count - 1; i >= 0; i--) {
4014                         if (rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table.entries[i].v <= max_limits->vddc) {
4015                                 pi->dpm_level_enable_mask.acp_dpm_enable_mask |= 1 << i;
4016
4017                                 if (!pi->caps_acp_dpm)
4018                                         break;
4019                         }
4020                 }
4021
4022                 ci_send_msg_to_smc_with_parameter(rdev,
4023                                                   PPSMC_MSG_ACPDPM_SetEnabledMask,
4024                                                   pi->dpm_level_enable_mask.acp_dpm_enable_mask);
4025         }
4026
4027         return (ci_send_msg_to_smc(rdev, enable ?
4028                                    PPSMC_MSG_ACPDPM_Enable : PPSMC_MSG_ACPDPM_Disable) == PPSMC_Result_OK) ?
4029                 0 : -EINVAL;
4030 }
4031 #endif
4032
4033 static int ci_update_uvd_dpm(struct radeon_device *rdev, bool gate)
4034 {
4035         struct ci_power_info *pi = ci_get_pi(rdev);
4036         u32 tmp;
4037
4038         if (!gate) {
4039                 if (pi->caps_uvd_dpm ||
4040                     (rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count <= 0))
4041                         pi->smc_state_table.UvdBootLevel = 0;
4042                 else
4043                         pi->smc_state_table.UvdBootLevel =
4044                                 rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table.count - 1;
4045
4046                 tmp = RREG32_SMC(DPM_TABLE_475);
4047                 tmp &= ~UvdBootLevel_MASK;
4048                 tmp |= UvdBootLevel(pi->smc_state_table.UvdBootLevel);
4049                 WREG32_SMC(DPM_TABLE_475, tmp);
4050         }
4051
4052         return ci_enable_uvd_dpm(rdev, !gate);
4053 }
4054
4055 static u8 ci_get_vce_boot_level(struct radeon_device *rdev)
4056 {
4057         u8 i;
4058         u32 min_evclk = 30000; /* ??? */
4059         struct radeon_vce_clock_voltage_dependency_table *table =
4060                 &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table;
4061
4062         for (i = 0; i < table->count; i++) {
4063                 if (table->entries[i].evclk >= min_evclk)
4064                         return i;
4065         }
4066
4067         return table->count - 1;
4068 }
4069
4070 static int ci_update_vce_dpm(struct radeon_device *rdev,
4071                              struct radeon_ps *radeon_new_state,
4072                              struct radeon_ps *radeon_current_state)
4073 {
4074         struct ci_power_info *pi = ci_get_pi(rdev);
4075         int ret = 0;
4076         u32 tmp;
4077
4078         if (radeon_current_state->evclk != radeon_new_state->evclk) {
4079                 if (radeon_new_state->evclk) {
4080                         /* turn the clocks on when encoding */
4081                         cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, false);
4082
4083                         pi->smc_state_table.VceBootLevel = ci_get_vce_boot_level(rdev);
4084                         tmp = RREG32_SMC(DPM_TABLE_475);
4085                         tmp &= ~VceBootLevel_MASK;
4086                         tmp |= VceBootLevel(pi->smc_state_table.VceBootLevel);
4087                         WREG32_SMC(DPM_TABLE_475, tmp);
4088
4089                         ret = ci_enable_vce_dpm(rdev, true);
4090                 } else {
4091                         /* turn the clocks off when not encoding */
4092                         cik_update_cg(rdev, RADEON_CG_BLOCK_VCE, true);
4093
4094                         ret = ci_enable_vce_dpm(rdev, false);
4095                 }
4096         }
4097         return ret;
4098 }
4099
4100 #if 0
4101 static int ci_update_samu_dpm(struct radeon_device *rdev, bool gate)
4102 {
4103         return ci_enable_samu_dpm(rdev, gate);
4104 }
4105
4106 static int ci_update_acp_dpm(struct radeon_device *rdev, bool gate)
4107 {
4108         struct ci_power_info *pi = ci_get_pi(rdev);
4109         u32 tmp;
4110
4111         if (!gate) {
4112                 pi->smc_state_table.AcpBootLevel = 0;
4113
4114                 tmp = RREG32_SMC(DPM_TABLE_475);
4115                 tmp &= ~AcpBootLevel_MASK;
4116                 tmp |= AcpBootLevel(pi->smc_state_table.AcpBootLevel);
4117                 WREG32_SMC(DPM_TABLE_475, tmp);
4118         }
4119
4120         return ci_enable_acp_dpm(rdev, !gate);
4121 }
4122 #endif
4123
4124 static int ci_generate_dpm_level_enable_mask(struct radeon_device *rdev,
4125                                              struct radeon_ps *radeon_state)
4126 {
4127         struct ci_power_info *pi = ci_get_pi(rdev);
4128         int ret;
4129
4130         ret = ci_trim_dpm_states(rdev, radeon_state);
4131         if (ret)
4132                 return ret;
4133
4134         pi->dpm_level_enable_mask.sclk_dpm_enable_mask =
4135                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.sclk_table);
4136         pi->dpm_level_enable_mask.mclk_dpm_enable_mask =
4137                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.mclk_table);
4138         pi->last_mclk_dpm_enable_mask =
4139                 pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4140         if (pi->uvd_enabled) {
4141                 if (pi->dpm_level_enable_mask.mclk_dpm_enable_mask & 1)
4142                         pi->dpm_level_enable_mask.mclk_dpm_enable_mask &= 0xFFFFFFFE;
4143         }
4144         pi->dpm_level_enable_mask.pcie_dpm_enable_mask =
4145                 ci_get_dpm_level_enable_mask_value(&pi->dpm_table.pcie_speed_table);
4146
4147         return 0;
4148 }
4149
4150 static u32 ci_get_lowest_enabled_level(struct radeon_device *rdev,
4151                                        u32 level_mask)
4152 {
4153         u32 level = 0;
4154
4155         while ((level_mask & (1 << level)) == 0)
4156                 level++;
4157
4158         return level;
4159 }
4160
4161
4162 int ci_dpm_force_performance_level(struct radeon_device *rdev,
4163                                    enum radeon_dpm_forced_level level)
4164 {
4165         struct ci_power_info *pi = ci_get_pi(rdev);
4166         u32 tmp, levels, i;
4167         int ret;
4168
4169         if (level == RADEON_DPM_FORCED_LEVEL_HIGH) {
4170                 if ((!pi->pcie_dpm_key_disabled) &&
4171                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4172                         levels = 0;
4173                         tmp = pi->dpm_level_enable_mask.pcie_dpm_enable_mask;
4174                         while (tmp >>= 1)
4175                                 levels++;
4176                         if (levels) {
4177                                 ret = ci_dpm_force_state_pcie(rdev, level);
4178                                 if (ret)
4179                                         return ret;
4180                                 for (i = 0; i < rdev->usec_timeout; i++) {
4181                                         tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) &
4182                                                CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT;
4183                                         if (tmp == levels)
4184                                                 break;
4185                                         udelay(1);
4186                                 }
4187                         }
4188                 }
4189                 if ((!pi->sclk_dpm_key_disabled) &&
4190                     pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4191                         levels = 0;
4192                         tmp = pi->dpm_level_enable_mask.sclk_dpm_enable_mask;
4193                         while (tmp >>= 1)
4194                                 levels++;
4195                         if (levels) {
4196                                 ret = ci_dpm_force_state_sclk(rdev, levels);
4197                                 if (ret)
4198                                         return ret;
4199                                 for (i = 0; i < rdev->usec_timeout; i++) {
4200                                         tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4201                                                CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT;
4202                                         if (tmp == levels)
4203                                                 break;
4204                                         udelay(1);
4205                                 }
4206                         }
4207                 }
4208                 if ((!pi->mclk_dpm_key_disabled) &&
4209                     pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4210                         levels = 0;
4211                         tmp = pi->dpm_level_enable_mask.mclk_dpm_enable_mask;
4212                         while (tmp >>= 1)
4213                                 levels++;
4214                         if (levels) {
4215                                 ret = ci_dpm_force_state_mclk(rdev, levels);
4216                                 if (ret)
4217                                         return ret;
4218                                 for (i = 0; i < rdev->usec_timeout; i++) {
4219                                         tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4220                                                CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT;
4221                                         if (tmp == levels)
4222                                                 break;
4223                                         udelay(1);
4224                                 }
4225                         }
4226                 }
4227         } else if (level == RADEON_DPM_FORCED_LEVEL_LOW) {
4228                 if ((!pi->sclk_dpm_key_disabled) &&
4229                     pi->dpm_level_enable_mask.sclk_dpm_enable_mask) {
4230                         levels = ci_get_lowest_enabled_level(rdev,
4231                                                              pi->dpm_level_enable_mask.sclk_dpm_enable_mask);
4232                         ret = ci_dpm_force_state_sclk(rdev, levels);
4233                         if (ret)
4234                                 return ret;
4235                         for (i = 0; i < rdev->usec_timeout; i++) {
4236                                 tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4237                                        CURR_SCLK_INDEX_MASK) >> CURR_SCLK_INDEX_SHIFT;
4238                                 if (tmp == levels)
4239                                         break;
4240                                 udelay(1);
4241                         }
4242                 }
4243                 if ((!pi->mclk_dpm_key_disabled) &&
4244                     pi->dpm_level_enable_mask.mclk_dpm_enable_mask) {
4245                         levels = ci_get_lowest_enabled_level(rdev,
4246                                                              pi->dpm_level_enable_mask.mclk_dpm_enable_mask);
4247                         ret = ci_dpm_force_state_mclk(rdev, levels);
4248                         if (ret)
4249                                 return ret;
4250                         for (i = 0; i < rdev->usec_timeout; i++) {
4251                                 tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX) &
4252                                        CURR_MCLK_INDEX_MASK) >> CURR_MCLK_INDEX_SHIFT;
4253                                 if (tmp == levels)
4254                                         break;
4255                                 udelay(1);
4256                         }
4257                 }
4258                 if ((!pi->pcie_dpm_key_disabled) &&
4259                     pi->dpm_level_enable_mask.pcie_dpm_enable_mask) {
4260                         levels = ci_get_lowest_enabled_level(rdev,
4261                                                              pi->dpm_level_enable_mask.pcie_dpm_enable_mask);
4262                         ret = ci_dpm_force_state_pcie(rdev, levels);
4263                         if (ret)
4264                                 return ret;
4265                         for (i = 0; i < rdev->usec_timeout; i++) {
4266                                 tmp = (RREG32_SMC(TARGET_AND_CURRENT_PROFILE_INDEX_1) &
4267                                        CURR_PCIE_INDEX_MASK) >> CURR_PCIE_INDEX_SHIFT;
4268                                 if (tmp == levels)
4269                                         break;
4270                                 udelay(1);
4271                         }
4272                 }
4273         } else if (level == RADEON_DPM_FORCED_LEVEL_AUTO) {
4274                 if (!pi->pcie_dpm_key_disabled) {
4275                         PPSMC_Result smc_result;
4276
4277                         smc_result = ci_send_msg_to_smc(rdev,
4278                                                         PPSMC_MSG_PCIeDPM_UnForceLevel);
4279                         if (smc_result != PPSMC_Result_OK)
4280                                 return -EINVAL;
4281                 }
4282                 ret = ci_upload_dpm_level_enable_mask(rdev);
4283                 if (ret)
4284                         return ret;
4285         }
4286
4287         rdev->pm.dpm.forced_level = level;
4288
4289         return 0;
4290 }
4291
4292 static int ci_set_mc_special_registers(struct radeon_device *rdev,
4293                                        struct ci_mc_reg_table *table)
4294 {
4295         struct ci_power_info *pi = ci_get_pi(rdev);
4296         u8 i, j, k;
4297         u32 temp_reg;
4298
4299         for (i = 0, j = table->last; i < table->last; i++) {
4300                 if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4301                         return -EINVAL;
4302                 switch (table->mc_reg_address[i].s1 << 2) {
4303                 case MC_SEQ_MISC1:
4304                         temp_reg = RREG32(MC_PMG_CMD_EMRS);
4305                         table->mc_reg_address[j].s1 = MC_PMG_CMD_EMRS >> 2;
4306                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
4307                         for (k = 0; k < table->num_entries; k++) {
4308                                 table->mc_reg_table_entry[k].mc_data[j] =
4309                                         ((temp_reg & 0xffff0000)) | ((table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16);
4310                         }
4311                         j++;
4312                         if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4313                                 return -EINVAL;
4314
4315                         temp_reg = RREG32(MC_PMG_CMD_MRS);
4316                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS >> 2;
4317                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS_LP >> 2;
4318                         for (k = 0; k < table->num_entries; k++) {
4319                                 table->mc_reg_table_entry[k].mc_data[j] =
4320                                         (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4321                                 if (!pi->mem_gddr5)
4322                                         table->mc_reg_table_entry[k].mc_data[j] |= 0x100;
4323                         }
4324                         j++;
4325                         if (j >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4326                                 return -EINVAL;
4327
4328                         if (!pi->mem_gddr5) {
4329                                 table->mc_reg_address[j].s1 = MC_PMG_AUTO_CMD >> 2;
4330                                 table->mc_reg_address[j].s0 = MC_PMG_AUTO_CMD >> 2;
4331                                 for (k = 0; k < table->num_entries; k++) {
4332                                         table->mc_reg_table_entry[k].mc_data[j] =
4333                                                 (table->mc_reg_table_entry[k].mc_data[i] & 0xffff0000) >> 16;
4334                                 }
4335                                 j++;
4336                                 if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4337                                         return -EINVAL;
4338                         }
4339                         break;
4340                 case MC_SEQ_RESERVE_M:
4341                         temp_reg = RREG32(MC_PMG_CMD_MRS1);
4342                         table->mc_reg_address[j].s1 = MC_PMG_CMD_MRS1 >> 2;
4343                         table->mc_reg_address[j].s0 = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
4344                         for (k = 0; k < table->num_entries; k++) {
4345                                 table->mc_reg_table_entry[k].mc_data[j] =
4346                                         (temp_reg & 0xffff0000) | (table->mc_reg_table_entry[k].mc_data[i] & 0x0000ffff);
4347                         }
4348                         j++;
4349                         if (j > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4350                                 return -EINVAL;
4351                         break;
4352                 default:
4353                         break;
4354                 }
4355
4356         }
4357
4358         table->last = j;
4359
4360         return 0;
4361 }
4362
4363 static bool ci_check_s0_mc_reg_index(u16 in_reg, u16 *out_reg)
4364 {
4365         bool result = true;
4366
4367         switch (in_reg) {
4368         case MC_SEQ_RAS_TIMING >> 2:
4369                 *out_reg = MC_SEQ_RAS_TIMING_LP >> 2;
4370                 break;
4371         case MC_SEQ_DLL_STBY >> 2:
4372                 *out_reg = MC_SEQ_DLL_STBY_LP >> 2;
4373                 break;
4374         case MC_SEQ_G5PDX_CMD0 >> 2:
4375                 *out_reg = MC_SEQ_G5PDX_CMD0_LP >> 2;
4376                 break;
4377         case MC_SEQ_G5PDX_CMD1 >> 2:
4378                 *out_reg = MC_SEQ_G5PDX_CMD1_LP >> 2;
4379                 break;
4380         case MC_SEQ_G5PDX_CTRL >> 2:
4381                 *out_reg = MC_SEQ_G5PDX_CTRL_LP >> 2;
4382                 break;
4383         case MC_SEQ_CAS_TIMING >> 2:
4384                 *out_reg = MC_SEQ_CAS_TIMING_LP >> 2;
4385                 break;
4386         case MC_SEQ_MISC_TIMING >> 2:
4387                 *out_reg = MC_SEQ_MISC_TIMING_LP >> 2;
4388                 break;
4389         case MC_SEQ_MISC_TIMING2 >> 2:
4390                 *out_reg = MC_SEQ_MISC_TIMING2_LP >> 2;
4391                 break;
4392         case MC_SEQ_PMG_DVS_CMD >> 2:
4393                 *out_reg = MC_SEQ_PMG_DVS_CMD_LP >> 2;
4394                 break;
4395         case MC_SEQ_PMG_DVS_CTL >> 2:
4396                 *out_reg = MC_SEQ_PMG_DVS_CTL_LP >> 2;
4397                 break;
4398         case MC_SEQ_RD_CTL_D0 >> 2:
4399                 *out_reg = MC_SEQ_RD_CTL_D0_LP >> 2;
4400                 break;
4401         case MC_SEQ_RD_CTL_D1 >> 2:
4402                 *out_reg = MC_SEQ_RD_CTL_D1_LP >> 2;
4403                 break;
4404         case MC_SEQ_WR_CTL_D0 >> 2:
4405                 *out_reg = MC_SEQ_WR_CTL_D0_LP >> 2;
4406                 break;
4407         case MC_SEQ_WR_CTL_D1 >> 2:
4408                 *out_reg = MC_SEQ_WR_CTL_D1_LP >> 2;
4409                 break;
4410         case MC_PMG_CMD_EMRS >> 2:
4411                 *out_reg = MC_SEQ_PMG_CMD_EMRS_LP >> 2;
4412                 break;
4413         case MC_PMG_CMD_MRS >> 2:
4414                 *out_reg = MC_SEQ_PMG_CMD_MRS_LP >> 2;
4415                 break;
4416         case MC_PMG_CMD_MRS1 >> 2:
4417                 *out_reg = MC_SEQ_PMG_CMD_MRS1_LP >> 2;
4418                 break;
4419         case MC_SEQ_PMG_TIMING >> 2:
4420                 *out_reg = MC_SEQ_PMG_TIMING_LP >> 2;
4421                 break;
4422         case MC_PMG_CMD_MRS2 >> 2:
4423                 *out_reg = MC_SEQ_PMG_CMD_MRS2_LP >> 2;
4424                 break;
4425         case MC_SEQ_WR_CTL_2 >> 2:
4426                 *out_reg = MC_SEQ_WR_CTL_2_LP >> 2;
4427                 break;
4428         default:
4429                 result = false;
4430                 break;
4431         }
4432
4433         return result;
4434 }
4435
4436 static void ci_set_valid_flag(struct ci_mc_reg_table *table)
4437 {
4438         u8 i, j;
4439
4440         for (i = 0; i < table->last; i++) {
4441                 for (j = 1; j < table->num_entries; j++) {
4442                         if (table->mc_reg_table_entry[j-1].mc_data[i] !=
4443                             table->mc_reg_table_entry[j].mc_data[i]) {
4444                                 table->valid_flag |= 1 << i;
4445                                 break;
4446                         }
4447                 }
4448         }
4449 }
4450
4451 static void ci_set_s0_mc_reg_index(struct ci_mc_reg_table *table)
4452 {
4453         u32 i;
4454         u16 address;
4455
4456         for (i = 0; i < table->last; i++) {
4457                 table->mc_reg_address[i].s0 =
4458                         ci_check_s0_mc_reg_index(table->mc_reg_address[i].s1, &address) ?
4459                         address : table->mc_reg_address[i].s1;
4460         }
4461 }
4462
4463 static int ci_copy_vbios_mc_reg_table(const struct atom_mc_reg_table *table,
4464                                       struct ci_mc_reg_table *ci_table)
4465 {
4466         u8 i, j;
4467
4468         if (table->last > SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4469                 return -EINVAL;
4470         if (table->num_entries > MAX_AC_TIMING_ENTRIES)
4471                 return -EINVAL;
4472
4473         for (i = 0; i < table->last; i++)
4474                 ci_table->mc_reg_address[i].s1 = table->mc_reg_address[i].s1;
4475
4476         ci_table->last = table->last;
4477
4478         for (i = 0; i < table->num_entries; i++) {
4479                 ci_table->mc_reg_table_entry[i].mclk_max =
4480                         table->mc_reg_table_entry[i].mclk_max;
4481                 for (j = 0; j < table->last; j++)
4482                         ci_table->mc_reg_table_entry[i].mc_data[j] =
4483                                 table->mc_reg_table_entry[i].mc_data[j];
4484         }
4485         ci_table->num_entries = table->num_entries;
4486
4487         return 0;
4488 }
4489
4490 static int ci_register_patching_mc_seq(struct radeon_device *rdev,
4491                                        struct ci_mc_reg_table *table)
4492 {
4493         u8 i, k;
4494         u32 tmp;
4495         bool patch;
4496
4497         tmp = RREG32(MC_SEQ_MISC0);
4498         patch = ((tmp & 0x0000f00) == 0x300) ? true : false;
4499
4500         if (patch &&
4501             ((rdev->pdev->device == 0x67B0) ||
4502              (rdev->pdev->device == 0x67B1))) {
4503                 for (i = 0; i < table->last; i++) {
4504                         if (table->last >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4505                                 return -EINVAL;
4506                         switch (table->mc_reg_address[i].s1 >> 2) {
4507                         case MC_SEQ_MISC1:
4508                                 for (k = 0; k < table->num_entries; k++) {
4509                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4510                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4511                                                 table->mc_reg_table_entry[k].mc_data[i] =
4512                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFF8) |
4513                                                         0x00000007;
4514                                 }
4515                                 break;
4516                         case MC_SEQ_WR_CTL_D0:
4517                                 for (k = 0; k < table->num_entries; k++) {
4518                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4519                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4520                                                 table->mc_reg_table_entry[k].mc_data[i] =
4521                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4522                                                         0x0000D0DD;
4523                                 }
4524                                 break;
4525                         case MC_SEQ_WR_CTL_D1:
4526                                 for (k = 0; k < table->num_entries; k++) {
4527                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4528                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4529                                                 table->mc_reg_table_entry[k].mc_data[i] =
4530                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFF0F00) |
4531                                                         0x0000D0DD;
4532                                 }
4533                                 break;
4534                         case MC_SEQ_WR_CTL_2:
4535                                 for (k = 0; k < table->num_entries; k++) {
4536                                         if ((table->mc_reg_table_entry[k].mclk_max == 125000) ||
4537                                             (table->mc_reg_table_entry[k].mclk_max == 137500))
4538                                                 table->mc_reg_table_entry[k].mc_data[i] = 0;
4539                                 }
4540                                 break;
4541                         case MC_SEQ_CAS_TIMING:
4542                                 for (k = 0; k < table->num_entries; k++) {
4543                                         if (table->mc_reg_table_entry[k].mclk_max == 125000)
4544                                                 table->mc_reg_table_entry[k].mc_data[i] =
4545                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4546                                                         0x000C0140;
4547                                         else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4548                                                 table->mc_reg_table_entry[k].mc_data[i] =
4549                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFE0FE0F) |
4550                                                         0x000C0150;
4551                                 }
4552                                 break;
4553                         case MC_SEQ_MISC_TIMING:
4554                                 for (k = 0; k < table->num_entries; k++) {
4555                                         if (table->mc_reg_table_entry[k].mclk_max == 125000)
4556                                                 table->mc_reg_table_entry[k].mc_data[i] =
4557                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4558                                                         0x00000030;
4559                                         else if (table->mc_reg_table_entry[k].mclk_max == 137500)
4560                                                 table->mc_reg_table_entry[k].mc_data[i] =
4561                                                         (table->mc_reg_table_entry[k].mc_data[i] & 0xFFFFFFE0) |
4562                                                         0x00000035;
4563                                 }
4564                                 break;
4565                         default:
4566                                 break;
4567                         }
4568                 }
4569
4570                 WREG32(MC_SEQ_IO_DEBUG_INDEX, 3);
4571                 tmp = RREG32(MC_SEQ_IO_DEBUG_DATA);
4572                 tmp = (tmp & 0xFFF8FFFF) | (1 << 16);
4573                 WREG32(MC_SEQ_IO_DEBUG_INDEX, 3);
4574                 WREG32(MC_SEQ_IO_DEBUG_DATA, tmp);
4575         }
4576
4577         return 0;
4578 }
4579
4580 static int ci_initialize_mc_reg_table(struct radeon_device *rdev)
4581 {
4582         struct ci_power_info *pi = ci_get_pi(rdev);
4583         struct atom_mc_reg_table *table;
4584         struct ci_mc_reg_table *ci_table = &pi->mc_reg_table;
4585         u8 module_index = rv770_get_memory_module_index(rdev);
4586         int ret;
4587
4588         table = kzalloc(sizeof(struct atom_mc_reg_table), GFP_KERNEL);
4589         if (!table)
4590                 return -ENOMEM;
4591
4592         WREG32(MC_SEQ_RAS_TIMING_LP, RREG32(MC_SEQ_RAS_TIMING));
4593         WREG32(MC_SEQ_CAS_TIMING_LP, RREG32(MC_SEQ_CAS_TIMING));
4594         WREG32(MC_SEQ_DLL_STBY_LP, RREG32(MC_SEQ_DLL_STBY));
4595         WREG32(MC_SEQ_G5PDX_CMD0_LP, RREG32(MC_SEQ_G5PDX_CMD0));
4596         WREG32(MC_SEQ_G5PDX_CMD1_LP, RREG32(MC_SEQ_G5PDX_CMD1));
4597         WREG32(MC_SEQ_G5PDX_CTRL_LP, RREG32(MC_SEQ_G5PDX_CTRL));
4598         WREG32(MC_SEQ_PMG_DVS_CMD_LP, RREG32(MC_SEQ_PMG_DVS_CMD));
4599         WREG32(MC_SEQ_PMG_DVS_CTL_LP, RREG32(MC_SEQ_PMG_DVS_CTL));
4600         WREG32(MC_SEQ_MISC_TIMING_LP, RREG32(MC_SEQ_MISC_TIMING));
4601         WREG32(MC_SEQ_MISC_TIMING2_LP, RREG32(MC_SEQ_MISC_TIMING2));
4602         WREG32(MC_SEQ_PMG_CMD_EMRS_LP, RREG32(MC_PMG_CMD_EMRS));
4603         WREG32(MC_SEQ_PMG_CMD_MRS_LP, RREG32(MC_PMG_CMD_MRS));
4604         WREG32(MC_SEQ_PMG_CMD_MRS1_LP, RREG32(MC_PMG_CMD_MRS1));
4605         WREG32(MC_SEQ_WR_CTL_D0_LP, RREG32(MC_SEQ_WR_CTL_D0));
4606         WREG32(MC_SEQ_WR_CTL_D1_LP, RREG32(MC_SEQ_WR_CTL_D1));
4607         WREG32(MC_SEQ_RD_CTL_D0_LP, RREG32(MC_SEQ_RD_CTL_D0));
4608         WREG32(MC_SEQ_RD_CTL_D1_LP, RREG32(MC_SEQ_RD_CTL_D1));
4609         WREG32(MC_SEQ_PMG_TIMING_LP, RREG32(MC_SEQ_PMG_TIMING));
4610         WREG32(MC_SEQ_PMG_CMD_MRS2_LP, RREG32(MC_PMG_CMD_MRS2));
4611         WREG32(MC_SEQ_WR_CTL_2_LP, RREG32(MC_SEQ_WR_CTL_2));
4612
4613         ret = radeon_atom_init_mc_reg_table(rdev, module_index, table);
4614         if (ret)
4615                 goto init_mc_done;
4616
4617         ret = ci_copy_vbios_mc_reg_table(table, ci_table);
4618         if (ret)
4619                 goto init_mc_done;
4620
4621         ci_set_s0_mc_reg_index(ci_table);
4622
4623         ret = ci_register_patching_mc_seq(rdev, ci_table);
4624         if (ret)
4625                 goto init_mc_done;
4626
4627         ret = ci_set_mc_special_registers(rdev, ci_table);
4628         if (ret)
4629                 goto init_mc_done;
4630
4631         ci_set_valid_flag(ci_table);
4632
4633 init_mc_done:
4634         kfree(table);
4635
4636         return ret;
4637 }
4638
4639 static int ci_populate_mc_reg_addresses(struct radeon_device *rdev,
4640                                         SMU7_Discrete_MCRegisters *mc_reg_table)
4641 {
4642         struct ci_power_info *pi = ci_get_pi(rdev);
4643         u32 i, j;
4644
4645         for (i = 0, j = 0; j < pi->mc_reg_table.last; j++) {
4646                 if (pi->mc_reg_table.valid_flag & (1 << j)) {
4647                         if (i >= SMU7_DISCRETE_MC_REGISTER_ARRAY_SIZE)
4648                                 return -EINVAL;
4649                         mc_reg_table->address[i].s0 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s0);
4650                         mc_reg_table->address[i].s1 = cpu_to_be16(pi->mc_reg_table.mc_reg_address[j].s1);
4651                         i++;
4652                 }
4653         }
4654
4655         mc_reg_table->last = (u8)i;
4656
4657         return 0;
4658 }
4659
4660 static void ci_convert_mc_registers(const struct ci_mc_reg_entry *entry,
4661                                     SMU7_Discrete_MCRegisterSet *data,
4662                                     u32 num_entries, u32 valid_flag)
4663 {
4664         u32 i, j;
4665
4666         for (i = 0, j = 0; j < num_entries; j++) {
4667                 if (valid_flag & (1 << j)) {
4668                         data->value[i] = cpu_to_be32(entry->mc_data[j]);
4669                         i++;
4670                 }
4671         }
4672 }
4673
4674 static void ci_convert_mc_reg_table_entry_to_smc(struct radeon_device *rdev,
4675                                                  const u32 memory_clock,
4676                                                  SMU7_Discrete_MCRegisterSet *mc_reg_table_data)
4677 {
4678         struct ci_power_info *pi = ci_get_pi(rdev);
4679         u32 i = 0;
4680
4681         for (i = 0; i < pi->mc_reg_table.num_entries; i++) {
4682                 if (memory_clock <= pi->mc_reg_table.mc_reg_table_entry[i].mclk_max)
4683                         break;
4684         }
4685
4686         if ((i == pi->mc_reg_table.num_entries) && (i > 0))
4687                 --i;
4688
4689         ci_convert_mc_registers(&pi->mc_reg_table.mc_reg_table_entry[i],
4690                                 mc_reg_table_data, pi->mc_reg_table.last,
4691                                 pi->mc_reg_table.valid_flag);
4692 }
4693
4694 static void ci_convert_mc_reg_table_to_smc(struct radeon_device *rdev,
4695                                            SMU7_Discrete_MCRegisters *mc_reg_table)
4696 {
4697         struct ci_power_info *pi = ci_get_pi(rdev);
4698         u32 i;
4699
4700         for (i = 0; i < pi->dpm_table.mclk_table.count; i++)
4701                 ci_convert_mc_reg_table_entry_to_smc(rdev,
4702                                                      pi->dpm_table.mclk_table.dpm_levels[i].value,
4703                                                      &mc_reg_table->data[i]);
4704 }
4705
4706 static int ci_populate_initial_mc_reg_table(struct radeon_device *rdev)
4707 {
4708         struct ci_power_info *pi = ci_get_pi(rdev);
4709         int ret;
4710
4711         memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4712
4713         ret = ci_populate_mc_reg_addresses(rdev, &pi->smc_mc_reg_table);
4714         if (ret)
4715                 return ret;
4716         ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table);
4717
4718         return ci_copy_bytes_to_smc(rdev,
4719                                     pi->mc_reg_table_start,
4720                                     (u8 *)&pi->smc_mc_reg_table,
4721                                     sizeof(SMU7_Discrete_MCRegisters),
4722                                     pi->sram_end);
4723 }
4724
4725 static int ci_update_and_upload_mc_reg_table(struct radeon_device *rdev)
4726 {
4727         struct ci_power_info *pi = ci_get_pi(rdev);
4728
4729         if (!(pi->need_update_smu7_dpm_table & DPMTABLE_OD_UPDATE_MCLK))
4730                 return 0;
4731
4732         memset(&pi->smc_mc_reg_table, 0, sizeof(SMU7_Discrete_MCRegisters));
4733
4734         ci_convert_mc_reg_table_to_smc(rdev, &pi->smc_mc_reg_table);
4735
4736         return ci_copy_bytes_to_smc(rdev,
4737                                     pi->mc_reg_table_start +
4738                                     offsetof(SMU7_Discrete_MCRegisters, data[0]),
4739                                     (u8 *)&pi->smc_mc_reg_table.data[0],
4740                                     sizeof(SMU7_Discrete_MCRegisterSet) *
4741                                     pi->dpm_table.mclk_table.count,
4742                                     pi->sram_end);
4743 }
4744
4745 static void ci_enable_voltage_control(struct radeon_device *rdev)
4746 {
4747         u32 tmp = RREG32_SMC(GENERAL_PWRMGT);
4748
4749         tmp |= VOLT_PWRMGT_EN;
4750         WREG32_SMC(GENERAL_PWRMGT, tmp);
4751 }
4752
4753 static enum radeon_pcie_gen ci_get_maximum_link_speed(struct radeon_device *rdev,
4754                                                       struct radeon_ps *radeon_state)
4755 {
4756         struct ci_ps *state = ci_get_ps(radeon_state);
4757         int i;
4758         u16 pcie_speed, max_speed = 0;
4759
4760         for (i = 0; i < state->performance_level_count; i++) {
4761                 pcie_speed = state->performance_levels[i].pcie_gen;
4762                 if (max_speed < pcie_speed)
4763                         max_speed = pcie_speed;
4764         }
4765
4766         return max_speed;
4767 }
4768
4769 static u16 ci_get_current_pcie_speed(struct radeon_device *rdev)
4770 {
4771         u32 speed_cntl = 0;
4772
4773         speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL) & LC_CURRENT_DATA_RATE_MASK;
4774         speed_cntl >>= LC_CURRENT_DATA_RATE_SHIFT;
4775
4776         return (u16)speed_cntl;
4777 }
4778
4779 static int ci_get_current_pcie_lane_number(struct radeon_device *rdev)
4780 {
4781         u32 link_width = 0;
4782
4783         link_width = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL) & LC_LINK_WIDTH_RD_MASK;
4784         link_width >>= LC_LINK_WIDTH_RD_SHIFT;
4785
4786         switch (link_width) {
4787         case RADEON_PCIE_LC_LINK_WIDTH_X1:
4788                 return 1;
4789         case RADEON_PCIE_LC_LINK_WIDTH_X2:
4790                 return 2;
4791         case RADEON_PCIE_LC_LINK_WIDTH_X4:
4792                 return 4;
4793         case RADEON_PCIE_LC_LINK_WIDTH_X8:
4794                 return 8;
4795         case RADEON_PCIE_LC_LINK_WIDTH_X12:
4796                 /* not actually supported */
4797                 return 12;
4798         case RADEON_PCIE_LC_LINK_WIDTH_X0:
4799         case RADEON_PCIE_LC_LINK_WIDTH_X16:
4800         default:
4801                 return 16;
4802         }
4803 }
4804
4805 static void ci_request_link_speed_change_before_state_change(struct radeon_device *rdev,
4806                                                              struct radeon_ps *radeon_new_state,
4807                                                              struct radeon_ps *radeon_current_state)
4808 {
4809         struct ci_power_info *pi = ci_get_pi(rdev);
4810         enum radeon_pcie_gen target_link_speed =
4811                 ci_get_maximum_link_speed(rdev, radeon_new_state);
4812         enum radeon_pcie_gen current_link_speed;
4813
4814         if (pi->force_pcie_gen == RADEON_PCIE_GEN_INVALID)
4815                 current_link_speed = ci_get_maximum_link_speed(rdev, radeon_current_state);
4816         else
4817                 current_link_speed = pi->force_pcie_gen;
4818
4819         pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
4820         pi->pspp_notify_required = false;
4821         if (target_link_speed > current_link_speed) {
4822                 switch (target_link_speed) {
4823 #ifdef CONFIG_ACPI
4824                 case RADEON_PCIE_GEN3:
4825                         if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN3, false) == 0)
4826                                 break;
4827                         pi->force_pcie_gen = RADEON_PCIE_GEN2;
4828                         if (current_link_speed == RADEON_PCIE_GEN2)
4829                                 break;
4830                         fallthrough;
4831                 case RADEON_PCIE_GEN2:
4832                         if (radeon_acpi_pcie_performance_request(rdev, PCIE_PERF_REQ_PECI_GEN2, false) == 0)
4833                                 break;
4834                         fallthrough;
4835 #endif
4836                 default:
4837                         pi->force_pcie_gen = ci_get_current_pcie_speed(rdev);
4838                         break;
4839                 }
4840         } else {
4841                 if (target_link_speed < current_link_speed)
4842                         pi->pspp_notify_required = true;
4843         }
4844 }
4845
4846 static void ci_notify_link_speed_change_after_state_change(struct radeon_device *rdev,
4847                                                            struct radeon_ps *radeon_new_state,
4848                                                            struct radeon_ps *radeon_current_state)
4849 {
4850         struct ci_power_info *pi = ci_get_pi(rdev);
4851         enum radeon_pcie_gen target_link_speed =
4852                 ci_get_maximum_link_speed(rdev, radeon_new_state);
4853         u8 request;
4854
4855         if (pi->pspp_notify_required) {
4856                 if (target_link_speed == RADEON_PCIE_GEN3)
4857                         request = PCIE_PERF_REQ_PECI_GEN3;
4858                 else if (target_link_speed == RADEON_PCIE_GEN2)
4859                         request = PCIE_PERF_REQ_PECI_GEN2;
4860                 else
4861                         request = PCIE_PERF_REQ_PECI_GEN1;
4862
4863                 if ((request == PCIE_PERF_REQ_PECI_GEN1) &&
4864                     (ci_get_current_pcie_speed(rdev) > 0))
4865                         return;
4866
4867 #ifdef CONFIG_ACPI
4868                 radeon_acpi_pcie_performance_request(rdev, request, false);
4869 #endif
4870         }
4871 }
4872
4873 static int ci_set_private_data_variables_based_on_pptable(struct radeon_device *rdev)
4874 {
4875         struct ci_power_info *pi = ci_get_pi(rdev);
4876         struct radeon_clock_voltage_dependency_table *allowed_sclk_vddc_table =
4877                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk;
4878         struct radeon_clock_voltage_dependency_table *allowed_mclk_vddc_table =
4879                 &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk;
4880         struct radeon_clock_voltage_dependency_table *allowed_mclk_vddci_table =
4881                 &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk;
4882
4883         if (allowed_sclk_vddc_table == NULL)
4884                 return -EINVAL;
4885         if (allowed_sclk_vddc_table->count < 1)
4886                 return -EINVAL;
4887         if (allowed_mclk_vddc_table == NULL)
4888                 return -EINVAL;
4889         if (allowed_mclk_vddc_table->count < 1)
4890                 return -EINVAL;
4891         if (allowed_mclk_vddci_table == NULL)
4892                 return -EINVAL;
4893         if (allowed_mclk_vddci_table->count < 1)
4894                 return -EINVAL;
4895
4896         pi->min_vddc_in_pp_table = allowed_sclk_vddc_table->entries[0].v;
4897         pi->max_vddc_in_pp_table =
4898                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
4899
4900         pi->min_vddci_in_pp_table = allowed_mclk_vddci_table->entries[0].v;
4901         pi->max_vddci_in_pp_table =
4902                 allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
4903
4904         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.sclk =
4905                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
4906         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.mclk =
4907                 allowed_mclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].clk;
4908         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddc =
4909                 allowed_sclk_vddc_table->entries[allowed_sclk_vddc_table->count - 1].v;
4910         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac.vddci =
4911                 allowed_mclk_vddci_table->entries[allowed_mclk_vddci_table->count - 1].v;
4912
4913         return 0;
4914 }
4915
4916 static void ci_patch_with_vddc_leakage(struct radeon_device *rdev, u16 *vddc)
4917 {
4918         struct ci_power_info *pi = ci_get_pi(rdev);
4919         struct ci_leakage_voltage *leakage_table = &pi->vddc_leakage;
4920         u32 leakage_index;
4921
4922         for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
4923                 if (leakage_table->leakage_id[leakage_index] == *vddc) {
4924                         *vddc = leakage_table->actual_voltage[leakage_index];
4925                         break;
4926                 }
4927         }
4928 }
4929
4930 static void ci_patch_with_vddci_leakage(struct radeon_device *rdev, u16 *vddci)
4931 {
4932         struct ci_power_info *pi = ci_get_pi(rdev);
4933         struct ci_leakage_voltage *leakage_table = &pi->vddci_leakage;
4934         u32 leakage_index;
4935
4936         for (leakage_index = 0; leakage_index < leakage_table->count; leakage_index++) {
4937                 if (leakage_table->leakage_id[leakage_index] == *vddci) {
4938                         *vddci = leakage_table->actual_voltage[leakage_index];
4939                         break;
4940                 }
4941         }
4942 }
4943
4944 static void ci_patch_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4945                                                                       struct radeon_clock_voltage_dependency_table *table)
4946 {
4947         u32 i;
4948
4949         if (table) {
4950                 for (i = 0; i < table->count; i++)
4951                         ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4952         }
4953 }
4954
4955 static void ci_patch_clock_voltage_dependency_table_with_vddci_leakage(struct radeon_device *rdev,
4956                                                                        struct radeon_clock_voltage_dependency_table *table)
4957 {
4958         u32 i;
4959
4960         if (table) {
4961                 for (i = 0; i < table->count; i++)
4962                         ci_patch_with_vddci_leakage(rdev, &table->entries[i].v);
4963         }
4964 }
4965
4966 static void ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4967                                                                           struct radeon_vce_clock_voltage_dependency_table *table)
4968 {
4969         u32 i;
4970
4971         if (table) {
4972                 for (i = 0; i < table->count; i++)
4973                         ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4974         }
4975 }
4976
4977 static void ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(struct radeon_device *rdev,
4978                                                                           struct radeon_uvd_clock_voltage_dependency_table *table)
4979 {
4980         u32 i;
4981
4982         if (table) {
4983                 for (i = 0; i < table->count; i++)
4984                         ci_patch_with_vddc_leakage(rdev, &table->entries[i].v);
4985         }
4986 }
4987
4988 static void ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(struct radeon_device *rdev,
4989                                                                    struct radeon_phase_shedding_limits_table *table)
4990 {
4991         u32 i;
4992
4993         if (table) {
4994                 for (i = 0; i < table->count; i++)
4995                         ci_patch_with_vddc_leakage(rdev, &table->entries[i].voltage);
4996         }
4997 }
4998
4999 static void ci_patch_clock_voltage_limits_with_vddc_leakage(struct radeon_device *rdev,
5000                                                             struct radeon_clock_and_voltage_limits *table)
5001 {
5002         if (table) {
5003                 ci_patch_with_vddc_leakage(rdev, (u16 *)&table->vddc);
5004                 ci_patch_with_vddci_leakage(rdev, (u16 *)&table->vddci);
5005         }
5006 }
5007
5008 static void ci_patch_cac_leakage_table_with_vddc_leakage(struct radeon_device *rdev,
5009                                                          struct radeon_cac_leakage_table *table)
5010 {
5011         u32 i;
5012
5013         if (table) {
5014                 for (i = 0; i < table->count; i++)
5015                         ci_patch_with_vddc_leakage(rdev, &table->entries[i].vddc);
5016         }
5017 }
5018
5019 static void ci_patch_dependency_tables_with_leakage(struct radeon_device *rdev)
5020 {
5021
5022         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5023                                                                   &rdev->pm.dpm.dyn_state.vddc_dependency_on_sclk);
5024         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5025                                                                   &rdev->pm.dpm.dyn_state.vddc_dependency_on_mclk);
5026         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5027                                                                   &rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk);
5028         ci_patch_clock_voltage_dependency_table_with_vddci_leakage(rdev,
5029                                                                    &rdev->pm.dpm.dyn_state.vddci_dependency_on_mclk);
5030         ci_patch_vce_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5031                                                                       &rdev->pm.dpm.dyn_state.vce_clock_voltage_dependency_table);
5032         ci_patch_uvd_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5033                                                                       &rdev->pm.dpm.dyn_state.uvd_clock_voltage_dependency_table);
5034         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5035                                                                   &rdev->pm.dpm.dyn_state.samu_clock_voltage_dependency_table);
5036         ci_patch_clock_voltage_dependency_table_with_vddc_leakage(rdev,
5037                                                                   &rdev->pm.dpm.dyn_state.acp_clock_voltage_dependency_table);
5038         ci_patch_vddc_phase_shed_limit_table_with_vddc_leakage(rdev,
5039                                                                &rdev->pm.dpm.dyn_state.phase_shedding_limits_table);
5040         ci_patch_clock_voltage_limits_with_vddc_leakage(rdev,
5041                                                         &rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac);
5042         ci_patch_clock_voltage_limits_with_vddc_leakage(rdev,
5043                                                         &rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc);
5044         ci_patch_cac_leakage_table_with_vddc_leakage(rdev,
5045                                                      &rdev->pm.dpm.dyn_state.cac_leakage_table);
5046
5047 }
5048
5049 static void ci_get_memory_type(struct radeon_device *rdev)
5050 {
5051         struct ci_power_info *pi = ci_get_pi(rdev);
5052         u32 tmp;
5053
5054         tmp = RREG32(MC_SEQ_MISC0);
5055
5056         if (((tmp & MC_SEQ_MISC0_GDDR5_MASK) >> MC_SEQ_MISC0_GDDR5_SHIFT) ==
5057             MC_SEQ_MISC0_GDDR5_VALUE)
5058                 pi->mem_gddr5 = true;
5059         else
5060                 pi->mem_gddr5 = false;
5061
5062 }
5063
5064 static void ci_update_current_ps(struct radeon_device *rdev,
5065                                  struct radeon_ps *rps)
5066 {
5067         struct ci_ps *new_ps = ci_get_ps(rps);
5068         struct ci_power_info *pi = ci_get_pi(rdev);
5069
5070         pi->current_rps = *rps;
5071         pi->current_ps = *new_ps;
5072         pi->current_rps.ps_priv = &pi->current_ps;
5073 }
5074
5075 static void ci_update_requested_ps(struct radeon_device *rdev,
5076                                    struct radeon_ps *rps)
5077 {
5078         struct ci_ps *new_ps = ci_get_ps(rps);
5079         struct ci_power_info *pi = ci_get_pi(rdev);
5080
5081         pi->requested_rps = *rps;
5082         pi->requested_ps = *new_ps;
5083         pi->requested_rps.ps_priv = &pi->requested_ps;
5084 }
5085
5086 int ci_dpm_pre_set_power_state(struct radeon_device *rdev)
5087 {
5088         struct ci_power_info *pi = ci_get_pi(rdev);
5089         struct radeon_ps requested_ps = *rdev->pm.dpm.requested_ps;
5090         struct radeon_ps *new_ps = &requested_ps;
5091
5092         ci_update_requested_ps(rdev, new_ps);
5093
5094         ci_apply_state_adjust_rules(rdev, &pi->requested_rps);
5095
5096         return 0;
5097 }
5098
5099 void ci_dpm_post_set_power_state(struct radeon_device *rdev)
5100 {
5101         struct ci_power_info *pi = ci_get_pi(rdev);
5102         struct radeon_ps *new_ps = &pi->requested_rps;
5103
5104         ci_update_current_ps(rdev, new_ps);
5105 }
5106
5107
5108 void ci_dpm_setup_asic(struct radeon_device *rdev)
5109 {
5110         int r;
5111
5112         r = ci_mc_load_microcode(rdev);
5113         if (r)
5114                 DRM_ERROR("Failed to load MC firmware!\n");
5115         ci_read_clock_registers(rdev);
5116         ci_get_memory_type(rdev);
5117         ci_enable_acpi_power_management(rdev);
5118         ci_init_sclk_t(rdev);
5119 }
5120
5121 int ci_dpm_enable(struct radeon_device *rdev)
5122 {
5123         struct ci_power_info *pi = ci_get_pi(rdev);
5124         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
5125         int ret;
5126
5127         if (ci_is_smc_running(rdev))
5128                 return -EINVAL;
5129         if (pi->voltage_control != CISLANDS_VOLTAGE_CONTROL_NONE) {
5130                 ci_enable_voltage_control(rdev);
5131                 ret = ci_construct_voltage_tables(rdev);
5132                 if (ret) {
5133                         DRM_ERROR("ci_construct_voltage_tables failed\n");
5134                         return ret;
5135                 }
5136         }
5137         if (pi->caps_dynamic_ac_timing) {
5138                 ret = ci_initialize_mc_reg_table(rdev);
5139                 if (ret)
5140                         pi->caps_dynamic_ac_timing = false;
5141         }
5142         if (pi->dynamic_ss)
5143                 ci_enable_spread_spectrum(rdev, true);
5144         if (pi->thermal_protection)
5145                 ci_enable_thermal_protection(rdev, true);
5146         ci_program_sstp(rdev);
5147         ci_enable_display_gap(rdev);
5148         ci_program_vc(rdev);
5149         ret = ci_upload_firmware(rdev);
5150         if (ret) {
5151                 DRM_ERROR("ci_upload_firmware failed\n");
5152                 return ret;
5153         }
5154         ret = ci_process_firmware_header(rdev);
5155         if (ret) {
5156                 DRM_ERROR("ci_process_firmware_header failed\n");
5157                 return ret;
5158         }
5159         ret = ci_initial_switch_from_arb_f0_to_f1(rdev);
5160         if (ret) {
5161                 DRM_ERROR("ci_initial_switch_from_arb_f0_to_f1 failed\n");
5162                 return ret;
5163         }
5164         ret = ci_init_smc_table(rdev);
5165         if (ret) {
5166                 DRM_ERROR("ci_init_smc_table failed\n");
5167                 return ret;
5168         }
5169         ret = ci_init_arb_table_index(rdev);
5170         if (ret) {
5171                 DRM_ERROR("ci_init_arb_table_index failed\n");
5172                 return ret;
5173         }
5174         if (pi->caps_dynamic_ac_timing) {
5175                 ret = ci_populate_initial_mc_reg_table(rdev);
5176                 if (ret) {
5177                         DRM_ERROR("ci_populate_initial_mc_reg_table failed\n");
5178                         return ret;
5179                 }
5180         }
5181         ret = ci_populate_pm_base(rdev);
5182         if (ret) {
5183                 DRM_ERROR("ci_populate_pm_base failed\n");
5184                 return ret;
5185         }
5186         ci_dpm_start_smc(rdev);
5187         ci_enable_vr_hot_gpio_interrupt(rdev);
5188         ret = ci_notify_smc_display_change(rdev, false);
5189         if (ret) {
5190                 DRM_ERROR("ci_notify_smc_display_change failed\n");
5191                 return ret;
5192         }
5193         ci_enable_sclk_control(rdev, true);
5194         ret = ci_enable_ulv(rdev, true);
5195         if (ret) {
5196                 DRM_ERROR("ci_enable_ulv failed\n");
5197                 return ret;
5198         }
5199         ret = ci_enable_ds_master_switch(rdev, true);
5200         if (ret) {
5201                 DRM_ERROR("ci_enable_ds_master_switch failed\n");
5202                 return ret;
5203         }
5204         ret = ci_start_dpm(rdev);
5205         if (ret) {
5206                 DRM_ERROR("ci_start_dpm failed\n");
5207                 return ret;
5208         }
5209         ret = ci_enable_didt(rdev, true);
5210         if (ret) {
5211                 DRM_ERROR("ci_enable_didt failed\n");
5212                 return ret;
5213         }
5214         ret = ci_enable_smc_cac(rdev, true);
5215         if (ret) {
5216                 DRM_ERROR("ci_enable_smc_cac failed\n");
5217                 return ret;
5218         }
5219         ret = ci_enable_power_containment(rdev, true);
5220         if (ret) {
5221                 DRM_ERROR("ci_enable_power_containment failed\n");
5222                 return ret;
5223         }
5224
5225         ret = ci_power_control_set_level(rdev);
5226         if (ret) {
5227                 DRM_ERROR("ci_power_control_set_level failed\n");
5228                 return ret;
5229         }
5230
5231         ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, true);
5232
5233         ret = ci_enable_thermal_based_sclk_dpm(rdev, true);
5234         if (ret) {
5235                 DRM_ERROR("ci_enable_thermal_based_sclk_dpm failed\n");
5236                 return ret;
5237         }
5238
5239         ci_thermal_start_thermal_controller(rdev);
5240
5241         ci_update_current_ps(rdev, boot_ps);
5242
5243         return 0;
5244 }
5245
5246 static int ci_set_temperature_range(struct radeon_device *rdev)
5247 {
5248         int ret;
5249
5250         ret = ci_thermal_enable_alert(rdev, false);
5251         if (ret)
5252                 return ret;
5253         ret = ci_thermal_set_temperature_range(rdev, R600_TEMP_RANGE_MIN, R600_TEMP_RANGE_MAX);
5254         if (ret)
5255                 return ret;
5256         ret = ci_thermal_enable_alert(rdev, true);
5257         if (ret)
5258                 return ret;
5259
5260         return ret;
5261 }
5262
5263 int ci_dpm_late_enable(struct radeon_device *rdev)
5264 {
5265         int ret;
5266
5267         ret = ci_set_temperature_range(rdev);
5268         if (ret)
5269                 return ret;
5270
5271         ci_dpm_powergate_uvd(rdev, true);
5272
5273         return 0;
5274 }
5275
5276 void ci_dpm_disable(struct radeon_device *rdev)
5277 {
5278         struct ci_power_info *pi = ci_get_pi(rdev);
5279         struct radeon_ps *boot_ps = rdev->pm.dpm.boot_ps;
5280
5281         ci_dpm_powergate_uvd(rdev, false);
5282
5283         if (!ci_is_smc_running(rdev))
5284                 return;
5285
5286         ci_thermal_stop_thermal_controller(rdev);
5287
5288         if (pi->thermal_protection)
5289                 ci_enable_thermal_protection(rdev, false);
5290         ci_enable_power_containment(rdev, false);
5291         ci_enable_smc_cac(rdev, false);
5292         ci_enable_didt(rdev, false);
5293         ci_enable_spread_spectrum(rdev, false);
5294         ci_enable_auto_throttle_source(rdev, RADEON_DPM_AUTO_THROTTLE_SRC_THERMAL, false);
5295         ci_stop_dpm(rdev);
5296         ci_enable_ds_master_switch(rdev, false);
5297         ci_enable_ulv(rdev, false);
5298         ci_clear_vc(rdev);
5299         ci_reset_to_default(rdev);
5300         ci_dpm_stop_smc(rdev);
5301         ci_force_switch_to_arb_f0(rdev);
5302         ci_enable_thermal_based_sclk_dpm(rdev, false);
5303
5304         ci_update_current_ps(rdev, boot_ps);
5305 }
5306
5307 int ci_dpm_set_power_state(struct radeon_device *rdev)
5308 {
5309         struct ci_power_info *pi = ci_get_pi(rdev);
5310         struct radeon_ps *new_ps = &pi->requested_rps;
5311         struct radeon_ps *old_ps = &pi->current_rps;
5312         int ret;
5313
5314         ci_find_dpm_states_clocks_in_dpm_table(rdev, new_ps);
5315         if (pi->pcie_performance_request)
5316                 ci_request_link_speed_change_before_state_change(rdev, new_ps, old_ps);
5317         ret = ci_freeze_sclk_mclk_dpm(rdev);
5318         if (ret) {
5319                 DRM_ERROR("ci_freeze_sclk_mclk_dpm failed\n");
5320                 return ret;
5321         }
5322         ret = ci_populate_and_upload_sclk_mclk_dpm_levels(rdev, new_ps);
5323         if (ret) {
5324                 DRM_ERROR("ci_populate_and_upload_sclk_mclk_dpm_levels failed\n");
5325                 return ret;
5326         }
5327         ret = ci_generate_dpm_level_enable_mask(rdev, new_ps);
5328         if (ret) {
5329                 DRM_ERROR("ci_generate_dpm_level_enable_mask failed\n");
5330                 return ret;
5331         }
5332
5333         ret = ci_update_vce_dpm(rdev, new_ps, old_ps);
5334         if (ret) {
5335                 DRM_ERROR("ci_update_vce_dpm failed\n");
5336                 return ret;
5337         }
5338
5339         ret = ci_update_sclk_t(rdev);
5340         if (ret) {
5341                 DRM_ERROR("ci_update_sclk_t failed\n");
5342                 return ret;
5343         }
5344         if (pi->caps_dynamic_ac_timing) {
5345                 ret = ci_update_and_upload_mc_reg_table(rdev);
5346                 if (ret) {
5347                         DRM_ERROR("ci_update_and_upload_mc_reg_table failed\n");
5348                         return ret;
5349                 }
5350         }
5351         ret = ci_program_memory_timing_parameters(rdev);
5352         if (ret) {
5353                 DRM_ERROR("ci_program_memory_timing_parameters failed\n");
5354                 return ret;
5355         }
5356         ret = ci_unfreeze_sclk_mclk_dpm(rdev);
5357         if (ret) {
5358                 DRM_ERROR("ci_unfreeze_sclk_mclk_dpm failed\n");
5359                 return ret;
5360         }
5361         ret = ci_upload_dpm_level_enable_mask(rdev);
5362         if (ret) {
5363                 DRM_ERROR("ci_upload_dpm_level_enable_mask failed\n");
5364                 return ret;
5365         }
5366         if (pi->pcie_performance_request)
5367                 ci_notify_link_speed_change_after_state_change(rdev, new_ps, old_ps);
5368
5369         return 0;
5370 }
5371
5372 #if 0
5373 void ci_dpm_reset_asic(struct radeon_device *rdev)
5374 {
5375         ci_set_boot_state(rdev);
5376 }
5377 #endif
5378
5379 void ci_dpm_display_configuration_changed(struct radeon_device *rdev)
5380 {
5381         ci_program_display_gap(rdev);
5382 }
5383
5384 union power_info {
5385         struct _ATOM_POWERPLAY_INFO info;
5386         struct _ATOM_POWERPLAY_INFO_V2 info_2;
5387         struct _ATOM_POWERPLAY_INFO_V3 info_3;
5388         struct _ATOM_PPLIB_POWERPLAYTABLE pplib;
5389         struct _ATOM_PPLIB_POWERPLAYTABLE2 pplib2;
5390         struct _ATOM_PPLIB_POWERPLAYTABLE3 pplib3;
5391 };
5392
5393 union pplib_clock_info {
5394         struct _ATOM_PPLIB_R600_CLOCK_INFO r600;
5395         struct _ATOM_PPLIB_RS780_CLOCK_INFO rs780;
5396         struct _ATOM_PPLIB_EVERGREEN_CLOCK_INFO evergreen;
5397         struct _ATOM_PPLIB_SUMO_CLOCK_INFO sumo;
5398         struct _ATOM_PPLIB_SI_CLOCK_INFO si;
5399         struct _ATOM_PPLIB_CI_CLOCK_INFO ci;
5400 };
5401
5402 union pplib_power_state {
5403         struct _ATOM_PPLIB_STATE v1;
5404         struct _ATOM_PPLIB_STATE_V2 v2;
5405 };
5406
5407 static void ci_parse_pplib_non_clock_info(struct radeon_device *rdev,
5408                                           struct radeon_ps *rps,
5409                                           struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info,
5410                                           u8 table_rev)
5411 {
5412         rps->caps = le32_to_cpu(non_clock_info->ulCapsAndSettings);
5413         rps->class = le16_to_cpu(non_clock_info->usClassification);
5414         rps->class2 = le16_to_cpu(non_clock_info->usClassification2);
5415
5416         if (ATOM_PPLIB_NONCLOCKINFO_VER1 < table_rev) {
5417                 rps->vclk = le32_to_cpu(non_clock_info->ulVCLK);
5418                 rps->dclk = le32_to_cpu(non_clock_info->ulDCLK);
5419         } else {
5420                 rps->vclk = 0;
5421                 rps->dclk = 0;
5422         }
5423
5424         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT)
5425                 rdev->pm.dpm.boot_ps = rps;
5426         if (rps->class & ATOM_PPLIB_CLASSIFICATION_UVDSTATE)
5427                 rdev->pm.dpm.uvd_ps = rps;
5428 }
5429
5430 static void ci_parse_pplib_clock_info(struct radeon_device *rdev,
5431                                       struct radeon_ps *rps, int index,
5432                                       union pplib_clock_info *clock_info)
5433 {
5434         struct ci_power_info *pi = ci_get_pi(rdev);
5435         struct ci_ps *ps = ci_get_ps(rps);
5436         struct ci_pl *pl = &ps->performance_levels[index];
5437
5438         ps->performance_level_count = index + 1;
5439
5440         pl->sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5441         pl->sclk |= clock_info->ci.ucEngineClockHigh << 16;
5442         pl->mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5443         pl->mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5444
5445         pl->pcie_gen = r600_get_pcie_gen_support(rdev,
5446                                                  pi->sys_pcie_mask,
5447                                                  pi->vbios_boot_state.pcie_gen_bootup_value,
5448                                                  clock_info->ci.ucPCIEGen);
5449         pl->pcie_lane = r600_get_pcie_lane_support(rdev,
5450                                                    pi->vbios_boot_state.pcie_lane_bootup_value,
5451                                                    le16_to_cpu(clock_info->ci.usPCIELane));
5452
5453         if (rps->class & ATOM_PPLIB_CLASSIFICATION_ACPI) {
5454                 pi->acpi_pcie_gen = pl->pcie_gen;
5455         }
5456
5457         if (rps->class2 & ATOM_PPLIB_CLASSIFICATION2_ULV) {
5458                 pi->ulv.supported = true;
5459                 pi->ulv.pl = *pl;
5460                 pi->ulv.cg_ulv_parameter = CISLANDS_CGULVPARAMETER_DFLT;
5461         }
5462
5463         /* patch up boot state */
5464         if (rps->class & ATOM_PPLIB_CLASSIFICATION_BOOT) {
5465                 pl->mclk = pi->vbios_boot_state.mclk_bootup_value;
5466                 pl->sclk = pi->vbios_boot_state.sclk_bootup_value;
5467                 pl->pcie_gen = pi->vbios_boot_state.pcie_gen_bootup_value;
5468                 pl->pcie_lane = pi->vbios_boot_state.pcie_lane_bootup_value;
5469         }
5470
5471         switch (rps->class & ATOM_PPLIB_CLASSIFICATION_UI_MASK) {
5472         case ATOM_PPLIB_CLASSIFICATION_UI_BATTERY:
5473                 pi->use_pcie_powersaving_levels = true;
5474                 if (pi->pcie_gen_powersaving.max < pl->pcie_gen)
5475                         pi->pcie_gen_powersaving.max = pl->pcie_gen;
5476                 if (pi->pcie_gen_powersaving.min > pl->pcie_gen)
5477                         pi->pcie_gen_powersaving.min = pl->pcie_gen;
5478                 if (pi->pcie_lane_powersaving.max < pl->pcie_lane)
5479                         pi->pcie_lane_powersaving.max = pl->pcie_lane;
5480                 if (pi->pcie_lane_powersaving.min > pl->pcie_lane)
5481                         pi->pcie_lane_powersaving.min = pl->pcie_lane;
5482                 break;
5483         case ATOM_PPLIB_CLASSIFICATION_UI_PERFORMANCE:
5484                 pi->use_pcie_performance_levels = true;
5485                 if (pi->pcie_gen_performance.max < pl->pcie_gen)
5486                         pi->pcie_gen_performance.max = pl->pcie_gen;
5487                 if (pi->pcie_gen_performance.min > pl->pcie_gen)
5488                         pi->pcie_gen_performance.min = pl->pcie_gen;
5489                 if (pi->pcie_lane_performance.max < pl->pcie_lane)
5490                         pi->pcie_lane_performance.max = pl->pcie_lane;
5491                 if (pi->pcie_lane_performance.min > pl->pcie_lane)
5492                         pi->pcie_lane_performance.min = pl->pcie_lane;
5493                 break;
5494         default:
5495                 break;
5496         }
5497 }
5498
5499 static int ci_parse_power_table(struct radeon_device *rdev)
5500 {
5501         struct radeon_mode_info *mode_info = &rdev->mode_info;
5502         struct _ATOM_PPLIB_NONCLOCK_INFO *non_clock_info;
5503         union pplib_power_state *power_state;
5504         int i, j, k, non_clock_array_index, clock_array_index;
5505         union pplib_clock_info *clock_info;
5506         struct _StateArray *state_array;
5507         struct _ClockInfoArray *clock_info_array;
5508         struct _NonClockInfoArray *non_clock_info_array;
5509         union power_info *power_info;
5510         int index = GetIndexIntoMasterTable(DATA, PowerPlayInfo);
5511         u16 data_offset;
5512         u8 frev, crev;
5513         u8 *power_state_offset;
5514         struct ci_ps *ps;
5515         int ret;
5516
5517         if (!atom_parse_data_header(mode_info->atom_context, index, NULL,
5518                                    &frev, &crev, &data_offset))
5519                 return -EINVAL;
5520         power_info = (union power_info *)(mode_info->atom_context->bios + data_offset);
5521
5522         state_array = (struct _StateArray *)
5523                 (mode_info->atom_context->bios + data_offset +
5524                  le16_to_cpu(power_info->pplib.usStateArrayOffset));
5525         clock_info_array = (struct _ClockInfoArray *)
5526                 (mode_info->atom_context->bios + data_offset +
5527                  le16_to_cpu(power_info->pplib.usClockInfoArrayOffset));
5528         non_clock_info_array = (struct _NonClockInfoArray *)
5529                 (mode_info->atom_context->bios + data_offset +
5530                  le16_to_cpu(power_info->pplib.usNonClockInfoArrayOffset));
5531
5532         rdev->pm.dpm.ps = kcalloc(state_array->ucNumEntries,
5533                                   sizeof(struct radeon_ps),
5534                                   GFP_KERNEL);
5535         if (!rdev->pm.dpm.ps)
5536                 return -ENOMEM;
5537         power_state_offset = (u8 *)state_array->states;
5538         rdev->pm.dpm.num_ps = 0;
5539         for (i = 0; i < state_array->ucNumEntries; i++) {
5540                 u8 *idx;
5541                 power_state = (union pplib_power_state *)power_state_offset;
5542                 non_clock_array_index = power_state->v2.nonClockInfoIndex;
5543                 non_clock_info = (struct _ATOM_PPLIB_NONCLOCK_INFO *)
5544                         &non_clock_info_array->nonClockInfo[non_clock_array_index];
5545                 if (!rdev->pm.power_state[i].clock_info) {
5546                         ret = -EINVAL;
5547                         goto err_free_ps;
5548                 }
5549                 ps = kzalloc(sizeof(struct ci_ps), GFP_KERNEL);
5550                 if (ps == NULL) {
5551                         ret = -ENOMEM;
5552                         goto err_free_ps;
5553                 }
5554                 rdev->pm.dpm.ps[i].ps_priv = ps;
5555                 ci_parse_pplib_non_clock_info(rdev, &rdev->pm.dpm.ps[i],
5556                                               non_clock_info,
5557                                               non_clock_info_array->ucEntrySize);
5558                 k = 0;
5559                 idx = (u8 *)&power_state->v2.clockInfoIndex[0];
5560                 for (j = 0; j < power_state->v2.ucNumDPMLevels; j++) {
5561                         clock_array_index = idx[j];
5562                         if (clock_array_index >= clock_info_array->ucNumEntries)
5563                                 continue;
5564                         if (k >= CISLANDS_MAX_HARDWARE_POWERLEVELS)
5565                                 break;
5566                         clock_info = (union pplib_clock_info *)
5567                                 ((u8 *)&clock_info_array->clockInfo[0] +
5568                                  (clock_array_index * clock_info_array->ucEntrySize));
5569                         ci_parse_pplib_clock_info(rdev,
5570                                                   &rdev->pm.dpm.ps[i], k,
5571                                                   clock_info);
5572                         k++;
5573                 }
5574                 power_state_offset += 2 + power_state->v2.ucNumDPMLevels;
5575                 rdev->pm.dpm.num_ps = i + 1;
5576         }
5577
5578         /* fill in the vce power states */
5579         for (i = 0; i < RADEON_MAX_VCE_LEVELS; i++) {
5580                 u32 sclk, mclk;
5581                 clock_array_index = rdev->pm.dpm.vce_states[i].clk_idx;
5582                 clock_info = (union pplib_clock_info *)
5583                         &clock_info_array->clockInfo[clock_array_index * clock_info_array->ucEntrySize];
5584                 sclk = le16_to_cpu(clock_info->ci.usEngineClockLow);
5585                 sclk |= clock_info->ci.ucEngineClockHigh << 16;
5586                 mclk = le16_to_cpu(clock_info->ci.usMemoryClockLow);
5587                 mclk |= clock_info->ci.ucMemoryClockHigh << 16;
5588                 rdev->pm.dpm.vce_states[i].sclk = sclk;
5589                 rdev->pm.dpm.vce_states[i].mclk = mclk;
5590         }
5591
5592         return 0;
5593
5594 err_free_ps:
5595         for (i = 0; i < rdev->pm.dpm.num_ps; i++)
5596                 kfree(rdev->pm.dpm.ps[i].ps_priv);
5597         kfree(rdev->pm.dpm.ps);
5598         return ret;
5599 }
5600
5601 static int ci_get_vbios_boot_values(struct radeon_device *rdev,
5602                                     struct ci_vbios_boot_state *boot_state)
5603 {
5604         struct radeon_mode_info *mode_info = &rdev->mode_info;
5605         int index = GetIndexIntoMasterTable(DATA, FirmwareInfo);
5606         ATOM_FIRMWARE_INFO_V2_2 *firmware_info;
5607         u8 frev, crev;
5608         u16 data_offset;
5609
5610         if (atom_parse_data_header(mode_info->atom_context, index, NULL,
5611                                    &frev, &crev, &data_offset)) {
5612                 firmware_info =
5613                         (ATOM_FIRMWARE_INFO_V2_2 *)(mode_info->atom_context->bios +
5614                                                     data_offset);
5615                 boot_state->mvdd_bootup_value = le16_to_cpu(firmware_info->usBootUpMVDDCVoltage);
5616                 boot_state->vddc_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCVoltage);
5617                 boot_state->vddci_bootup_value = le16_to_cpu(firmware_info->usBootUpVDDCIVoltage);
5618                 boot_state->pcie_gen_bootup_value = ci_get_current_pcie_speed(rdev);
5619                 boot_state->pcie_lane_bootup_value = ci_get_current_pcie_lane_number(rdev);
5620                 boot_state->sclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultEngineClock);
5621                 boot_state->mclk_bootup_value = le32_to_cpu(firmware_info->ulDefaultMemoryClock);
5622
5623                 return 0;
5624         }
5625         return -EINVAL;
5626 }
5627
5628 void ci_dpm_fini(struct radeon_device *rdev)
5629 {
5630         int i;
5631
5632         for (i = 0; i < rdev->pm.dpm.num_ps; i++) {
5633                 kfree(rdev->pm.dpm.ps[i].ps_priv);
5634         }
5635         kfree(rdev->pm.dpm.ps);
5636         kfree(rdev->pm.dpm.priv);
5637         kfree(rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries);
5638         r600_free_extended_power_table(rdev);
5639 }
5640
5641 int ci_dpm_init(struct radeon_device *rdev)
5642 {
5643         int index = GetIndexIntoMasterTable(DATA, ASIC_InternalSS_Info);
5644         SMU7_Discrete_DpmTable  *dpm_table;
5645         struct radeon_gpio_rec gpio;
5646         u16 data_offset, size;
5647         u8 frev, crev;
5648         struct ci_power_info *pi;
5649         enum pci_bus_speed speed_cap = PCI_SPEED_UNKNOWN;
5650         struct pci_dev *root = rdev->pdev->bus->self;
5651         int ret;
5652
5653         pi = kzalloc(sizeof(struct ci_power_info), GFP_KERNEL);
5654         if (pi == NULL)
5655                 return -ENOMEM;
5656         rdev->pm.dpm.priv = pi;
5657
5658         if (!pci_is_root_bus(rdev->pdev->bus))
5659                 speed_cap = pcie_get_speed_cap(root);
5660         if (speed_cap == PCI_SPEED_UNKNOWN) {
5661                 pi->sys_pcie_mask = 0;
5662         } else {
5663                 if (speed_cap == PCIE_SPEED_8_0GT)
5664                         pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
5665                                 RADEON_PCIE_SPEED_50 |
5666                                 RADEON_PCIE_SPEED_80;
5667                 else if (speed_cap == PCIE_SPEED_5_0GT)
5668                         pi->sys_pcie_mask = RADEON_PCIE_SPEED_25 |
5669                                 RADEON_PCIE_SPEED_50;
5670                 else
5671                         pi->sys_pcie_mask = RADEON_PCIE_SPEED_25;
5672         }
5673         pi->force_pcie_gen = RADEON_PCIE_GEN_INVALID;
5674
5675         pi->pcie_gen_performance.max = RADEON_PCIE_GEN1;
5676         pi->pcie_gen_performance.min = RADEON_PCIE_GEN3;
5677         pi->pcie_gen_powersaving.max = RADEON_PCIE_GEN1;
5678         pi->pcie_gen_powersaving.min = RADEON_PCIE_GEN3;
5679
5680         pi->pcie_lane_performance.max = 0;
5681         pi->pcie_lane_performance.min = 16;
5682         pi->pcie_lane_powersaving.max = 0;
5683         pi->pcie_lane_powersaving.min = 16;
5684
5685         ret = ci_get_vbios_boot_values(rdev, &pi->vbios_boot_state);
5686         if (ret) {
5687                 kfree(rdev->pm.dpm.priv);
5688                 return ret;
5689         }
5690
5691         ret = r600_get_platform_caps(rdev);
5692         if (ret) {
5693                 kfree(rdev->pm.dpm.priv);
5694                 return ret;
5695         }
5696
5697         ret = r600_parse_extended_power_table(rdev);
5698         if (ret) {
5699                 kfree(rdev->pm.dpm.priv);
5700                 return ret;
5701         }
5702
5703         ret = ci_parse_power_table(rdev);
5704         if (ret) {
5705                 kfree(rdev->pm.dpm.priv);
5706                 r600_free_extended_power_table(rdev);
5707                 return ret;
5708         }
5709
5710         pi->dll_default_on = false;
5711         pi->sram_end = SMC_RAM_END;
5712
5713         pi->activity_target[0] = CISLAND_TARGETACTIVITY_DFLT;
5714         pi->activity_target[1] = CISLAND_TARGETACTIVITY_DFLT;
5715         pi->activity_target[2] = CISLAND_TARGETACTIVITY_DFLT;
5716         pi->activity_target[3] = CISLAND_TARGETACTIVITY_DFLT;
5717         pi->activity_target[4] = CISLAND_TARGETACTIVITY_DFLT;
5718         pi->activity_target[5] = CISLAND_TARGETACTIVITY_DFLT;
5719         pi->activity_target[6] = CISLAND_TARGETACTIVITY_DFLT;
5720         pi->activity_target[7] = CISLAND_TARGETACTIVITY_DFLT;
5721
5722         pi->mclk_activity_target = CISLAND_MCLK_TARGETACTIVITY_DFLT;
5723
5724         pi->sclk_dpm_key_disabled = 0;
5725         pi->mclk_dpm_key_disabled = 0;
5726         pi->pcie_dpm_key_disabled = 0;
5727         pi->thermal_sclk_dpm_enabled = 0;
5728
5729         /* mclk dpm is unstable on some R7 260X cards with the old mc ucode */
5730         if ((rdev->pdev->device == 0x6658) &&
5731             (rdev->mc_fw->size == (BONAIRE_MC_UCODE_SIZE * 4))) {
5732                 pi->mclk_dpm_key_disabled = 1;
5733         }
5734
5735         pi->caps_sclk_ds = true;
5736
5737         pi->mclk_strobe_mode_threshold = 40000;
5738         pi->mclk_stutter_mode_threshold = 40000;
5739         pi->mclk_edc_enable_threshold = 40000;
5740         pi->mclk_edc_wr_enable_threshold = 40000;
5741
5742         ci_initialize_powertune_defaults(rdev);
5743
5744         pi->caps_fps = false;
5745
5746         pi->caps_sclk_throttle_low_notification = false;
5747
5748         pi->caps_uvd_dpm = true;
5749         pi->caps_vce_dpm = true;
5750
5751         ci_get_leakage_voltages(rdev);
5752         ci_patch_dependency_tables_with_leakage(rdev);
5753         ci_set_private_data_variables_based_on_pptable(rdev);
5754
5755         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries =
5756                 kcalloc(4,
5757                         sizeof(struct radeon_clock_voltage_dependency_entry),
5758                         GFP_KERNEL);
5759         if (!rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries) {
5760                 ci_dpm_fini(rdev);
5761                 return -ENOMEM;
5762         }
5763         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.count = 4;
5764         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].clk = 0;
5765         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[0].v = 0;
5766         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].clk = 36000;
5767         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[1].v = 720;
5768         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].clk = 54000;
5769         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[2].v = 810;
5770         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].clk = 72000;
5771         rdev->pm.dpm.dyn_state.vddc_dependency_on_dispclk.entries[3].v = 900;
5772
5773         rdev->pm.dpm.dyn_state.mclk_sclk_ratio = 4;
5774         rdev->pm.dpm.dyn_state.sclk_mclk_delta = 15000;
5775         rdev->pm.dpm.dyn_state.vddc_vddci_delta = 200;
5776
5777         rdev->pm.dpm.dyn_state.valid_sclk_values.count = 0;
5778         rdev->pm.dpm.dyn_state.valid_sclk_values.values = NULL;
5779         rdev->pm.dpm.dyn_state.valid_mclk_values.count = 0;
5780         rdev->pm.dpm.dyn_state.valid_mclk_values.values = NULL;
5781
5782         if (rdev->family == CHIP_HAWAII) {
5783                 pi->thermal_temp_setting.temperature_low = 94500;
5784                 pi->thermal_temp_setting.temperature_high = 95000;
5785                 pi->thermal_temp_setting.temperature_shutdown = 104000;
5786         } else {
5787                 pi->thermal_temp_setting.temperature_low = 99500;
5788                 pi->thermal_temp_setting.temperature_high = 100000;
5789                 pi->thermal_temp_setting.temperature_shutdown = 104000;
5790         }
5791
5792         pi->uvd_enabled = false;
5793
5794         dpm_table = &pi->smc_state_table;
5795
5796         gpio = radeon_atombios_lookup_gpio(rdev, VDDC_VRHOT_GPIO_PINID);
5797         if (gpio.valid) {
5798                 dpm_table->VRHotGpio = gpio.shift;
5799                 rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5800         } else {
5801                 dpm_table->VRHotGpio = CISLANDS_UNUSED_GPIO_PIN;
5802                 rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_REGULATOR_HOT;
5803         }
5804
5805         gpio = radeon_atombios_lookup_gpio(rdev, PP_AC_DC_SWITCH_GPIO_PINID);
5806         if (gpio.valid) {
5807                 dpm_table->AcDcGpio = gpio.shift;
5808                 rdev->pm.dpm.platform_caps |= ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5809         } else {
5810                 dpm_table->AcDcGpio = CISLANDS_UNUSED_GPIO_PIN;
5811                 rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_HARDWAREDC;
5812         }
5813
5814         gpio = radeon_atombios_lookup_gpio(rdev, VDDC_PCC_GPIO_PINID);
5815         if (gpio.valid) {
5816                 u32 tmp = RREG32_SMC(CNB_PWRMGT_CNTL);
5817
5818                 switch (gpio.shift) {
5819                 case 0:
5820                         tmp &= ~GNB_SLOW_MODE_MASK;
5821                         tmp |= GNB_SLOW_MODE(1);
5822                         break;
5823                 case 1:
5824                         tmp &= ~GNB_SLOW_MODE_MASK;
5825                         tmp |= GNB_SLOW_MODE(2);
5826                         break;
5827                 case 2:
5828                         tmp |= GNB_SLOW;
5829                         break;
5830                 case 3:
5831                         tmp |= FORCE_NB_PS1;
5832                         break;
5833                 case 4:
5834                         tmp |= DPM_ENABLED;
5835                         break;
5836                 default:
5837                         DRM_DEBUG("Invalid PCC GPIO: %u!\n", gpio.shift);
5838                         break;
5839                 }
5840                 WREG32_SMC(CNB_PWRMGT_CNTL, tmp);
5841         }
5842
5843         pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5844         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5845         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_NONE;
5846         if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_GPIO_LUT))
5847                 pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5848         else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDC, VOLTAGE_OBJ_SVID2))
5849                 pi->voltage_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5850
5851         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL) {
5852                 if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_GPIO_LUT))
5853                         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5854                 else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_VDDCI, VOLTAGE_OBJ_SVID2))
5855                         pi->vddci_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5856                 else
5857                         rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_VDDCI_CONTROL;
5858         }
5859
5860         if (rdev->pm.dpm.platform_caps & ATOM_PP_PLATFORM_CAP_MVDDCONTROL) {
5861                 if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_GPIO_LUT))
5862                         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_GPIO;
5863                 else if (radeon_atom_is_voltage_gpio(rdev, VOLTAGE_TYPE_MVDDC, VOLTAGE_OBJ_SVID2))
5864                         pi->mvdd_control = CISLANDS_VOLTAGE_CONTROL_BY_SVID2;
5865                 else
5866                         rdev->pm.dpm.platform_caps &= ~ATOM_PP_PLATFORM_CAP_MVDDCONTROL;
5867         }
5868
5869         pi->vddc_phase_shed_control = true;
5870
5871 #if defined(CONFIG_ACPI)
5872         pi->pcie_performance_request =
5873                 radeon_acpi_is_pcie_performance_request_supported(rdev);
5874 #else
5875         pi->pcie_performance_request = false;
5876 #endif
5877
5878         if (atom_parse_data_header(rdev->mode_info.atom_context, index, &size,
5879                                    &frev, &crev, &data_offset)) {
5880                 pi->caps_sclk_ss_support = true;
5881                 pi->caps_mclk_ss_support = true;
5882                 pi->dynamic_ss = true;
5883         } else {
5884                 pi->caps_sclk_ss_support = false;
5885                 pi->caps_mclk_ss_support = false;
5886                 pi->dynamic_ss = true;
5887         }
5888
5889         if (rdev->pm.int_thermal_type != THERMAL_TYPE_NONE)
5890                 pi->thermal_protection = true;
5891         else
5892                 pi->thermal_protection = false;
5893
5894         pi->caps_dynamic_ac_timing = true;
5895
5896         pi->uvd_power_gated = false;
5897
5898         /* make sure dc limits are valid */
5899         if ((rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.sclk == 0) ||
5900             (rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc.mclk == 0))
5901                 rdev->pm.dpm.dyn_state.max_clock_voltage_on_dc =
5902                         rdev->pm.dpm.dyn_state.max_clock_voltage_on_ac;
5903
5904         pi->fan_ctrl_is_in_default_mode = true;
5905
5906         return 0;
5907 }
5908
5909 void ci_dpm_debugfs_print_current_performance_level(struct radeon_device *rdev,
5910                                                     struct seq_file *m)
5911 {
5912         struct ci_power_info *pi = ci_get_pi(rdev);
5913         struct radeon_ps *rps = &pi->current_rps;
5914         u32 sclk = ci_get_average_sclk_freq(rdev);
5915         u32 mclk = ci_get_average_mclk_freq(rdev);
5916
5917         seq_printf(m, "uvd    %sabled\n", pi->uvd_enabled ? "en" : "dis");
5918         seq_printf(m, "vce    %sabled\n", rps->vce_active ? "en" : "dis");
5919         seq_printf(m, "power level avg    sclk: %u mclk: %u\n",
5920                    sclk, mclk);
5921 }
5922
5923 void ci_dpm_print_power_state(struct radeon_device *rdev,
5924                               struct radeon_ps *rps)
5925 {
5926         struct ci_ps *ps = ci_get_ps(rps);
5927         struct ci_pl *pl;
5928         int i;
5929
5930         r600_dpm_print_class_info(rps->class, rps->class2);
5931         r600_dpm_print_cap_info(rps->caps);
5932         printk("\tuvd    vclk: %d dclk: %d\n", rps->vclk, rps->dclk);
5933         for (i = 0; i < ps->performance_level_count; i++) {
5934                 pl = &ps->performance_levels[i];
5935                 printk("\t\tpower level %d    sclk: %u mclk: %u pcie gen: %u pcie lanes: %u\n",
5936                        i, pl->sclk, pl->mclk, pl->pcie_gen + 1, pl->pcie_lane);
5937         }
5938         r600_dpm_print_ps_status(rdev, rps);
5939 }
5940
5941 u32 ci_dpm_get_current_sclk(struct radeon_device *rdev)
5942 {
5943         u32 sclk = ci_get_average_sclk_freq(rdev);
5944
5945         return sclk;
5946 }
5947
5948 u32 ci_dpm_get_current_mclk(struct radeon_device *rdev)
5949 {
5950         u32 mclk = ci_get_average_mclk_freq(rdev);
5951
5952         return mclk;
5953 }
5954
5955 u32 ci_dpm_get_sclk(struct radeon_device *rdev, bool low)
5956 {
5957         struct ci_power_info *pi = ci_get_pi(rdev);
5958         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
5959
5960         if (low)
5961                 return requested_state->performance_levels[0].sclk;
5962         else
5963                 return requested_state->performance_levels[requested_state->performance_level_count - 1].sclk;
5964 }
5965
5966 u32 ci_dpm_get_mclk(struct radeon_device *rdev, bool low)
5967 {
5968         struct ci_power_info *pi = ci_get_pi(rdev);
5969         struct ci_ps *requested_state = ci_get_ps(&pi->requested_rps);
5970
5971         if (low)
5972                 return requested_state->performance_levels[0].mclk;
5973         else
5974                 return requested_state->performance_levels[requested_state->performance_level_count - 1].mclk;
5975 }