drivers/gpu/drm/radeon/si.c

   1 /*
   2  * Copyright 2011 Advanced Micro Devices, Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Alex Deucher
  23  */
  24 #include <linux/firmware.h>
  25 #include <linux/platform_device.h>
  26 #include <linux/slab.h>
  27 #include <linux/module.h>
  28 #include <drm/drmP.h>
  29 #include "radeon.h"
  30 #include "radeon_asic.h"
  31 #include <drm/radeon_drm.h>
  32 #include "sid.h"
  33 #include "atom.h"
  34 #include "si_blit_shaders.h"
  35
  36 #define SI_PFP_UCODE_SIZE 2144
  37 #define SI_PM4_UCODE_SIZE 2144
  38 #define SI_CE_UCODE_SIZE 2144
  39 #define SI_RLC_UCODE_SIZE 2048
  40 #define SI_MC_UCODE_SIZE 7769
  41 #define OLAND_MC_UCODE_SIZE 7863
  42
  43 MODULE_FIRMWARE("radeon/TAHITI_pfp.bin");
  44 MODULE_FIRMWARE("radeon/TAHITI_me.bin");
  45 MODULE_FIRMWARE("radeon/TAHITI_ce.bin");
  46 MODULE_FIRMWARE("radeon/TAHITI_mc.bin");
  47 MODULE_FIRMWARE("radeon/TAHITI_rlc.bin");
  48 MODULE_FIRMWARE("radeon/PITCAIRN_pfp.bin");
  49 MODULE_FIRMWARE("radeon/PITCAIRN_me.bin");
  50 MODULE_FIRMWARE("radeon/PITCAIRN_ce.bin");
  51 MODULE_FIRMWARE("radeon/PITCAIRN_mc.bin");
  52 MODULE_FIRMWARE("radeon/PITCAIRN_rlc.bin");
  53 MODULE_FIRMWARE("radeon/VERDE_pfp.bin");
  54 MODULE_FIRMWARE("radeon/VERDE_me.bin");
  55 MODULE_FIRMWARE("radeon/VERDE_ce.bin");
  56 MODULE_FIRMWARE("radeon/VERDE_mc.bin");
  57 MODULE_FIRMWARE("radeon/VERDE_rlc.bin");
  58 MODULE_FIRMWARE("radeon/OLAND_pfp.bin");
  59 MODULE_FIRMWARE("radeon/OLAND_me.bin");
  60 MODULE_FIRMWARE("radeon/OLAND_ce.bin");
  61 MODULE_FIRMWARE("radeon/OLAND_mc.bin");
  62 MODULE_FIRMWARE("radeon/OLAND_rlc.bin");
  63
  64 extern int r600_ih_ring_alloc(struct radeon_device *rdev);
  65 extern void r600_ih_ring_fini(struct radeon_device *rdev);
  66 extern void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev);
  67 extern void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save);
  68 extern void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save);
  69 extern u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev);
  70 extern void evergreen_print_gpu_status_regs(struct radeon_device *rdev);
  71 extern bool evergreen_is_display_hung(struct radeon_device *rdev);
  72
  73 static const u32 tahiti_golden_rlc_registers[] =
  74 {
  75         0xc424, 0xffffffff, 0x00601005,
  76         0xc47c, 0xffffffff, 0x10104040,
  77         0xc488, 0xffffffff, 0x0100000a,
  78         0xc314, 0xffffffff, 0x00000800,
  79         0xc30c, 0xffffffff, 0x800000f4,
  80         0xf4a8, 0xffffffff, 0x00000000
  81 };
  82
  83 static const u32 tahiti_golden_registers[] =
  84 {
  85         0x9a10, 0x00010000, 0x00018208,
  86         0x9830, 0xffffffff, 0x00000000,
  87         0x9834, 0xf00fffff, 0x00000400,
  88         0x9838, 0x0002021c, 0x00020200,
  89         0xc78, 0x00000080, 0x00000000,
  90         0xd030, 0x000300c0, 0x00800040,
  91         0xd830, 0x000300c0, 0x00800040,
  92         0x5bb0, 0x000000f0, 0x00000070,
  93         0x5bc0, 0x00200000, 0x50100000,
  94         0x7030, 0x31000311, 0x00000011,
  95         0x277c, 0x00000003, 0x000007ff,
  96         0x240c, 0x000007ff, 0x00000000,
  97         0x8a14, 0xf000001f, 0x00000007,
  98         0x8b24, 0xffffffff, 0x00ffffff,
  99         0x8b10, 0x0000ff0f, 0x00000000,
 100         0x28a4c, 0x07ffffff, 0x4e000000,
 101         0x28350, 0x3f3f3fff, 0x2a00126a,
 102         0x30, 0x000000ff, 0x0040,
 103         0x34, 0x00000040, 0x00004040,
 104         0x9100, 0x07ffffff, 0x03000000,
 105         0x8e88, 0x01ff1f3f, 0x00000000,
 106         0x8e84, 0x01ff1f3f, 0x00000000,
 107         0x9060, 0x0000007f, 0x00000020,
 108         0x9508, 0x00010000, 0x00010000,
 109         0xac14, 0x00000200, 0x000002fb,
 110         0xac10, 0xffffffff, 0x0000543b,
 111         0xac0c, 0xffffffff, 0xa9210876,
 112         0x88d0, 0xffffffff, 0x000fff40,
 113         0x88d4, 0x0000001f, 0x00000010,
 114         0x1410, 0x20000000, 0x20fffed8,
 115         0x15c0, 0x000c0fc0, 0x000c0400
 116 };
 117
 118 static const u32 tahiti_golden_registers2[] =
 119 {
 120         0xc64, 0x00000001, 0x00000001
 121 };
 122
 123 static const u32 pitcairn_golden_rlc_registers[] =
 124 {
 125         0xc424, 0xffffffff, 0x00601004,
 126         0xc47c, 0xffffffff, 0x10102020,
 127         0xc488, 0xffffffff, 0x01000020,
 128         0xc314, 0xffffffff, 0x00000800,
 129         0xc30c, 0xffffffff, 0x800000a4
 130 };
 131
 132 static const u32 pitcairn_golden_registers[] =
 133 {
 134         0x9a10, 0x00010000, 0x00018208,
 135         0x9830, 0xffffffff, 0x00000000,
 136         0x9834, 0xf00fffff, 0x00000400,
 137         0x9838, 0x0002021c, 0x00020200,
 138         0xc78, 0x00000080, 0x00000000,
 139         0xd030, 0x000300c0, 0x00800040,
 140         0xd830, 0x000300c0, 0x00800040,
 141         0x5bb0, 0x000000f0, 0x00000070,
 142         0x5bc0, 0x00200000, 0x50100000,
 143         0x7030, 0x31000311, 0x00000011,
 144         0x2ae4, 0x00073ffe, 0x000022a2,
 145         0x240c, 0x000007ff, 0x00000000,
 146         0x8a14, 0xf000001f, 0x00000007,
 147         0x8b24, 0xffffffff, 0x00ffffff,
 148         0x8b10, 0x0000ff0f, 0x00000000,
 149         0x28a4c, 0x07ffffff, 0x4e000000,
 150         0x28350, 0x3f3f3fff, 0x2a00126a,
 151         0x30, 0x000000ff, 0x0040,
 152         0x34, 0x00000040, 0x00004040,
 153         0x9100, 0x07ffffff, 0x03000000,
 154         0x9060, 0x0000007f, 0x00000020,
 155         0x9508, 0x00010000, 0x00010000,
 156         0xac14, 0x000003ff, 0x000000f7,
 157         0xac10, 0xffffffff, 0x00000000,
 158         0xac0c, 0xffffffff, 0x32761054,
 159         0x88d4, 0x0000001f, 0x00000010,
 160         0x15c0, 0x000c0fc0, 0x000c0400
 161 };
 162
 163 static const u32 verde_golden_rlc_registers[] =
 164 {
 165         0xc424, 0xffffffff, 0x033f1005,
 166         0xc47c, 0xffffffff, 0x10808020,
 167         0xc488, 0xffffffff, 0x00800008,
 168         0xc314, 0xffffffff, 0x00001000,
 169         0xc30c, 0xffffffff, 0x80010014
 170 };
 171
 172 static const u32 verde_golden_registers[] =
 173 {
 174         0x9a10, 0x00010000, 0x00018208,
 175         0x9830, 0xffffffff, 0x00000000,
 176         0x9834, 0xf00fffff, 0x00000400,
 177         0x9838, 0x0002021c, 0x00020200,
 178         0xc78, 0x00000080, 0x00000000,
 179         0xd030, 0x000300c0, 0x00800040,
 180         0xd030, 0x000300c0, 0x00800040,
 181         0xd830, 0x000300c0, 0x00800040,
 182         0xd830, 0x000300c0, 0x00800040,
 183         0x5bb0, 0x000000f0, 0x00000070,
 184         0x5bc0, 0x00200000, 0x50100000,
 185         0x7030, 0x31000311, 0x00000011,
 186         0x2ae4, 0x00073ffe, 0x000022a2,
 187         0x2ae4, 0x00073ffe, 0x000022a2,
 188         0x2ae4, 0x00073ffe, 0x000022a2,
 189         0x240c, 0x000007ff, 0x00000000,
 190         0x240c, 0x000007ff, 0x00000000,
 191         0x240c, 0x000007ff, 0x00000000,
 192         0x8a14, 0xf000001f, 0x00000007,
 193         0x8a14, 0xf000001f, 0x00000007,
 194         0x8a14, 0xf000001f, 0x00000007,
 195         0x8b24, 0xffffffff, 0x00ffffff,
 196         0x8b10, 0x0000ff0f, 0x00000000,
 197         0x28a4c, 0x07ffffff, 0x4e000000,
 198         0x28350, 0x3f3f3fff, 0x0000124a,
 199         0x28350, 0x3f3f3fff, 0x0000124a,
 200         0x28350, 0x3f3f3fff, 0x0000124a,
 201         0x30, 0x000000ff, 0x0040,
 202         0x34, 0x00000040, 0x00004040,
 203         0x9100, 0x07ffffff, 0x03000000,
 204         0x9100, 0x07ffffff, 0x03000000,
 205         0x8e88, 0x01ff1f3f, 0x00000000,
 206         0x8e88, 0x01ff1f3f, 0x00000000,
 207         0x8e88, 0x01ff1f3f, 0x00000000,
 208         0x8e84, 0x01ff1f3f, 0x00000000,
 209         0x8e84, 0x01ff1f3f, 0x00000000,
 210         0x8e84, 0x01ff1f3f, 0x00000000,
 211         0x9060, 0x0000007f, 0x00000020,
 212         0x9508, 0x00010000, 0x00010000,
 213         0xac14, 0x000003ff, 0x00000003,
 214         0xac14, 0x000003ff, 0x00000003,
 215         0xac14, 0x000003ff, 0x00000003,
 216         0xac10, 0xffffffff, 0x00000000,
 217         0xac10, 0xffffffff, 0x00000000,
 218         0xac10, 0xffffffff, 0x00000000,
 219         0xac0c, 0xffffffff, 0x00001032,
 220         0xac0c, 0xffffffff, 0x00001032,
 221         0xac0c, 0xffffffff, 0x00001032,
 222         0x88d4, 0x0000001f, 0x00000010,
 223         0x88d4, 0x0000001f, 0x00000010,
 224         0x88d4, 0x0000001f, 0x00000010,
 225         0x15c0, 0x000c0fc0, 0x000c0400
 226 };
 227
 228 static const u32 oland_golden_rlc_registers[] =
 229 {
 230         0xc424, 0xffffffff, 0x00601005,
 231         0xc47c, 0xffffffff, 0x10104040,
 232         0xc488, 0xffffffff, 0x0100000a,
 233         0xc314, 0xffffffff, 0x00000800,
 234         0xc30c, 0xffffffff, 0x800000f4
 235 };
 236
 237 static const u32 oland_golden_registers[] =
 238 {
 239         0x9a10, 0x00010000, 0x00018208,
 240         0x9830, 0xffffffff, 0x00000000,
 241         0x9834, 0xf00fffff, 0x00000400,
 242         0x9838, 0x0002021c, 0x00020200,
 243         0xc78, 0x00000080, 0x00000000,
 244         0xd030, 0x000300c0, 0x00800040,
 245         0xd830, 0x000300c0, 0x00800040,
 246         0x5bb0, 0x000000f0, 0x00000070,
 247         0x5bc0, 0x00200000, 0x50100000,
 248         0x7030, 0x31000311, 0x00000011,
 249         0x2ae4, 0x00073ffe, 0x000022a2,
 250         0x240c, 0x000007ff, 0x00000000,
 251         0x8a14, 0xf000001f, 0x00000007,
 252         0x8b24, 0xffffffff, 0x00ffffff,
 253         0x8b10, 0x0000ff0f, 0x00000000,
 254         0x28a4c, 0x07ffffff, 0x4e000000,
 255         0x28350, 0x3f3f3fff, 0x00000082,
 256         0x30, 0x000000ff, 0x0040,
 257         0x34, 0x00000040, 0x00004040,
 258         0x9100, 0x07ffffff, 0x03000000,
 259         0x9060, 0x0000007f, 0x00000020,
 260         0x9508, 0x00010000, 0x00010000,
 261         0xac14, 0x000003ff, 0x000000f3,
 262         0xac10, 0xffffffff, 0x00000000,
 263         0xac0c, 0xffffffff, 0x00003210,
 264         0x88d4, 0x0000001f, 0x00000010,
 265         0x15c0, 0x000c0fc0, 0x000c0400
 266 };
 267
 268 static const u32 tahiti_mgcg_cgcg_init[] =
 269 {
 270         0xc400, 0xffffffff, 0xfffffffc,
 271         0x802c, 0xffffffff, 0xe0000000,
 272         0x9a60, 0xffffffff, 0x00000100,
 273         0x92a4, 0xffffffff, 0x00000100,
 274         0xc164, 0xffffffff, 0x00000100,
 275         0x9774, 0xffffffff, 0x00000100,
 276         0x8984, 0xffffffff, 0x06000100,
 277         0x8a18, 0xffffffff, 0x00000100,
 278         0x92a0, 0xffffffff, 0x00000100,
 279         0xc380, 0xffffffff, 0x00000100,
 280         0x8b28, 0xffffffff, 0x00000100,
 281         0x9144, 0xffffffff, 0x00000100,
 282         0x8d88, 0xffffffff, 0x00000100,
 283         0x8d8c, 0xffffffff, 0x00000100,
 284         0x9030, 0xffffffff, 0x00000100,
 285         0x9034, 0xffffffff, 0x00000100,
 286         0x9038, 0xffffffff, 0x00000100,
 287         0x903c, 0xffffffff, 0x00000100,
 288         0xad80, 0xffffffff, 0x00000100,
 289         0xac54, 0xffffffff, 0x00000100,
 290         0x897c, 0xffffffff, 0x06000100,
 291         0x9868, 0xffffffff, 0x00000100,
 292         0x9510, 0xffffffff, 0x00000100,
 293         0xaf04, 0xffffffff, 0x00000100,
 294         0xae04, 0xffffffff, 0x00000100,
 295         0x949c, 0xffffffff, 0x00000100,
 296         0x802c, 0xffffffff, 0xe0000000,
 297         0x9160, 0xffffffff, 0x00010000,
 298         0x9164, 0xffffffff, 0x00030002,
 299         0x9168, 0xffffffff, 0x00040007,
 300         0x916c, 0xffffffff, 0x00060005,
 301         0x9170, 0xffffffff, 0x00090008,
 302         0x9174, 0xffffffff, 0x00020001,
 303         0x9178, 0xffffffff, 0x00040003,
 304         0x917c, 0xffffffff, 0x00000007,
 305         0x9180, 0xffffffff, 0x00060005,
 306         0x9184, 0xffffffff, 0x00090008,
 307         0x9188, 0xffffffff, 0x00030002,
 308         0x918c, 0xffffffff, 0x00050004,
 309         0x9190, 0xffffffff, 0x00000008,
 310         0x9194, 0xffffffff, 0x00070006,
 311         0x9198, 0xffffffff, 0x000a0009,
 312         0x919c, 0xffffffff, 0x00040003,
 313         0x91a0, 0xffffffff, 0x00060005,
 314         0x91a4, 0xffffffff, 0x00000009,
 315         0x91a8, 0xffffffff, 0x00080007,
 316         0x91ac, 0xffffffff, 0x000b000a,
 317         0x91b0, 0xffffffff, 0x00050004,
 318         0x91b4, 0xffffffff, 0x00070006,
 319         0x91b8, 0xffffffff, 0x0008000b,
 320         0x91bc, 0xffffffff, 0x000a0009,
 321         0x91c0, 0xffffffff, 0x000d000c,
 322         0x91c4, 0xffffffff, 0x00060005,
 323         0x91c8, 0xffffffff, 0x00080007,
 324         0x91cc, 0xffffffff, 0x0000000b,
 325         0x91d0, 0xffffffff, 0x000a0009,
 326         0x91d4, 0xffffffff, 0x000d000c,
 327         0x91d8, 0xffffffff, 0x00070006,
 328         0x91dc, 0xffffffff, 0x00090008,
 329         0x91e0, 0xffffffff, 0x0000000c,
 330         0x91e4, 0xffffffff, 0x000b000a,
 331         0x91e8, 0xffffffff, 0x000e000d,
 332         0x91ec, 0xffffffff, 0x00080007,
 333         0x91f0, 0xffffffff, 0x000a0009,
 334         0x91f4, 0xffffffff, 0x0000000d,
 335         0x91f8, 0xffffffff, 0x000c000b,
 336         0x91fc, 0xffffffff, 0x000f000e,
 337         0x9200, 0xffffffff, 0x00090008,
 338         0x9204, 0xffffffff, 0x000b000a,
 339         0x9208, 0xffffffff, 0x000c000f,
 340         0x920c, 0xffffffff, 0x000e000d,
 341         0x9210, 0xffffffff, 0x00110010,
 342         0x9214, 0xffffffff, 0x000a0009,
 343         0x9218, 0xffffffff, 0x000c000b,
 344         0x921c, 0xffffffff, 0x0000000f,
 345         0x9220, 0xffffffff, 0x000e000d,
 346         0x9224, 0xffffffff, 0x00110010,
 347         0x9228, 0xffffffff, 0x000b000a,
 348         0x922c, 0xffffffff, 0x000d000c,
 349         0x9230, 0xffffffff, 0x00000010,
 350         0x9234, 0xffffffff, 0x000f000e,
 351         0x9238, 0xffffffff, 0x00120011,
 352         0x923c, 0xffffffff, 0x000c000b,
 353         0x9240, 0xffffffff, 0x000e000d,
 354         0x9244, 0xffffffff, 0x00000011,
 355         0x9248, 0xffffffff, 0x0010000f,
 356         0x924c, 0xffffffff, 0x00130012,
 357         0x9250, 0xffffffff, 0x000d000c,
 358         0x9254, 0xffffffff, 0x000f000e,
 359         0x9258, 0xffffffff, 0x00100013,
 360         0x925c, 0xffffffff, 0x00120011,
 361         0x9260, 0xffffffff, 0x00150014,
 362         0x9264, 0xffffffff, 0x000e000d,
 363         0x9268, 0xffffffff, 0x0010000f,
 364         0x926c, 0xffffffff, 0x00000013,
 365         0x9270, 0xffffffff, 0x00120011,
 366         0x9274, 0xffffffff, 0x00150014,
 367         0x9278, 0xffffffff, 0x000f000e,
 368         0x927c, 0xffffffff, 0x00110010,
 369         0x9280, 0xffffffff, 0x00000014,
 370         0x9284, 0xffffffff, 0x00130012,
 371         0x9288, 0xffffffff, 0x00160015,
 372         0x928c, 0xffffffff, 0x0010000f,
 373         0x9290, 0xffffffff, 0x00120011,
 374         0x9294, 0xffffffff, 0x00000015,
 375         0x9298, 0xffffffff, 0x00140013,
 376         0x929c, 0xffffffff, 0x00170016,
 377         0x9150, 0xffffffff, 0x96940200,
 378         0x8708, 0xffffffff, 0x00900100,
 379         0xc478, 0xffffffff, 0x00000080,
 380         0xc404, 0xffffffff, 0x0020003f,
 381         0x30, 0xffffffff, 0x0000001c,
 382         0x34, 0x000f0000, 0x000f0000,
 383         0x160c, 0xffffffff, 0x00000100,
 384         0x1024, 0xffffffff, 0x00000100,
 385         0x102c, 0x00000101, 0x00000000,
 386         0x20a8, 0xffffffff, 0x00000104,
 387         0x264c, 0x000c0000, 0x000c0000,
 388         0x2648, 0x000c0000, 0x000c0000,
 389         0x55e4, 0xff000fff, 0x00000100,
 390         0x55e8, 0x00000001, 0x00000001,
 391         0x2f50, 0x00000001, 0x00000001,
 392         0x30cc, 0xc0000fff, 0x00000104,
 393         0xc1e4, 0x00000001, 0x00000001,
 394         0xd0c0, 0xfffffff0, 0x00000100,
 395         0xd8c0, 0xfffffff0, 0x00000100
 396 };
 397
 398 static const u32 pitcairn_mgcg_cgcg_init[] =
 399 {
 400         0xc400, 0xffffffff, 0xfffffffc,
 401         0x802c, 0xffffffff, 0xe0000000,
 402         0x9a60, 0xffffffff, 0x00000100,
 403         0x92a4, 0xffffffff, 0x00000100,
 404         0xc164, 0xffffffff, 0x00000100,
 405         0x9774, 0xffffffff, 0x00000100,
 406         0x8984, 0xffffffff, 0x06000100,
 407         0x8a18, 0xffffffff, 0x00000100,
 408         0x92a0, 0xffffffff, 0x00000100,
 409         0xc380, 0xffffffff, 0x00000100,
 410         0x8b28, 0xffffffff, 0x00000100,
 411         0x9144, 0xffffffff, 0x00000100,
 412         0x8d88, 0xffffffff, 0x00000100,
 413         0x8d8c, 0xffffffff, 0x00000100,
 414         0x9030, 0xffffffff, 0x00000100,
 415         0x9034, 0xffffffff, 0x00000100,
 416         0x9038, 0xffffffff, 0x00000100,
 417         0x903c, 0xffffffff, 0x00000100,
 418         0xad80, 0xffffffff, 0x00000100,
 419         0xac54, 0xffffffff, 0x00000100,
 420         0x897c, 0xffffffff, 0x06000100,
 421         0x9868, 0xffffffff, 0x00000100,
 422         0x9510, 0xffffffff, 0x00000100,
 423         0xaf04, 0xffffffff, 0x00000100,
 424         0xae04, 0xffffffff, 0x00000100,
 425         0x949c, 0xffffffff, 0x00000100,
 426         0x802c, 0xffffffff, 0xe0000000,
 427         0x9160, 0xffffffff, 0x00010000,
 428         0x9164, 0xffffffff, 0x00030002,
 429         0x9168, 0xffffffff, 0x00040007,
 430         0x916c, 0xffffffff, 0x00060005,
 431         0x9170, 0xffffffff, 0x00090008,
 432         0x9174, 0xffffffff, 0x00020001,
 433         0x9178, 0xffffffff, 0x00040003,
 434         0x917c, 0xffffffff, 0x00000007,
 435         0x9180, 0xffffffff, 0x00060005,
 436         0x9184, 0xffffffff, 0x00090008,
 437         0x9188, 0xffffffff, 0x00030002,
 438         0x918c, 0xffffffff, 0x00050004,
 439         0x9190, 0xffffffff, 0x00000008,
 440         0x9194, 0xffffffff, 0x00070006,
 441         0x9198, 0xffffffff, 0x000a0009,
 442         0x919c, 0xffffffff, 0x00040003,
 443         0x91a0, 0xffffffff, 0x00060005,
 444         0x91a4, 0xffffffff, 0x00000009,
 445         0x91a8, 0xffffffff, 0x00080007,
 446         0x91ac, 0xffffffff, 0x000b000a,
 447         0x91b0, 0xffffffff, 0x00050004,
 448         0x91b4, 0xffffffff, 0x00070006,
 449         0x91b8, 0xffffffff, 0x0008000b,
 450         0x91bc, 0xffffffff, 0x000a0009,
 451         0x91c0, 0xffffffff, 0x000d000c,
 452         0x9200, 0xffffffff, 0x00090008,
 453         0x9204, 0xffffffff, 0x000b000a,
 454         0x9208, 0xffffffff, 0x000c000f,
 455         0x920c, 0xffffffff, 0x000e000d,
 456         0x9210, 0xffffffff, 0x00110010,
 457         0x9214, 0xffffffff, 0x000a0009,
 458         0x9218, 0xffffffff, 0x000c000b,
 459         0x921c, 0xffffffff, 0x0000000f,
 460         0x9220, 0xffffffff, 0x000e000d,
 461         0x9224, 0xffffffff, 0x00110010,
 462         0x9228, 0xffffffff, 0x000b000a,
 463         0x922c, 0xffffffff, 0x000d000c,
 464         0x9230, 0xffffffff, 0x00000010,
 465         0x9234, 0xffffffff, 0x000f000e,
 466         0x9238, 0xffffffff, 0x00120011,
 467         0x923c, 0xffffffff, 0x000c000b,
 468         0x9240, 0xffffffff, 0x000e000d,
 469         0x9244, 0xffffffff, 0x00000011,
 470         0x9248, 0xffffffff, 0x0010000f,
 471         0x924c, 0xffffffff, 0x00130012,
 472         0x9250, 0xffffffff, 0x000d000c,
 473         0x9254, 0xffffffff, 0x000f000e,
 474         0x9258, 0xffffffff, 0x00100013,
 475         0x925c, 0xffffffff, 0x00120011,
 476         0x9260, 0xffffffff, 0x00150014,
 477         0x9150, 0xffffffff, 0x96940200,
 478         0x8708, 0xffffffff, 0x00900100,
 479         0xc478, 0xffffffff, 0x00000080,
 480         0xc404, 0xffffffff, 0x0020003f,
 481         0x30, 0xffffffff, 0x0000001c,
 482         0x34, 0x000f0000, 0x000f0000,
 483         0x160c, 0xffffffff, 0x00000100,
 484         0x1024, 0xffffffff, 0x00000100,
 485         0x102c, 0x00000101, 0x00000000,
 486         0x20a8, 0xffffffff, 0x00000104,
 487         0x55e4, 0xff000fff, 0x00000100,
 488         0x55e8, 0x00000001, 0x00000001,
 489         0x2f50, 0x00000001, 0x00000001,
 490         0x30cc, 0xc0000fff, 0x00000104,
 491         0xc1e4, 0x00000001, 0x00000001,
 492         0xd0c0, 0xfffffff0, 0x00000100,
 493         0xd8c0, 0xfffffff0, 0x00000100
 494 };
 495
 496 static const u32 verde_mgcg_cgcg_init[] =
 497 {
 498         0xc400, 0xffffffff, 0xfffffffc,
 499         0x802c, 0xffffffff, 0xe0000000,
 500         0x9a60, 0xffffffff, 0x00000100,
 501         0x92a4, 0xffffffff, 0x00000100,
 502         0xc164, 0xffffffff, 0x00000100,
 503         0x9774, 0xffffffff, 0x00000100,
 504         0x8984, 0xffffffff, 0x06000100,
 505         0x8a18, 0xffffffff, 0x00000100,
 506         0x92a0, 0xffffffff, 0x00000100,
 507         0xc380, 0xffffffff, 0x00000100,
 508         0x8b28, 0xffffffff, 0x00000100,
 509         0x9144, 0xffffffff, 0x00000100,
 510         0x8d88, 0xffffffff, 0x00000100,
 511         0x8d8c, 0xffffffff, 0x00000100,
 512         0x9030, 0xffffffff, 0x00000100,
 513         0x9034, 0xffffffff, 0x00000100,
 514         0x9038, 0xffffffff, 0x00000100,
 515         0x903c, 0xffffffff, 0x00000100,
 516         0xad80, 0xffffffff, 0x00000100,
 517         0xac54, 0xffffffff, 0x00000100,
 518         0x897c, 0xffffffff, 0x06000100,
 519         0x9868, 0xffffffff, 0x00000100,
 520         0x9510, 0xffffffff, 0x00000100,
 521         0xaf04, 0xffffffff, 0x00000100,
 522         0xae04, 0xffffffff, 0x00000100,
 523         0x949c, 0xffffffff, 0x00000100,
 524         0x802c, 0xffffffff, 0xe0000000,
 525         0x9160, 0xffffffff, 0x00010000,
 526         0x9164, 0xffffffff, 0x00030002,
 527         0x9168, 0xffffffff, 0x00040007,
 528         0x916c, 0xffffffff, 0x00060005,
 529         0x9170, 0xffffffff, 0x00090008,
 530         0x9174, 0xffffffff, 0x00020001,
 531         0x9178, 0xffffffff, 0x00040003,
 532         0x917c, 0xffffffff, 0x00000007,
 533         0x9180, 0xffffffff, 0x00060005,
 534         0x9184, 0xffffffff, 0x00090008,
 535         0x9188, 0xffffffff, 0x00030002,
 536         0x918c, 0xffffffff, 0x00050004,
 537         0x9190, 0xffffffff, 0x00000008,
 538         0x9194, 0xffffffff, 0x00070006,
 539         0x9198, 0xffffffff, 0x000a0009,
 540         0x919c, 0xffffffff, 0x00040003,
 541         0x91a0, 0xffffffff, 0x00060005,
 542         0x91a4, 0xffffffff, 0x00000009,
 543         0x91a8, 0xffffffff, 0x00080007,
 544         0x91ac, 0xffffffff, 0x000b000a,
 545         0x91b0, 0xffffffff, 0x00050004,
 546         0x91b4, 0xffffffff, 0x00070006,
 547         0x91b8, 0xffffffff, 0x0008000b,
 548         0x91bc, 0xffffffff, 0x000a0009,
 549         0x91c0, 0xffffffff, 0x000d000c,
 550         0x9200, 0xffffffff, 0x00090008,
 551         0x9204, 0xffffffff, 0x000b000a,
 552         0x9208, 0xffffffff, 0x000c000f,
 553         0x920c, 0xffffffff, 0x000e000d,
 554         0x9210, 0xffffffff, 0x00110010,
 555         0x9214, 0xffffffff, 0x000a0009,
 556         0x9218, 0xffffffff, 0x000c000b,
 557         0x921c, 0xffffffff, 0x0000000f,
 558         0x9220, 0xffffffff, 0x000e000d,
 559         0x9224, 0xffffffff, 0x00110010,
 560         0x9228, 0xffffffff, 0x000b000a,
 561         0x922c, 0xffffffff, 0x000d000c,
 562         0x9230, 0xffffffff, 0x00000010,
 563         0x9234, 0xffffffff, 0x000f000e,
 564         0x9238, 0xffffffff, 0x00120011,
 565         0x923c, 0xffffffff, 0x000c000b,
 566         0x9240, 0xffffffff, 0x000e000d,
 567         0x9244, 0xffffffff, 0x00000011,
 568         0x9248, 0xffffffff, 0x0010000f,
 569         0x924c, 0xffffffff, 0x00130012,
 570         0x9250, 0xffffffff, 0x000d000c,
 571         0x9254, 0xffffffff, 0x000f000e,
 572         0x9258, 0xffffffff, 0x00100013,
 573         0x925c, 0xffffffff, 0x00120011,
 574         0x9260, 0xffffffff, 0x00150014,
 575         0x9150, 0xffffffff, 0x96940200,
 576         0x8708, 0xffffffff, 0x00900100,
 577         0xc478, 0xffffffff, 0x00000080,
 578         0xc404, 0xffffffff, 0x0020003f,
 579         0x30, 0xffffffff, 0x0000001c,
 580         0x34, 0x000f0000, 0x000f0000,
 581         0x160c, 0xffffffff, 0x00000100,
 582         0x1024, 0xffffffff, 0x00000100,
 583         0x102c, 0x00000101, 0x00000000,
 584         0x20a8, 0xffffffff, 0x00000104,
 585         0x264c, 0x000c0000, 0x000c0000,
 586         0x2648, 0x000c0000, 0x000c0000,
 587         0x55e4, 0xff000fff, 0x00000100,
 588         0x55e8, 0x00000001, 0x00000001,
 589         0x2f50, 0x00000001, 0x00000001,
 590         0x30cc, 0xc0000fff, 0x00000104,
 591         0xc1e4, 0x00000001, 0x00000001,
 592         0xd0c0, 0xfffffff0, 0x00000100,
 593         0xd8c0, 0xfffffff0, 0x00000100
 594 };
 595
 596 static const u32 oland_mgcg_cgcg_init[] =
 597 {
 598         0xc400, 0xffffffff, 0xfffffffc,
 599         0x802c, 0xffffffff, 0xe0000000,
 600         0x9a60, 0xffffffff, 0x00000100,
 601         0x92a4, 0xffffffff, 0x00000100,
 602         0xc164, 0xffffffff, 0x00000100,
 603         0x9774, 0xffffffff, 0x00000100,
 604         0x8984, 0xffffffff, 0x06000100,
 605         0x8a18, 0xffffffff, 0x00000100,
 606         0x92a0, 0xffffffff, 0x00000100,
 607         0xc380, 0xffffffff, 0x00000100,
 608         0x8b28, 0xffffffff, 0x00000100,
 609         0x9144, 0xffffffff, 0x00000100,
 610         0x8d88, 0xffffffff, 0x00000100,
 611         0x8d8c, 0xffffffff, 0x00000100,
 612         0x9030, 0xffffffff, 0x00000100,
 613         0x9034, 0xffffffff, 0x00000100,
 614         0x9038, 0xffffffff, 0x00000100,
 615         0x903c, 0xffffffff, 0x00000100,
 616         0xad80, 0xffffffff, 0x00000100,
 617         0xac54, 0xffffffff, 0x00000100,
 618         0x897c, 0xffffffff, 0x06000100,
 619         0x9868, 0xffffffff, 0x00000100,
 620         0x9510, 0xffffffff, 0x00000100,
 621         0xaf04, 0xffffffff, 0x00000100,
 622         0xae04, 0xffffffff, 0x00000100,
 623         0x949c, 0xffffffff, 0x00000100,
 624         0x802c, 0xffffffff, 0xe0000000,
 625         0x9160, 0xffffffff, 0x00010000,
 626         0x9164, 0xffffffff, 0x00030002,
 627         0x9168, 0xffffffff, 0x00040007,
 628         0x916c, 0xffffffff, 0x00060005,
 629         0x9170, 0xffffffff, 0x00090008,
 630         0x9174, 0xffffffff, 0x00020001,
 631         0x9178, 0xffffffff, 0x00040003,
 632         0x917c, 0xffffffff, 0x00000007,
 633         0x9180, 0xffffffff, 0x00060005,
 634         0x9184, 0xffffffff, 0x00090008,
 635         0x9188, 0xffffffff, 0x00030002,
 636         0x918c, 0xffffffff, 0x00050004,
 637         0x9190, 0xffffffff, 0x00000008,
 638         0x9194, 0xffffffff, 0x00070006,
 639         0x9198, 0xffffffff, 0x000a0009,
 640         0x919c, 0xffffffff, 0x00040003,
 641         0x91a0, 0xffffffff, 0x00060005,
 642         0x91a4, 0xffffffff, 0x00000009,
 643         0x91a8, 0xffffffff, 0x00080007,
 644         0x91ac, 0xffffffff, 0x000b000a,
 645         0x91b0, 0xffffffff, 0x00050004,
 646         0x91b4, 0xffffffff, 0x00070006,
 647         0x91b8, 0xffffffff, 0x0008000b,
 648         0x91bc, 0xffffffff, 0x000a0009,
 649         0x91c0, 0xffffffff, 0x000d000c,
 650         0x91c4, 0xffffffff, 0x00060005,
 651         0x91c8, 0xffffffff, 0x00080007,
 652         0x91cc, 0xffffffff, 0x0000000b,
 653         0x91d0, 0xffffffff, 0x000a0009,
 654         0x91d4, 0xffffffff, 0x000d000c,
 655         0x9150, 0xffffffff, 0x96940200,
 656         0x8708, 0xffffffff, 0x00900100,
 657         0xc478, 0xffffffff, 0x00000080,
 658         0xc404, 0xffffffff, 0x0020003f,
 659         0x30, 0xffffffff, 0x0000001c,
 660         0x34, 0x000f0000, 0x000f0000,
 661         0x160c, 0xffffffff, 0x00000100,
 662         0x1024, 0xffffffff, 0x00000100,
 663         0x102c, 0x00000101, 0x00000000,
 664         0x20a8, 0xffffffff, 0x00000104,
 665         0x264c, 0x000c0000, 0x000c0000,
 666         0x2648, 0x000c0000, 0x000c0000,
 667         0x55e4, 0xff000fff, 0x00000100,
 668         0x55e8, 0x00000001, 0x00000001,
 669         0x2f50, 0x00000001, 0x00000001,
 670         0x30cc, 0xc0000fff, 0x00000104,
 671         0xc1e4, 0x00000001, 0x00000001,
 672         0xd0c0, 0xfffffff0, 0x00000100,
 673         0xd8c0, 0xfffffff0, 0x00000100
 674 };
 675
 676 static u32 verde_pg_init[] =
 677 {
 678         0x353c, 0xffffffff, 0x40000,
 679         0x3538, 0xffffffff, 0x200010ff,
 680         0x353c, 0xffffffff, 0x0,
 681         0x353c, 0xffffffff, 0x0,
 682         0x353c, 0xffffffff, 0x0,
 683         0x353c, 0xffffffff, 0x0,
 684         0x353c, 0xffffffff, 0x0,
 685         0x353c, 0xffffffff, 0x7007,
 686         0x3538, 0xffffffff, 0x300010ff,
 687         0x353c, 0xffffffff, 0x0,
 688         0x353c, 0xffffffff, 0x0,
 689         0x353c, 0xffffffff, 0x0,
 690         0x353c, 0xffffffff, 0x0,
 691         0x353c, 0xffffffff, 0x0,
 692         0x353c, 0xffffffff, 0x400000,
 693         0x3538, 0xffffffff, 0x100010ff,
 694         0x353c, 0xffffffff, 0x0,
 695         0x353c, 0xffffffff, 0x0,
 696         0x353c, 0xffffffff, 0x0,
 697         0x353c, 0xffffffff, 0x0,
 698         0x353c, 0xffffffff, 0x0,
 699         0x353c, 0xffffffff, 0x120200,
 700         0x3538, 0xffffffff, 0x500010ff,
 701         0x353c, 0xffffffff, 0x0,
 702         0x353c, 0xffffffff, 0x0,
 703         0x353c, 0xffffffff, 0x0,
 704         0x353c, 0xffffffff, 0x0,
 705         0x353c, 0xffffffff, 0x0,
 706         0x353c, 0xffffffff, 0x1e1e16,
 707         0x3538, 0xffffffff, 0x600010ff,
 708         0x353c, 0xffffffff, 0x0,
 709         0x353c, 0xffffffff, 0x0,
 710         0x353c, 0xffffffff, 0x0,
 711         0x353c, 0xffffffff, 0x0,
 712         0x353c, 0xffffffff, 0x0,
 713         0x353c, 0xffffffff, 0x171f1e,
 714         0x3538, 0xffffffff, 0x700010ff,
 715         0x353c, 0xffffffff, 0x0,
 716         0x353c, 0xffffffff, 0x0,
 717         0x353c, 0xffffffff, 0x0,
 718         0x353c, 0xffffffff, 0x0,
 719         0x353c, 0xffffffff, 0x0,
 720         0x353c, 0xffffffff, 0x0,
 721         0x3538, 0xffffffff, 0x9ff,
 722         0x3500, 0xffffffff, 0x0,
 723         0x3504, 0xffffffff, 0x10000800,
 724         0x3504, 0xffffffff, 0xf,
 725         0x3504, 0xffffffff, 0xf,
 726         0x3500, 0xffffffff, 0x4,
 727         0x3504, 0xffffffff, 0x1000051e,
 728         0x3504, 0xffffffff, 0xffff,
 729         0x3504, 0xffffffff, 0xffff,
 730         0x3500, 0xffffffff, 0x8,
 731         0x3504, 0xffffffff, 0x80500,
 732         0x3500, 0xffffffff, 0x12,
 733         0x3504, 0xffffffff, 0x9050c,
 734         0x3500, 0xffffffff, 0x1d,
 735         0x3504, 0xffffffff, 0xb052c,
 736         0x3500, 0xffffffff, 0x2a,
 737         0x3504, 0xffffffff, 0x1053e,
 738         0x3500, 0xffffffff, 0x2d,
 739         0x3504, 0xffffffff, 0x10546,
 740         0x3500, 0xffffffff, 0x30,
 741         0x3504, 0xffffffff, 0xa054e,
 742         0x3500, 0xffffffff, 0x3c,
 743         0x3504, 0xffffffff, 0x1055f,
 744         0x3500, 0xffffffff, 0x3f,
 745         0x3504, 0xffffffff, 0x10567,
 746         0x3500, 0xffffffff, 0x42,
 747         0x3504, 0xffffffff, 0x1056f,
 748         0x3500, 0xffffffff, 0x45,
 749         0x3504, 0xffffffff, 0x10572,
 750         0x3500, 0xffffffff, 0x48,
 751         0x3504, 0xffffffff, 0x20575,
 752         0x3500, 0xffffffff, 0x4c,
 753         0x3504, 0xffffffff, 0x190801,
 754         0x3500, 0xffffffff, 0x67,
 755         0x3504, 0xffffffff, 0x1082a,
 756         0x3500, 0xffffffff, 0x6a,
 757         0x3504, 0xffffffff, 0x1b082d,
 758         0x3500, 0xffffffff, 0x87,
 759         0x3504, 0xffffffff, 0x310851,
 760         0x3500, 0xffffffff, 0xba,
 761         0x3504, 0xffffffff, 0x891,
 762         0x3500, 0xffffffff, 0xbc,
 763         0x3504, 0xffffffff, 0x893,
 764         0x3500, 0xffffffff, 0xbe,
 765         0x3504, 0xffffffff, 0x20895,
 766         0x3500, 0xffffffff, 0xc2,
 767         0x3504, 0xffffffff, 0x20899,
 768         0x3500, 0xffffffff, 0xc6,
 769         0x3504, 0xffffffff, 0x2089d,
 770         0x3500, 0xffffffff, 0xca,
 771         0x3504, 0xffffffff, 0x8a1,
 772         0x3500, 0xffffffff, 0xcc,
 773         0x3504, 0xffffffff, 0x8a3,
 774         0x3500, 0xffffffff, 0xce,
 775         0x3504, 0xffffffff, 0x308a5,
 776         0x3500, 0xffffffff, 0xd3,
 777         0x3504, 0xffffffff, 0x6d08cd,
 778         0x3500, 0xffffffff, 0x142,
 779         0x3504, 0xffffffff, 0x2000095a,
 780         0x3504, 0xffffffff, 0x1,
 781         0x3500, 0xffffffff, 0x144,
 782         0x3504, 0xffffffff, 0x301f095b,
 783         0x3500, 0xffffffff, 0x165,
 784         0x3504, 0xffffffff, 0xc094d,
 785         0x3500, 0xffffffff, 0x173,
 786         0x3504, 0xffffffff, 0xf096d,
 787         0x3500, 0xffffffff, 0x184,
 788         0x3504, 0xffffffff, 0x15097f,
 789         0x3500, 0xffffffff, 0x19b,
 790         0x3504, 0xffffffff, 0xc0998,
 791         0x3500, 0xffffffff, 0x1a9,
 792         0x3504, 0xffffffff, 0x409a7,
 793         0x3500, 0xffffffff, 0x1af,
 794         0x3504, 0xffffffff, 0xcdc,
 795         0x3500, 0xffffffff, 0x1b1,
 796         0x3504, 0xffffffff, 0x800,
 797         0x3508, 0xffffffff, 0x6c9b2000,
 798         0x3510, 0xfc00, 0x2000,
 799         0x3544, 0xffffffff, 0xfc0,
 800         0x28d4, 0x00000100, 0x100
 801 };
 802
 803 static void si_init_golden_registers(struct radeon_device *rdev)
 804 {
 805         switch (rdev->family) {
 806         case CHIP_TAHITI:
 807                 radeon_program_register_sequence(rdev,
 808                                                  tahiti_golden_registers,
 809                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers));
 810                 radeon_program_register_sequence(rdev,
 811                                                  tahiti_golden_rlc_registers,
 812                                                  (const u32)ARRAY_SIZE(tahiti_golden_rlc_registers));
 813                 radeon_program_register_sequence(rdev,
 814                                                  tahiti_mgcg_cgcg_init,
 815                                                  (const u32)ARRAY_SIZE(tahiti_mgcg_cgcg_init));
 816                 radeon_program_register_sequence(rdev,
 817                                                  tahiti_golden_registers2,
 818                                                  (const u32)ARRAY_SIZE(tahiti_golden_registers2));
 819                 break;
 820         case CHIP_PITCAIRN:
 821                 radeon_program_register_sequence(rdev,
 822                                                  pitcairn_golden_registers,
 823                                                  (const u32)ARRAY_SIZE(pitcairn_golden_registers));
 824                 radeon_program_register_sequence(rdev,
 825                                                  pitcairn_golden_rlc_registers,
 826                                                  (const u32)ARRAY_SIZE(pitcairn_golden_rlc_registers));
 827                 radeon_program_register_sequence(rdev,
 828                                                  pitcairn_mgcg_cgcg_init,
 829                                                  (const u32)ARRAY_SIZE(pitcairn_mgcg_cgcg_init));
 830                 break;
 831         case CHIP_VERDE:
 832                 radeon_program_register_sequence(rdev,
 833                                                  verde_golden_registers,
 834                                                  (const u32)ARRAY_SIZE(verde_golden_registers));
 835                 radeon_program_register_sequence(rdev,
 836                                                  verde_golden_rlc_registers,
 837                                                  (const u32)ARRAY_SIZE(verde_golden_rlc_registers));
 838                 radeon_program_register_sequence(rdev,
 839                                                  verde_mgcg_cgcg_init,
 840                                                  (const u32)ARRAY_SIZE(verde_mgcg_cgcg_init));
 841                 radeon_program_register_sequence(rdev,
 842                                                  verde_pg_init,
 843                                                  (const u32)ARRAY_SIZE(verde_pg_init));
 844                 break;
 845         case CHIP_OLAND:
 846                 radeon_program_register_sequence(rdev,
 847                                                  oland_golden_registers,
 848                                                  (const u32)ARRAY_SIZE(oland_golden_registers));
 849                 radeon_program_register_sequence(rdev,
 850                                                  oland_golden_rlc_registers,
 851                                                  (const u32)ARRAY_SIZE(oland_golden_rlc_registers));
 852                 radeon_program_register_sequence(rdev,
 853                                                  oland_mgcg_cgcg_init,
 854                                                  (const u32)ARRAY_SIZE(oland_mgcg_cgcg_init));
 855                 break;
 856         default:
 857                 break;
 858         }
 859 }
 860
 861 #define PCIE_BUS_CLK                10000
 862 #define TCLK                        (PCIE_BUS_CLK / 10)
 863
 864 /**
 865  * si_get_xclk - get the xclk
 866  *
 867  * @rdev: radeon_device pointer
 868  *
 869  * Returns the reference clock used by the gfx engine
 870  * (SI).
 871  */
 872 u32 si_get_xclk(struct radeon_device *rdev)
 873 {
 874         u32 reference_clock = rdev->clock.spll.reference_freq;
 875         u32 tmp;
 876
 877         tmp = RREG32(CG_CLKPIN_CNTL_2);
 878         if (tmp & MUX_TCLK_TO_XCLK)
 879                 return TCLK;
 880
 881         tmp = RREG32(CG_CLKPIN_CNTL);
 882         if (tmp & XTALIN_DIVIDE)
 883                 return reference_clock / 4;
 884
 885         return reference_clock;
 886 }
 887
 888 /* get temperature in millidegrees */
 889 int si_get_temp(struct radeon_device *rdev)
 890 {
 891         u32 temp;
 892         int actual_temp = 0;
 893
 894         temp = (RREG32(CG_MULT_THERMAL_STATUS) & CTF_TEMP_MASK) >>
 895                 CTF_TEMP_SHIFT;
 896
 897         if (temp & 0x200)
 898                 actual_temp = 255;
 899         else
 900                 actual_temp = temp & 0x1ff;
 901
 902         actual_temp = (actual_temp * 1000);
 903
 904         return actual_temp;
 905 }
 906
 907 #define TAHITI_IO_MC_REGS_SIZE 36
 908
 909 static const u32 tahiti_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
 910         {0x0000006f, 0x03044000},
 911         {0x00000070, 0x0480c018},
 912         {0x00000071, 0x00000040},
 913         {0x00000072, 0x01000000},
 914         {0x00000074, 0x000000ff},
 915         {0x00000075, 0x00143400},
 916         {0x00000076, 0x08ec0800},
 917         {0x00000077, 0x040000cc},
 918         {0x00000079, 0x00000000},
 919         {0x0000007a, 0x21000409},
 920         {0x0000007c, 0x00000000},
 921         {0x0000007d, 0xe8000000},
 922         {0x0000007e, 0x044408a8},
 923         {0x0000007f, 0x00000003},
 924         {0x00000080, 0x00000000},
 925         {0x00000081, 0x01000000},
 926         {0x00000082, 0x02000000},
 927         {0x00000083, 0x00000000},
 928         {0x00000084, 0xe3f3e4f4},
 929         {0x00000085, 0x00052024},
 930         {0x00000087, 0x00000000},
 931         {0x00000088, 0x66036603},
 932         {0x00000089, 0x01000000},
 933         {0x0000008b, 0x1c0a0000},
 934         {0x0000008c, 0xff010000},
 935         {0x0000008e, 0xffffefff},
 936         {0x0000008f, 0xfff3efff},
 937         {0x00000090, 0xfff3efbf},
 938         {0x00000094, 0x00101101},
 939         {0x00000095, 0x00000fff},
 940         {0x00000096, 0x00116fff},
 941         {0x00000097, 0x60010000},
 942         {0x00000098, 0x10010000},
 943         {0x00000099, 0x00006000},
 944         {0x0000009a, 0x00001000},
 945         {0x0000009f, 0x00a77400}
 946 };
 947
 948 static const u32 pitcairn_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
 949         {0x0000006f, 0x03044000},
 950         {0x00000070, 0x0480c018},
 951         {0x00000071, 0x00000040},
 952         {0x00000072, 0x01000000},
 953         {0x00000074, 0x000000ff},
 954         {0x00000075, 0x00143400},
 955         {0x00000076, 0x08ec0800},
 956         {0x00000077, 0x040000cc},
 957         {0x00000079, 0x00000000},
 958         {0x0000007a, 0x21000409},
 959         {0x0000007c, 0x00000000},
 960         {0x0000007d, 0xe8000000},
 961         {0x0000007e, 0x044408a8},
 962         {0x0000007f, 0x00000003},
 963         {0x00000080, 0x00000000},
 964         {0x00000081, 0x01000000},
 965         {0x00000082, 0x02000000},
 966         {0x00000083, 0x00000000},
 967         {0x00000084, 0xe3f3e4f4},
 968         {0x00000085, 0x00052024},
 969         {0x00000087, 0x00000000},
 970         {0x00000088, 0x66036603},
 971         {0x00000089, 0x01000000},
 972         {0x0000008b, 0x1c0a0000},
 973         {0x0000008c, 0xff010000},
 974         {0x0000008e, 0xffffefff},
 975         {0x0000008f, 0xfff3efff},
 976         {0x00000090, 0xfff3efbf},
 977         {0x00000094, 0x00101101},
 978         {0x00000095, 0x00000fff},
 979         {0x00000096, 0x00116fff},
 980         {0x00000097, 0x60010000},
 981         {0x00000098, 0x10010000},
 982         {0x00000099, 0x00006000},
 983         {0x0000009a, 0x00001000},
 984         {0x0000009f, 0x00a47400}
 985 };
 986
 987 static const u32 verde_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
 988         {0x0000006f, 0x03044000},
 989         {0x00000070, 0x0480c018},
 990         {0x00000071, 0x00000040},
 991         {0x00000072, 0x01000000},
 992         {0x00000074, 0x000000ff},
 993         {0x00000075, 0x00143400},
 994         {0x00000076, 0x08ec0800},
 995         {0x00000077, 0x040000cc},
 996         {0x00000079, 0x00000000},
 997         {0x0000007a, 0x21000409},
 998         {0x0000007c, 0x00000000},
 999         {0x0000007d, 0xe8000000},
1000         {0x0000007e, 0x044408a8},
1001         {0x0000007f, 0x00000003},
1002         {0x00000080, 0x00000000},
1003         {0x00000081, 0x01000000},
1004         {0x00000082, 0x02000000},
1005         {0x00000083, 0x00000000},
1006         {0x00000084, 0xe3f3e4f4},
1007         {0x00000085, 0x00052024},
1008         {0x00000087, 0x00000000},
1009         {0x00000088, 0x66036603},
1010         {0x00000089, 0x01000000},
1011         {0x0000008b, 0x1c0a0000},
1012         {0x0000008c, 0xff010000},
1013         {0x0000008e, 0xffffefff},
1014         {0x0000008f, 0xfff3efff},
1015         {0x00000090, 0xfff3efbf},
1016         {0x00000094, 0x00101101},
1017         {0x00000095, 0x00000fff},
1018         {0x00000096, 0x00116fff},
1019         {0x00000097, 0x60010000},
1020         {0x00000098, 0x10010000},
1021         {0x00000099, 0x00006000},
1022         {0x0000009a, 0x00001000},
1023         {0x0000009f, 0x00a37400}
1024 };
1025
1026 static const u32 oland_io_mc_regs[TAHITI_IO_MC_REGS_SIZE][2] = {
1027         {0x0000006f, 0x03044000},
1028         {0x00000070, 0x0480c018},
1029         {0x00000071, 0x00000040},
1030         {0x00000072, 0x01000000},
1031         {0x00000074, 0x000000ff},
1032         {0x00000075, 0x00143400},
1033         {0x00000076, 0x08ec0800},
1034         {0x00000077, 0x040000cc},
1035         {0x00000079, 0x00000000},
1036         {0x0000007a, 0x21000409},
1037         {0x0000007c, 0x00000000},
1038         {0x0000007d, 0xe8000000},
1039         {0x0000007e, 0x044408a8},
1040         {0x0000007f, 0x00000003},
1041         {0x00000080, 0x00000000},
1042         {0x00000081, 0x01000000},
1043         {0x00000082, 0x02000000},
1044         {0x00000083, 0x00000000},
1045         {0x00000084, 0xe3f3e4f4},
1046         {0x00000085, 0x00052024},
1047         {0x00000087, 0x00000000},
1048         {0x00000088, 0x66036603},
1049         {0x00000089, 0x01000000},
1050         {0x0000008b, 0x1c0a0000},
1051         {0x0000008c, 0xff010000},
1052         {0x0000008e, 0xffffefff},
1053         {0x0000008f, 0xfff3efff},
1054         {0x00000090, 0xfff3efbf},
1055         {0x00000094, 0x00101101},
1056         {0x00000095, 0x00000fff},
1057         {0x00000096, 0x00116fff},
1058         {0x00000097, 0x60010000},
1059         {0x00000098, 0x10010000},
1060         {0x00000099, 0x00006000},
1061         {0x0000009a, 0x00001000},
1062         {0x0000009f, 0x00a17730}
1063 };
1064
1065 /* ucode loading */
1066 static int si_mc_load_microcode(struct radeon_device *rdev)
1067 {
1068         const __be32 *fw_data;
1069         u32 running, blackout = 0;
1070         u32 *io_mc_regs;
1071         int i, ucode_size, regs_size;
1072
1073         if (!rdev->mc_fw)
1074                 return -EINVAL;
1075
1076         switch (rdev->family) {
1077         case CHIP_TAHITI:
1078                 io_mc_regs = (u32 *)&tahiti_io_mc_regs;
1079                 ucode_size = SI_MC_UCODE_SIZE;
1080                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1081                 break;
1082         case CHIP_PITCAIRN:
1083                 io_mc_regs = (u32 *)&pitcairn_io_mc_regs;
1084                 ucode_size = SI_MC_UCODE_SIZE;
1085                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1086                 break;
1087         case CHIP_VERDE:
1088         default:
1089                 io_mc_regs = (u32 *)&verde_io_mc_regs;
1090                 ucode_size = SI_MC_UCODE_SIZE;
1091                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1092                 break;
1093         case CHIP_OLAND:
1094                 io_mc_regs = (u32 *)&oland_io_mc_regs;
1095                 ucode_size = OLAND_MC_UCODE_SIZE;
1096                 regs_size = TAHITI_IO_MC_REGS_SIZE;
1097                 break;
1098         }
1099
1100         running = RREG32(MC_SEQ_SUP_CNTL) & RUN_MASK;
1101
1102         if (running == 0) {
1103                 if (running) {
1104                         blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
1105                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
1106                 }
1107
1108                 /* reset the engine and set to writable */
1109                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1110                 WREG32(MC_SEQ_SUP_CNTL, 0x00000010);
1111
1112                 /* load mc io regs */
1113                 for (i = 0; i < regs_size; i++) {
1114                         WREG32(MC_SEQ_IO_DEBUG_INDEX, io_mc_regs[(i << 1)]);
1115                         WREG32(MC_SEQ_IO_DEBUG_DATA, io_mc_regs[(i << 1) + 1]);
1116                 }
1117                 /* load the MC ucode */
1118                 fw_data = (const __be32 *)rdev->mc_fw->data;
1119                 for (i = 0; i < ucode_size; i++)
1120                         WREG32(MC_SEQ_SUP_PGM, be32_to_cpup(fw_data++));
1121
1122                 /* put the engine back into the active state */
1123                 WREG32(MC_SEQ_SUP_CNTL, 0x00000008);
1124                 WREG32(MC_SEQ_SUP_CNTL, 0x00000004);
1125                 WREG32(MC_SEQ_SUP_CNTL, 0x00000001);
1126
1127                 /* wait for training to complete */
1128                 for (i = 0; i < rdev->usec_timeout; i++) {
1129                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D0)
1130                                 break;
1131                         udelay(1);
1132                 }
1133                 for (i = 0; i < rdev->usec_timeout; i++) {
1134                         if (RREG32(MC_SEQ_TRAIN_WAKEUP_CNTL) & TRAIN_DONE_D1)
1135                                 break;
1136                         udelay(1);
1137                 }
1138
1139                 if (running)
1140                         WREG32(MC_SHARED_BLACKOUT_CNTL, blackout);
1141         }
1142
1143         return 0;
1144 }
1145
1146 static int si_init_microcode(struct radeon_device *rdev)
1147 {
1148         struct platform_device *pdev;
1149         const char *chip_name;
1150         const char *rlc_chip_name;
1151         size_t pfp_req_size, me_req_size, ce_req_size, rlc_req_size, mc_req_size;
1152         char fw_name[30];
1153         int err;
1154
1155         DRM_DEBUG("\n");
1156
1157         pdev = platform_device_register_simple("radeon_cp", 0, NULL, 0);
1158         err = IS_ERR(pdev);
1159         if (err) {
1160                 printk(KERN_ERR "radeon_cp: Failed to register firmware\n");
1161                 return -EINVAL;
1162         }
1163
1164         switch (rdev->family) {
1165         case CHIP_TAHITI:
1166                 chip_name = "TAHITI";
1167                 rlc_chip_name = "TAHITI";
1168                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1169                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1170                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1171                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1172                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1173                 break;
1174         case CHIP_PITCAIRN:
1175                 chip_name = "PITCAIRN";
1176                 rlc_chip_name = "PITCAIRN";
1177                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1178                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1179                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1180                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1181                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1182                 break;
1183         case CHIP_VERDE:
1184                 chip_name = "VERDE";
1185                 rlc_chip_name = "VERDE";
1186                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1187                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1188                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1189                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1190                 mc_req_size = SI_MC_UCODE_SIZE * 4;
1191                 break;
1192         case CHIP_OLAND:
1193                 chip_name = "OLAND";
1194                 rlc_chip_name = "OLAND";
1195                 pfp_req_size = SI_PFP_UCODE_SIZE * 4;
1196                 me_req_size = SI_PM4_UCODE_SIZE * 4;
1197                 ce_req_size = SI_CE_UCODE_SIZE * 4;
1198                 rlc_req_size = SI_RLC_UCODE_SIZE * 4;
1199                 mc_req_size = OLAND_MC_UCODE_SIZE * 4;
1200                 break;
1201         default: BUG();
1202         }
1203
1204         DRM_INFO("Loading %s Microcode\n", chip_name);
1205
1206         snprintf(fw_name, sizeof(fw_name), "radeon/%s_pfp.bin", chip_name);
1207         err = request_firmware(&rdev->pfp_fw, fw_name, &pdev->dev);
1208         if (err)
1209                 goto out;
1210         if (rdev->pfp_fw->size != pfp_req_size) {
1211                 printk(KERN_ERR
1212                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1213                        rdev->pfp_fw->size, fw_name);
1214                 err = -EINVAL;
1215                 goto out;
1216         }
1217
1218         snprintf(fw_name, sizeof(fw_name), "radeon/%s_me.bin", chip_name);
1219         err = request_firmware(&rdev->me_fw, fw_name, &pdev->dev);
1220         if (err)
1221                 goto out;
1222         if (rdev->me_fw->size != me_req_size) {
1223                 printk(KERN_ERR
1224                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1225                        rdev->me_fw->size, fw_name);
1226                 err = -EINVAL;
1227         }
1228
1229         snprintf(fw_name, sizeof(fw_name), "radeon/%s_ce.bin", chip_name);
1230         err = request_firmware(&rdev->ce_fw, fw_name, &pdev->dev);
1231         if (err)
1232                 goto out;
1233         if (rdev->ce_fw->size != ce_req_size) {
1234                 printk(KERN_ERR
1235                        "si_cp: Bogus length %zu in firmware \"%s\"\n",
1236                        rdev->ce_fw->size, fw_name);
1237                 err = -EINVAL;
1238         }
1239
1240         snprintf(fw_name, sizeof(fw_name), "radeon/%s_rlc.bin", rlc_chip_name);
1241         err = request_firmware(&rdev->rlc_fw, fw_name, &pdev->dev);
1242         if (err)
1243                 goto out;
1244         if (rdev->rlc_fw->size != rlc_req_size) {
1245                 printk(KERN_ERR
1246                        "si_rlc: Bogus length %zu in firmware \"%s\"\n",
1247                        rdev->rlc_fw->size, fw_name);
1248                 err = -EINVAL;
1249         }
1250
1251         snprintf(fw_name, sizeof(fw_name), "radeon/%s_mc.bin", chip_name);
1252         err = request_firmware(&rdev->mc_fw, fw_name, &pdev->dev);
1253         if (err)
1254                 goto out;
1255         if (rdev->mc_fw->size != mc_req_size) {
1256                 printk(KERN_ERR
1257                        "si_mc: Bogus length %zu in firmware \"%s\"\n",
1258                        rdev->mc_fw->size, fw_name);
1259                 err = -EINVAL;
1260         }
1261
1262 out:
1263         platform_device_unregister(pdev);
1264
1265         if (err) {
1266                 if (err != -EINVAL)
1267                         printk(KERN_ERR
1268                                "si_cp: Failed to load firmware \"%s\"\n",
1269                                fw_name);
1270                 release_firmware(rdev->pfp_fw);
1271                 rdev->pfp_fw = NULL;
1272                 release_firmware(rdev->me_fw);
1273                 rdev->me_fw = NULL;
1274                 release_firmware(rdev->ce_fw);
1275                 rdev->ce_fw = NULL;
1276                 release_firmware(rdev->rlc_fw);
1277                 rdev->rlc_fw = NULL;
1278                 release_firmware(rdev->mc_fw);
1279                 rdev->mc_fw = NULL;
1280         }
1281         return err;
1282 }
1283
1284 /* watermark setup */
1285 static u32 dce6_line_buffer_adjust(struct radeon_device *rdev,
1286                                    struct radeon_crtc *radeon_crtc,
1287                                    struct drm_display_mode *mode,
1288                                    struct drm_display_mode *other_mode)
1289 {
1290         u32 tmp;
1291         /*
1292          * Line Buffer Setup
1293          * There are 3 line buffers, each one shared by 2 display controllers.
1294          * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1295          * the display controllers.  The paritioning is done via one of four
1296          * preset allocations specified in bits 21:20:
1297          *  0 - half lb
1298          *  2 - whole lb, other crtc must be disabled
1299          */
1300         /* this can get tricky if we have two large displays on a paired group
1301          * of crtcs.  Ideally for multiple large displays we'd assign them to
1302          * non-linked crtcs for maximum line buffer allocation.
1303          */
1304         if (radeon_crtc->base.enabled && mode) {
1305                 if (other_mode)
1306                         tmp = 0; /* 1/2 */
1307                 else
1308                         tmp = 2; /* whole */
1309         } else
1310                 tmp = 0;
1311
1312         WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset,
1313                DC_LB_MEMORY_CONFIG(tmp));
1314
1315         if (radeon_crtc->base.enabled && mode) {
1316                 switch (tmp) {
1317                 case 0:
1318                 default:
1319                         return 4096 * 2;
1320                 case 2:
1321                         return 8192 * 2;
1322                 }
1323         }
1324
1325         /* controller not enabled, so no lb used */
1326         return 0;
1327 }
1328
1329 static u32 si_get_number_of_dram_channels(struct radeon_device *rdev)
1330 {
1331         u32 tmp = RREG32(MC_SHARED_CHMAP);
1332
1333         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1334         case 0:
1335         default:
1336                 return 1;
1337         case 1:
1338                 return 2;
1339         case 2:
1340                 return 4;
1341         case 3:
1342                 return 8;
1343         case 4:
1344                 return 3;
1345         case 5:
1346                 return 6;
1347         case 6:
1348                 return 10;
1349         case 7:
1350                 return 12;
1351         case 8:
1352                 return 16;
1353         }
1354 }
1355
1356 struct dce6_wm_params {
1357         u32 dram_channels; /* number of dram channels */
1358         u32 yclk;          /* bandwidth per dram data pin in kHz */
1359         u32 sclk;          /* engine clock in kHz */
1360         u32 disp_clk;      /* display clock in kHz */
1361         u32 src_width;     /* viewport width */
1362         u32 active_time;   /* active display time in ns */
1363         u32 blank_time;    /* blank time in ns */
1364         bool interlaced;    /* mode is interlaced */
1365         fixed20_12 vsc;    /* vertical scale ratio */
1366         u32 num_heads;     /* number of active crtcs */
1367         u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1368         u32 lb_size;       /* line buffer allocated to pipe */
1369         u32 vtaps;         /* vertical scaler taps */
1370 };
1371
1372 static u32 dce6_dram_bandwidth(struct dce6_wm_params *wm)
1373 {
1374         /* Calculate raw DRAM Bandwidth */
1375         fixed20_12 dram_efficiency; /* 0.7 */
1376         fixed20_12 yclk, dram_channels, bandwidth;
1377         fixed20_12 a;
1378
1379         a.full = dfixed_const(1000);
1380         yclk.full = dfixed_const(wm->yclk);
1381         yclk.full = dfixed_div(yclk, a);
1382         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1383         a.full = dfixed_const(10);
1384         dram_efficiency.full = dfixed_const(7);
1385         dram_efficiency.full = dfixed_div(dram_efficiency, a);
1386         bandwidth.full = dfixed_mul(dram_channels, yclk);
1387         bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1388
1389         return dfixed_trunc(bandwidth);
1390 }
1391
1392 static u32 dce6_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1393 {
1394         /* Calculate DRAM Bandwidth and the part allocated to display. */
1395         fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1396         fixed20_12 yclk, dram_channels, bandwidth;
1397         fixed20_12 a;
1398
1399         a.full = dfixed_const(1000);
1400         yclk.full = dfixed_const(wm->yclk);
1401         yclk.full = dfixed_div(yclk, a);
1402         dram_channels.full = dfixed_const(wm->dram_channels * 4);
1403         a.full = dfixed_const(10);
1404         disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1405         disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1406         bandwidth.full = dfixed_mul(dram_channels, yclk);
1407         bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1408
1409         return dfixed_trunc(bandwidth);
1410 }
1411
1412 static u32 dce6_data_return_bandwidth(struct dce6_wm_params *wm)
1413 {
1414         /* Calculate the display Data return Bandwidth */
1415         fixed20_12 return_efficiency; /* 0.8 */
1416         fixed20_12 sclk, bandwidth;
1417         fixed20_12 a;
1418
1419         a.full = dfixed_const(1000);
1420         sclk.full = dfixed_const(wm->sclk);
1421         sclk.full = dfixed_div(sclk, a);
1422         a.full = dfixed_const(10);
1423         return_efficiency.full = dfixed_const(8);
1424         return_efficiency.full = dfixed_div(return_efficiency, a);
1425         a.full = dfixed_const(32);
1426         bandwidth.full = dfixed_mul(a, sclk);
1427         bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
1428
1429         return dfixed_trunc(bandwidth);
1430 }
1431
1432 static u32 dce6_get_dmif_bytes_per_request(struct dce6_wm_params *wm)
1433 {
1434         return 32;
1435 }
1436
1437 static u32 dce6_dmif_request_bandwidth(struct dce6_wm_params *wm)
1438 {
1439         /* Calculate the DMIF Request Bandwidth */
1440         fixed20_12 disp_clk_request_efficiency; /* 0.8 */
1441         fixed20_12 disp_clk, sclk, bandwidth;
1442         fixed20_12 a, b1, b2;
1443         u32 min_bandwidth;
1444
1445         a.full = dfixed_const(1000);
1446         disp_clk.full = dfixed_const(wm->disp_clk);
1447         disp_clk.full = dfixed_div(disp_clk, a);
1448         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm) / 2);
1449         b1.full = dfixed_mul(a, disp_clk);
1450
1451         a.full = dfixed_const(1000);
1452         sclk.full = dfixed_const(wm->sclk);
1453         sclk.full = dfixed_div(sclk, a);
1454         a.full = dfixed_const(dce6_get_dmif_bytes_per_request(wm));
1455         b2.full = dfixed_mul(a, sclk);
1456
1457         a.full = dfixed_const(10);
1458         disp_clk_request_efficiency.full = dfixed_const(8);
1459         disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
1460
1461         min_bandwidth = min(dfixed_trunc(b1), dfixed_trunc(b2));
1462
1463         a.full = dfixed_const(min_bandwidth);
1464         bandwidth.full = dfixed_mul(a, disp_clk_request_efficiency);
1465
1466         return dfixed_trunc(bandwidth);
1467 }
1468
1469 static u32 dce6_available_bandwidth(struct dce6_wm_params *wm)
1470 {
1471         /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
1472         u32 dram_bandwidth = dce6_dram_bandwidth(wm);
1473         u32 data_return_bandwidth = dce6_data_return_bandwidth(wm);
1474         u32 dmif_req_bandwidth = dce6_dmif_request_bandwidth(wm);
1475
1476         return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
1477 }
1478
1479 static u32 dce6_average_bandwidth(struct dce6_wm_params *wm)
1480 {
1481         /* Calculate the display mode Average Bandwidth
1482          * DisplayMode should contain the source and destination dimensions,
1483          * timing, etc.
1484          */
1485         fixed20_12 bpp;
1486         fixed20_12 line_time;
1487         fixed20_12 src_width;
1488         fixed20_12 bandwidth;
1489         fixed20_12 a;
1490
1491         a.full = dfixed_const(1000);
1492         line_time.full = dfixed_const(wm->active_time + wm->blank_time);
1493         line_time.full = dfixed_div(line_time, a);
1494         bpp.full = dfixed_const(wm->bytes_per_pixel);
1495         src_width.full = dfixed_const(wm->src_width);
1496         bandwidth.full = dfixed_mul(src_width, bpp);
1497         bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
1498         bandwidth.full = dfixed_div(bandwidth, line_time);
1499
1500         return dfixed_trunc(bandwidth);
1501 }
1502
1503 static u32 dce6_latency_watermark(struct dce6_wm_params *wm)
1504 {
1505         /* First calcualte the latency in ns */
1506         u32 mc_latency = 2000; /* 2000 ns. */
1507         u32 available_bandwidth = dce6_available_bandwidth(wm);
1508         u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
1509         u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
1510         u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
1511         u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
1512                 (wm->num_heads * cursor_line_pair_return_time);
1513         u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
1514         u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
1515         u32 tmp, dmif_size = 12288;
1516         fixed20_12 a, b, c;
1517
1518         if (wm->num_heads == 0)
1519                 return 0;
1520
1521         a.full = dfixed_const(2);
1522         b.full = dfixed_const(1);
1523         if ((wm->vsc.full > a.full) ||
1524             ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
1525             (wm->vtaps >= 5) ||
1526             ((wm->vsc.full >= a.full) && wm->interlaced))
1527                 max_src_lines_per_dst_line = 4;
1528         else
1529                 max_src_lines_per_dst_line = 2;
1530
1531         a.full = dfixed_const(available_bandwidth);
1532         b.full = dfixed_const(wm->num_heads);
1533         a.full = dfixed_div(a, b);
1534
1535         b.full = dfixed_const(mc_latency + 512);
1536         c.full = dfixed_const(wm->disp_clk);
1537         b.full = dfixed_div(b, c);
1538
1539         c.full = dfixed_const(dmif_size);
1540         b.full = dfixed_div(c, b);
1541
1542         tmp = min(dfixed_trunc(a), dfixed_trunc(b));
1543
1544         b.full = dfixed_const(1000);
1545         c.full = dfixed_const(wm->disp_clk);
1546         b.full = dfixed_div(c, b);
1547         c.full = dfixed_const(wm->bytes_per_pixel);
1548         b.full = dfixed_mul(b, c);
1549
1550         lb_fill_bw = min(tmp, dfixed_trunc(b));
1551
1552         a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
1553         b.full = dfixed_const(1000);
1554         c.full = dfixed_const(lb_fill_bw);
1555         b.full = dfixed_div(c, b);
1556         a.full = dfixed_div(a, b);
1557         line_fill_time = dfixed_trunc(a);
1558
1559         if (line_fill_time < wm->active_time)
1560                 return latency;
1561         else
1562                 return latency + (line_fill_time - wm->active_time);
1563
1564 }
1565
1566 static bool dce6_average_bandwidth_vs_dram_bandwidth_for_display(struct dce6_wm_params *wm)
1567 {
1568         if (dce6_average_bandwidth(wm) <=
1569             (dce6_dram_bandwidth_for_display(wm) / wm->num_heads))
1570                 return true;
1571         else
1572                 return false;
1573 };
1574
1575 static bool dce6_average_bandwidth_vs_available_bandwidth(struct dce6_wm_params *wm)
1576 {
1577         if (dce6_average_bandwidth(wm) <=
1578             (dce6_available_bandwidth(wm) / wm->num_heads))
1579                 return true;
1580         else
1581                 return false;
1582 };
1583
1584 static bool dce6_check_latency_hiding(struct dce6_wm_params *wm)
1585 {
1586         u32 lb_partitions = wm->lb_size / wm->src_width;
1587         u32 line_time = wm->active_time + wm->blank_time;
1588         u32 latency_tolerant_lines;
1589         u32 latency_hiding;
1590         fixed20_12 a;
1591
1592         a.full = dfixed_const(1);
1593         if (wm->vsc.full > a.full)
1594                 latency_tolerant_lines = 1;
1595         else {
1596                 if (lb_partitions <= (wm->vtaps + 1))
1597                         latency_tolerant_lines = 1;
1598                 else
1599                         latency_tolerant_lines = 2;
1600         }
1601
1602         latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
1603
1604         if (dce6_latency_watermark(wm) <= latency_hiding)
1605                 return true;
1606         else
1607                 return false;
1608 }
1609
1610 static void dce6_program_watermarks(struct radeon_device *rdev,
1611                                          struct radeon_crtc *radeon_crtc,
1612                                          u32 lb_size, u32 num_heads)
1613 {
1614         struct drm_display_mode *mode = &radeon_crtc->base.mode;
1615         struct dce6_wm_params wm;
1616         u32 pixel_period;
1617         u32 line_time = 0;
1618         u32 latency_watermark_a = 0, latency_watermark_b = 0;
1619         u32 priority_a_mark = 0, priority_b_mark = 0;
1620         u32 priority_a_cnt = PRIORITY_OFF;
1621         u32 priority_b_cnt = PRIORITY_OFF;
1622         u32 tmp, arb_control3;
1623         fixed20_12 a, b, c;
1624
1625         if (radeon_crtc->base.enabled && num_heads && mode) {
1626                 pixel_period = 1000000 / (u32)mode->clock;
1627                 line_time = min((u32)mode->crtc_htotal * pixel_period, (u32)65535);
1628                 priority_a_cnt = 0;
1629                 priority_b_cnt = 0;
1630
1631                 wm.yclk = rdev->pm.current_mclk * 10;
1632                 wm.sclk = rdev->pm.current_sclk * 10;
1633                 wm.disp_clk = mode->clock;
1634                 wm.src_width = mode->crtc_hdisplay;
1635                 wm.active_time = mode->crtc_hdisplay * pixel_period;
1636                 wm.blank_time = line_time - wm.active_time;
1637                 wm.interlaced = false;
1638                 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
1639                         wm.interlaced = true;
1640                 wm.vsc = radeon_crtc->vsc;
1641                 wm.vtaps = 1;
1642                 if (radeon_crtc->rmx_type != RMX_OFF)
1643                         wm.vtaps = 2;
1644                 wm.bytes_per_pixel = 4; /* XXX: get this from fb config */
1645                 wm.lb_size = lb_size;
1646                 if (rdev->family == CHIP_ARUBA)
1647                         wm.dram_channels = evergreen_get_number_of_dram_channels(rdev);
1648                 else
1649                         wm.dram_channels = si_get_number_of_dram_channels(rdev);
1650                 wm.num_heads = num_heads;
1651
1652                 /* set for high clocks */
1653                 latency_watermark_a = min(dce6_latency_watermark(&wm), (u32)65535);
1654                 /* set for low clocks */
1655                 /* wm.yclk = low clk; wm.sclk = low clk */
1656                 latency_watermark_b = min(dce6_latency_watermark(&wm), (u32)65535);
1657
1658                 /* possibly force display priority to high */
1659                 /* should really do this at mode validation time... */
1660                 if (!dce6_average_bandwidth_vs_dram_bandwidth_for_display(&wm) ||
1661                     !dce6_average_bandwidth_vs_available_bandwidth(&wm) ||
1662                     !dce6_check_latency_hiding(&wm) ||
1663                     (rdev->disp_priority == 2)) {
1664                         DRM_DEBUG_KMS("force priority to high\n");
1665                         priority_a_cnt |= PRIORITY_ALWAYS_ON;
1666                         priority_b_cnt |= PRIORITY_ALWAYS_ON;
1667                 }
1668
1669                 a.full = dfixed_const(1000);
1670                 b.full = dfixed_const(mode->clock);
1671                 b.full = dfixed_div(b, a);
1672                 c.full = dfixed_const(latency_watermark_a);
1673                 c.full = dfixed_mul(c, b);
1674                 c.full = dfixed_mul(c, radeon_crtc->hsc);
1675                 c.full = dfixed_div(c, a);
1676                 a.full = dfixed_const(16);
1677                 c.full = dfixed_div(c, a);
1678                 priority_a_mark = dfixed_trunc(c);
1679                 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
1680
1681                 a.full = dfixed_const(1000);
1682                 b.full = dfixed_const(mode->clock);
1683                 b.full = dfixed_div(b, a);
1684                 c.full = dfixed_const(latency_watermark_b);
1685                 c.full = dfixed_mul(c, b);
1686                 c.full = dfixed_mul(c, radeon_crtc->hsc);
1687                 c.full = dfixed_div(c, a);
1688                 a.full = dfixed_const(16);
1689                 c.full = dfixed_div(c, a);
1690                 priority_b_mark = dfixed_trunc(c);
1691                 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
1692         }
1693
1694         /* select wm A */
1695         arb_control3 = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1696         tmp = arb_control3;
1697         tmp &= ~LATENCY_WATERMARK_MASK(3);
1698         tmp |= LATENCY_WATERMARK_MASK(1);
1699         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1700         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1701                (LATENCY_LOW_WATERMARK(latency_watermark_a) |
1702                 LATENCY_HIGH_WATERMARK(line_time)));
1703         /* select wm B */
1704         tmp = RREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset);
1705         tmp &= ~LATENCY_WATERMARK_MASK(3);
1706         tmp |= LATENCY_WATERMARK_MASK(2);
1707         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, tmp);
1708         WREG32(DPG_PIPE_LATENCY_CONTROL + radeon_crtc->crtc_offset,
1709                (LATENCY_LOW_WATERMARK(latency_watermark_b) |
1710                 LATENCY_HIGH_WATERMARK(line_time)));
1711         /* restore original selection */
1712         WREG32(DPG_PIPE_ARBITRATION_CONTROL3 + radeon_crtc->crtc_offset, arb_control3);
1713
1714         /* write the priority marks */
1715         WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
1716         WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
1717
1718 }
1719
1720 void dce6_bandwidth_update(struct radeon_device *rdev)
1721 {
1722         struct drm_display_mode *mode0 = NULL;
1723         struct drm_display_mode *mode1 = NULL;
1724         u32 num_heads = 0, lb_size;
1725         int i;
1726
1727         radeon_update_display_priority(rdev);
1728
1729         for (i = 0; i < rdev->num_crtc; i++) {
1730                 if (rdev->mode_info.crtcs[i]->base.enabled)
1731                         num_heads++;
1732         }
1733         for (i = 0; i < rdev->num_crtc; i += 2) {
1734                 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
1735                 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
1736                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
1737                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
1738                 lb_size = dce6_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
1739                 dce6_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
1740         }
1741 }
1742
1743 /*
1744  * Core functions
1745  */
1746 static void si_tiling_mode_table_init(struct radeon_device *rdev)
1747 {
1748         const u32 num_tile_mode_states = 32;
1749         u32 reg_offset, gb_tile_moden, split_equal_to_row_size;
1750
1751         switch (rdev->config.si.mem_row_size_in_kb) {
1752         case 1:
1753                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_1KB;
1754                 break;
1755         case 2:
1756         default:
1757                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_2KB;
1758                 break;
1759         case 4:
1760                 split_equal_to_row_size = ADDR_SURF_TILE_SPLIT_4KB;
1761                 break;
1762         }
1763
1764         if ((rdev->family == CHIP_TAHITI) ||
1765             (rdev->family == CHIP_PITCAIRN)) {
1766                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
1767                         switch (reg_offset) {
1768                         case 0:  /* non-AA compressed depth or any compressed stencil */
1769                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1770                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1771                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1772                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1773                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1774                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1775                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1776                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1777                                 break;
1778                         case 1:  /* 2xAA/4xAA compressed depth only */
1779                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1780                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1781                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1782                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1783                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1784                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1785                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1786                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1787                                 break;
1788                         case 2:  /* 8xAA compressed depth only */
1789                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1790                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1791                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1792                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1793                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1794                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1795                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1796                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1797                                 break;
1798                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
1799                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1800                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1801                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1802                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
1803                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1804                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1805                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1806                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1807                                 break;
1808                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
1809                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1810                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1811                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1812                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1813                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1814                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1815                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1816                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1817                                 break;
1818                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
1819                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1820                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1821                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1822                                                  TILE_SPLIT(split_equal_to_row_size) |
1823                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1824                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1825                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1826                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1827                                 break;
1828                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
1829                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1830                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1831                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1832                                                  TILE_SPLIT(split_equal_to_row_size) |
1833                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1834                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1835                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1836                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1837                                 break;
1838                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
1839                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1840                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
1841                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1842                                                  TILE_SPLIT(split_equal_to_row_size) |
1843                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1844                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1845                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1846                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1847                                 break;
1848                         case 8:  /* 1D and 1D Array Surfaces */
1849                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
1850                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1851                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1852                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1853                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1854                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1855                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1856                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1857                                 break;
1858                         case 9:  /* Displayable maps. */
1859                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1860                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1861                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1862                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1863                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1864                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1865                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1866                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1867                                 break;
1868                         case 10:  /* Display 8bpp. */
1869                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1870                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1871                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1872                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1873                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1874                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1875                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1876                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1877                                 break;
1878                         case 11:  /* Display 16bpp. */
1879                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1880                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1881                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1882                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1883                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1884                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1885                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1886                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1887                                 break;
1888                         case 12:  /* Display 32bpp. */
1889                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1890                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
1891                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1892                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1893                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1894                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1895                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1896                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1897                                 break;
1898                         case 13:  /* Thin. */
1899                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
1900                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1901                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1902                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
1903                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1904                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1905                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1906                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1907                                 break;
1908                         case 14:  /* Thin 8 bpp. */
1909                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1910                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1911                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1912                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1913                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1914                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1915                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1916                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1917                                 break;
1918                         case 15:  /* Thin 16 bpp. */
1919                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1920                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1921                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1922                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1923                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1924                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1925                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1926                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1927                                 break;
1928                         case 16:  /* Thin 32 bpp. */
1929                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1930                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1931                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1932                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1933                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1934                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1935                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1936                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1937                                 break;
1938                         case 17:  /* Thin 64 bpp. */
1939                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1940                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1941                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1942                                                  TILE_SPLIT(split_equal_to_row_size) |
1943                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1944                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1945                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1946                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1947                                 break;
1948                         case 21:  /* 8 bpp PRT. */
1949                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1950                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1951                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1952                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1953                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1954                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
1955                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1956                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1957                                 break;
1958                         case 22:  /* 16 bpp PRT */
1959                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1960                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1961                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1962                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1963                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1964                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1965                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
1966                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
1967                                 break;
1968                         case 23:  /* 32 bpp PRT */
1969                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1970                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1971                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1972                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
1973                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1974                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1975                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
1976                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1977                                 break;
1978                         case 24:  /* 64 bpp PRT */
1979                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1980                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1981                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1982                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
1983                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
1984                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1985                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1986                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
1987                                 break;
1988                         case 25:  /* 128 bpp PRT */
1989                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
1990                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
1991                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
1992                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
1993                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
1994                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
1995                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
1996                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
1997                                 break;
1998                         default:
1999                                 gb_tile_moden = 0;
2000                                 break;
2001                         }
2002                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2003                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2004                 }
2005         } else if ((rdev->family == CHIP_VERDE) ||
2006                    (rdev->family == CHIP_OLAND)) {
2007                 for (reg_offset = 0; reg_offset < num_tile_mode_states; reg_offset++) {
2008                         switch (reg_offset) {
2009                         case 0:  /* non-AA compressed depth or any compressed stencil */
2010                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2011                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2012                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2013                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2014                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2015                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2016                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2017                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2018                                 break;
2019                         case 1:  /* 2xAA/4xAA compressed depth only */
2020                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2021                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2022                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2023                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2024                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2025                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2026                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2027                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2028                                 break;
2029                         case 2:  /* 8xAA compressed depth only */
2030                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2031                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2032                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2033                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2034                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2035                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2036                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2037                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2038                                 break;
2039                         case 3:  /* 2xAA/4xAA compressed depth with stencil (for depth buffer) */
2040                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2041                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2042                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2043                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_128B) |
2044                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2045                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2046                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2047                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2048                                 break;
2049                         case 4:  /* Maps w/ a dimension less than the 2D macro-tile dimensions (for mipmapped depth textures) */
2050                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2051                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2052                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2053                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2054                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2055                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2056                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2057                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2058                                 break;
2059                         case 5:  /* Uncompressed 16bpp depth - and stencil buffer allocated with it */
2060                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2061                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2062                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2063                                                  TILE_SPLIT(split_equal_to_row_size) |
2064                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2065                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2066                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2067                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2068                                 break;
2069                         case 6:  /* Uncompressed 32bpp depth - and stencil buffer allocated with it */
2070                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2071                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2072                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2073                                                  TILE_SPLIT(split_equal_to_row_size) |
2074                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2075                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2076                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2077                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2078                                 break;
2079                         case 7:  /* Uncompressed 8bpp stencil without depth (drivers typically do not use) */
2080                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2081                                                  MICRO_TILE_MODE(ADDR_SURF_DEPTH_MICRO_TILING) |
2082                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2083                                                  TILE_SPLIT(split_equal_to_row_size) |
2084                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2085                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2086                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2087                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2088                                 break;
2089                         case 8:  /* 1D and 1D Array Surfaces */
2090                                 gb_tile_moden = (ARRAY_MODE(ARRAY_LINEAR_ALIGNED) |
2091                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2092                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2093                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2094                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2095                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2096                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2097                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2098                                 break;
2099                         case 9:  /* Displayable maps. */
2100                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2101                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2102                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2103                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2104                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2105                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2106                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2107                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2108                                 break;
2109                         case 10:  /* Display 8bpp. */
2110                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2111                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2112                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2113                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2114                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2115                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2116                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2117                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2118                                 break;
2119                         case 11:  /* Display 16bpp. */
2120                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2121                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2122                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2123                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2124                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2125                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2126                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2127                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2128                                 break;
2129                         case 12:  /* Display 32bpp. */
2130                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2131                                                  MICRO_TILE_MODE(ADDR_SURF_DISPLAY_MICRO_TILING) |
2132                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2133                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2134                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2135                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2136                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2137                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2138                                 break;
2139                         case 13:  /* Thin. */
2140                                 gb_tile_moden = (ARRAY_MODE(ARRAY_1D_TILED_THIN1) |
2141                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2142                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2143                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_64B) |
2144                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2145                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2146                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2147                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2148                                 break;
2149                         case 14:  /* Thin 8 bpp. */
2150                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2151                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2152                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2153                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2154                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2155                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2156                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2157                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2158                                 break;
2159                         case 15:  /* Thin 16 bpp. */
2160                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2161                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2162                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2163                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2164                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2165                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2166                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2167                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2168                                 break;
2169                         case 16:  /* Thin 32 bpp. */
2170                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2171                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2172                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2173                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2174                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2175                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2176                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2177                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2178                                 break;
2179                         case 17:  /* Thin 64 bpp. */
2180                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2181                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2182                                                  PIPE_CONFIG(ADDR_SURF_P4_8x16) |
2183                                                  TILE_SPLIT(split_equal_to_row_size) |
2184                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2185                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2186                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2187                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2188                                 break;
2189                         case 21:  /* 8 bpp PRT. */
2190                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2191                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2192                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2193                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2194                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2195                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_2) |
2196                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2197                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2198                                 break;
2199                         case 22:  /* 16 bpp PRT */
2200                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2201                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2202                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2203                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2204                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2205                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2206                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_4) |
2207                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_4));
2208                                 break;
2209                         case 23:  /* 32 bpp PRT */
2210                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2211                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2212                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2213                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_256B) |
2214                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2215                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2216                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_2) |
2217                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2218                                 break;
2219                         case 24:  /* 64 bpp PRT */
2220                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2221                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2222                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2223                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_512B) |
2224                                                  NUM_BANKS(ADDR_SURF_16_BANK) |
2225                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2226                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2227                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_2));
2228                                 break;
2229                         case 25:  /* 128 bpp PRT */
2230                                 gb_tile_moden = (ARRAY_MODE(ARRAY_2D_TILED_THIN1) |
2231                                                  MICRO_TILE_MODE(ADDR_SURF_THIN_MICRO_TILING) |
2232                                                  PIPE_CONFIG(ADDR_SURF_P8_32x32_8x16) |
2233                                                  TILE_SPLIT(ADDR_SURF_TILE_SPLIT_1KB) |
2234                                                  NUM_BANKS(ADDR_SURF_8_BANK) |
2235                                                  BANK_WIDTH(ADDR_SURF_BANK_WIDTH_1) |
2236                                                  BANK_HEIGHT(ADDR_SURF_BANK_HEIGHT_1) |
2237                                                  MACRO_TILE_ASPECT(ADDR_SURF_MACRO_ASPECT_1));
2238                                 break;
2239                         default:
2240                                 gb_tile_moden = 0;
2241                                 break;
2242                         }
2243                         rdev->config.si.tile_mode_array[reg_offset] = gb_tile_moden;
2244                         WREG32(GB_TILE_MODE0 + (reg_offset * 4), gb_tile_moden);
2245                 }
2246         } else
2247                 DRM_ERROR("unknown asic: 0x%x\n", rdev->family);
2248 }
2249
2250 static void si_select_se_sh(struct radeon_device *rdev,
2251                             u32 se_num, u32 sh_num)
2252 {
2253         u32 data = INSTANCE_BROADCAST_WRITES;
2254
2255         if ((se_num == 0xffffffff) && (sh_num == 0xffffffff))
2256                 data |= SH_BROADCAST_WRITES | SE_BROADCAST_WRITES;
2257         else if (se_num == 0xffffffff)
2258                 data |= SE_BROADCAST_WRITES | SH_INDEX(sh_num);
2259         else if (sh_num == 0xffffffff)
2260                 data |= SH_BROADCAST_WRITES | SE_INDEX(se_num);
2261         else
2262                 data |= SH_INDEX(sh_num) | SE_INDEX(se_num);
2263         WREG32(GRBM_GFX_INDEX, data);
2264 }
2265
2266 static u32 si_create_bitmask(u32 bit_width)
2267 {
2268         u32 i, mask = 0;
2269
2270         for (i = 0; i < bit_width; i++) {
2271                 mask <<= 1;
2272                 mask |= 1;
2273         }
2274         return mask;
2275 }
2276
2277 static u32 si_get_cu_enabled(struct radeon_device *rdev, u32 cu_per_sh)
2278 {
2279         u32 data, mask;
2280
2281         data = RREG32(CC_GC_SHADER_ARRAY_CONFIG);
2282         if (data & 1)
2283                 data &= INACTIVE_CUS_MASK;
2284         else
2285                 data = 0;
2286         data |= RREG32(GC_USER_SHADER_ARRAY_CONFIG);
2287
2288         data >>= INACTIVE_CUS_SHIFT;
2289
2290         mask = si_create_bitmask(cu_per_sh);
2291
2292         return ~data & mask;
2293 }
2294
2295 static void si_setup_spi(struct radeon_device *rdev,
2296                          u32 se_num, u32 sh_per_se,
2297                          u32 cu_per_sh)
2298 {
2299         int i, j, k;
2300         u32 data, mask, active_cu;
2301
2302         for (i = 0; i < se_num; i++) {
2303                 for (j = 0; j < sh_per_se; j++) {
2304                         si_select_se_sh(rdev, i, j);
2305                         data = RREG32(SPI_STATIC_THREAD_MGMT_3);
2306                         active_cu = si_get_cu_enabled(rdev, cu_per_sh);
2307
2308                         mask = 1;
2309                         for (k = 0; k < 16; k++) {
2310                                 mask <<= k;
2311                                 if (active_cu & mask) {
2312                                         data &= ~mask;
2313                                         WREG32(SPI_STATIC_THREAD_MGMT_3, data);
2314                                         break;
2315                                 }
2316                         }
2317                 }
2318         }
2319         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2320 }
2321
2322 static u32 si_get_rb_disabled(struct radeon_device *rdev,
2323                               u32 max_rb_num, u32 se_num,
2324                               u32 sh_per_se)
2325 {
2326         u32 data, mask;
2327
2328         data = RREG32(CC_RB_BACKEND_DISABLE);
2329         if (data & 1)
2330                 data &= BACKEND_DISABLE_MASK;
2331         else
2332                 data = 0;
2333         data |= RREG32(GC_USER_RB_BACKEND_DISABLE);
2334
2335         data >>= BACKEND_DISABLE_SHIFT;
2336
2337         mask = si_create_bitmask(max_rb_num / se_num / sh_per_se);
2338
2339         return data & mask;
2340 }
2341
2342 static void si_setup_rb(struct radeon_device *rdev,
2343                         u32 se_num, u32 sh_per_se,
2344                         u32 max_rb_num)
2345 {
2346         int i, j;
2347         u32 data, mask;
2348         u32 disabled_rbs = 0;
2349         u32 enabled_rbs = 0;
2350
2351         for (i = 0; i < se_num; i++) {
2352                 for (j = 0; j < sh_per_se; j++) {
2353                         si_select_se_sh(rdev, i, j);
2354                         data = si_get_rb_disabled(rdev, max_rb_num, se_num, sh_per_se);
2355                         disabled_rbs |= data << ((i * sh_per_se + j) * TAHITI_RB_BITMAP_WIDTH_PER_SH);
2356                 }
2357         }
2358         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2359
2360         mask = 1;
2361         for (i = 0; i < max_rb_num; i++) {
2362                 if (!(disabled_rbs & mask))
2363                         enabled_rbs |= mask;
2364                 mask <<= 1;
2365         }
2366
2367         for (i = 0; i < se_num; i++) {
2368                 si_select_se_sh(rdev, i, 0xffffffff);
2369                 data = 0;
2370                 for (j = 0; j < sh_per_se; j++) {
2371                         switch (enabled_rbs & 3) {
2372                         case 1:
2373                                 data |= (RASTER_CONFIG_RB_MAP_0 << (i * sh_per_se + j) * 2);
2374                                 break;
2375                         case 2:
2376                                 data |= (RASTER_CONFIG_RB_MAP_3 << (i * sh_per_se + j) * 2);
2377                                 break;
2378                         case 3:
2379                         default:
2380                                 data |= (RASTER_CONFIG_RB_MAP_2 << (i * sh_per_se + j) * 2);
2381                                 break;
2382                         }
2383                         enabled_rbs >>= 2;
2384                 }
2385                 WREG32(PA_SC_RASTER_CONFIG, data);
2386         }
2387         si_select_se_sh(rdev, 0xffffffff, 0xffffffff);
2388 }
2389
2390 static void si_gpu_init(struct radeon_device *rdev)
2391 {
2392         u32 gb_addr_config = 0;
2393         u32 mc_shared_chmap, mc_arb_ramcfg;
2394         u32 sx_debug_1;
2395         u32 hdp_host_path_cntl;
2396         u32 tmp;
2397         int i, j;
2398
2399         switch (rdev->family) {
2400         case CHIP_TAHITI:
2401                 rdev->config.si.max_shader_engines = 2;
2402                 rdev->config.si.max_tile_pipes = 12;
2403                 rdev->config.si.max_cu_per_sh = 8;
2404                 rdev->config.si.max_sh_per_se = 2;
2405                 rdev->config.si.max_backends_per_se = 4;
2406                 rdev->config.si.max_texture_channel_caches = 12;
2407                 rdev->config.si.max_gprs = 256;
2408                 rdev->config.si.max_gs_threads = 32;
2409                 rdev->config.si.max_hw_contexts = 8;
2410
2411                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2412                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2413                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2414                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2415                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2416                 break;
2417         case CHIP_PITCAIRN:
2418                 rdev->config.si.max_shader_engines = 2;
2419                 rdev->config.si.max_tile_pipes = 8;
2420                 rdev->config.si.max_cu_per_sh = 5;
2421                 rdev->config.si.max_sh_per_se = 2;
2422                 rdev->config.si.max_backends_per_se = 4;
2423                 rdev->config.si.max_texture_channel_caches = 8;
2424                 rdev->config.si.max_gprs = 256;
2425                 rdev->config.si.max_gs_threads = 32;
2426                 rdev->config.si.max_hw_contexts = 8;
2427
2428                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2429                 rdev->config.si.sc_prim_fifo_size_backend = 0x100;
2430                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2431                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2432                 gb_addr_config = TAHITI_GB_ADDR_CONFIG_GOLDEN;
2433                 break;
2434         case CHIP_VERDE:
2435         default:
2436                 rdev->config.si.max_shader_engines = 1;
2437                 rdev->config.si.max_tile_pipes = 4;
2438                 rdev->config.si.max_cu_per_sh = 2;
2439                 rdev->config.si.max_sh_per_se = 2;
2440                 rdev->config.si.max_backends_per_se = 4;
2441                 rdev->config.si.max_texture_channel_caches = 4;
2442                 rdev->config.si.max_gprs = 256;
2443                 rdev->config.si.max_gs_threads = 32;
2444                 rdev->config.si.max_hw_contexts = 8;
2445
2446                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2447                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2448                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2449                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2450                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2451                 break;
2452         case CHIP_OLAND:
2453                 rdev->config.si.max_shader_engines = 1;
2454                 rdev->config.si.max_tile_pipes = 4;
2455                 rdev->config.si.max_cu_per_sh = 6;
2456                 rdev->config.si.max_sh_per_se = 1;
2457                 rdev->config.si.max_backends_per_se = 2;
2458                 rdev->config.si.max_texture_channel_caches = 4;
2459                 rdev->config.si.max_gprs = 256;
2460                 rdev->config.si.max_gs_threads = 16;
2461                 rdev->config.si.max_hw_contexts = 8;
2462
2463                 rdev->config.si.sc_prim_fifo_size_frontend = 0x20;
2464                 rdev->config.si.sc_prim_fifo_size_backend = 0x40;
2465                 rdev->config.si.sc_hiz_tile_fifo_size = 0x30;
2466                 rdev->config.si.sc_earlyz_tile_fifo_size = 0x130;
2467                 gb_addr_config = VERDE_GB_ADDR_CONFIG_GOLDEN;
2468                 break;
2469         }
2470
2471         /* Initialize HDP */
2472         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2473                 WREG32((0x2c14 + j), 0x00000000);
2474                 WREG32((0x2c18 + j), 0x00000000);
2475                 WREG32((0x2c1c + j), 0x00000000);
2476                 WREG32((0x2c20 + j), 0x00000000);
2477                 WREG32((0x2c24 + j), 0x00000000);
2478         }
2479
2480         WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
2481
2482         evergreen_fix_pci_max_read_req_size(rdev);
2483
2484         WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2485
2486         mc_shared_chmap = RREG32(MC_SHARED_CHMAP);
2487         mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
2488
2489         rdev->config.si.num_tile_pipes = rdev->config.si.max_tile_pipes;
2490         rdev->config.si.mem_max_burst_length_bytes = 256;
2491         tmp = (mc_arb_ramcfg & NOOFCOLS_MASK) >> NOOFCOLS_SHIFT;
2492         rdev->config.si.mem_row_size_in_kb = (4 * (1 << (8 + tmp))) / 1024;
2493         if (rdev->config.si.mem_row_size_in_kb > 4)
2494                 rdev->config.si.mem_row_size_in_kb = 4;
2495         /* XXX use MC settings? */
2496         rdev->config.si.shader_engine_tile_size = 32;
2497         rdev->config.si.num_gpus = 1;
2498         rdev->config.si.multi_gpu_tile_size = 64;
2499
2500         /* fix up row size */
2501         gb_addr_config &= ~ROW_SIZE_MASK;
2502         switch (rdev->config.si.mem_row_size_in_kb) {
2503         case 1:
2504         default:
2505                 gb_addr_config |= ROW_SIZE(0);
2506                 break;
2507         case 2:
2508                 gb_addr_config |= ROW_SIZE(1);
2509                 break;
2510         case 4:
2511                 gb_addr_config |= ROW_SIZE(2);
2512                 break;
2513         }
2514
2515         /* setup tiling info dword.  gb_addr_config is not adequate since it does
2516          * not have bank info, so create a custom tiling dword.
2517          * bits 3:0   num_pipes
2518          * bits 7:4   num_banks
2519          * bits 11:8  group_size
2520          * bits 15:12 row_size
2521          */
2522         rdev->config.si.tile_config = 0;
2523         switch (rdev->config.si.num_tile_pipes) {
2524         case 1:
2525                 rdev->config.si.tile_config |= (0 << 0);
2526                 break;
2527         case 2:
2528                 rdev->config.si.tile_config |= (1 << 0);
2529                 break;
2530         case 4:
2531                 rdev->config.si.tile_config |= (2 << 0);
2532                 break;
2533         case 8:
2534         default:
2535                 /* XXX what about 12? */
2536                 rdev->config.si.tile_config |= (3 << 0);
2537                 break;
2538         }
2539         switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
2540         case 0: /* four banks */
2541                 rdev->config.si.tile_config |= 0 << 4;
2542                 break;
2543         case 1: /* eight banks */
2544                 rdev->config.si.tile_config |= 1 << 4;
2545                 break;
2546         case 2: /* sixteen banks */
2547         default:
2548                 rdev->config.si.tile_config |= 2 << 4;
2549                 break;
2550         }
2551         rdev->config.si.tile_config |=
2552                 ((gb_addr_config & PIPE_INTERLEAVE_SIZE_MASK) >> PIPE_INTERLEAVE_SIZE_SHIFT) << 8;
2553         rdev->config.si.tile_config |=
2554                 ((gb_addr_config & ROW_SIZE_MASK) >> ROW_SIZE_SHIFT) << 12;
2555
2556         WREG32(GB_ADDR_CONFIG, gb_addr_config);
2557         WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
2558         WREG32(DMIF_ADDR_CALC, gb_addr_config);
2559         WREG32(HDP_ADDR_CONFIG, gb_addr_config);
2560         WREG32(DMA_TILING_CONFIG + DMA0_REGISTER_OFFSET, gb_addr_config);
2561         WREG32(DMA_TILING_CONFIG + DMA1_REGISTER_OFFSET, gb_addr_config);
2562         WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
2563         WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
2564         WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
2565
2566         si_tiling_mode_table_init(rdev);
2567
2568         si_setup_rb(rdev, rdev->config.si.max_shader_engines,
2569                     rdev->config.si.max_sh_per_se,
2570                     rdev->config.si.max_backends_per_se);
2571
2572         si_setup_spi(rdev, rdev->config.si.max_shader_engines,
2573                      rdev->config.si.max_sh_per_se,
2574                      rdev->config.si.max_cu_per_sh);
2575
2576
2577         /* set HW defaults for 3D engine */
2578         WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
2579                                      ROQ_IB2_START(0x2b)));
2580         WREG32(CP_MEQ_THRESHOLDS, MEQ1_START(0x30) | MEQ2_START(0x60));
2581
2582         sx_debug_1 = RREG32(SX_DEBUG_1);
2583         WREG32(SX_DEBUG_1, sx_debug_1);
2584
2585         WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
2586
2587         WREG32(PA_SC_FIFO_SIZE, (SC_FRONTEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_frontend) |
2588                                  SC_BACKEND_PRIM_FIFO_SIZE(rdev->config.si.sc_prim_fifo_size_backend) |
2589                                  SC_HIZ_TILE_FIFO_SIZE(rdev->config.si.sc_hiz_tile_fifo_size) |
2590                                  SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.si.sc_earlyz_tile_fifo_size)));
2591
2592         WREG32(VGT_NUM_INSTANCES, 1);
2593
2594         WREG32(CP_PERFMON_CNTL, 0);
2595
2596         WREG32(SQ_CONFIG, 0);
2597
2598         WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
2599                                           FORCE_EOV_MAX_REZ_CNT(255)));
2600
2601         WREG32(VGT_CACHE_INVALIDATION, CACHE_INVALIDATION(VC_AND_TC) |
2602                AUTO_INVLD_EN(ES_AND_GS_AUTO));
2603
2604         WREG32(VGT_GS_VERTEX_REUSE, 16);
2605         WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
2606
2607         WREG32(CB_PERFCOUNTER0_SELECT0, 0);
2608         WREG32(CB_PERFCOUNTER0_SELECT1, 0);
2609         WREG32(CB_PERFCOUNTER1_SELECT0, 0);
2610         WREG32(CB_PERFCOUNTER1_SELECT1, 0);
2611         WREG32(CB_PERFCOUNTER2_SELECT0, 0);
2612         WREG32(CB_PERFCOUNTER2_SELECT1, 0);
2613         WREG32(CB_PERFCOUNTER3_SELECT0, 0);
2614         WREG32(CB_PERFCOUNTER3_SELECT1, 0);
2615
2616         tmp = RREG32(HDP_MISC_CNTL);
2617         tmp |= HDP_FLUSH_INVALIDATE_CACHE;
2618         WREG32(HDP_MISC_CNTL, tmp);
2619
2620         hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
2621         WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
2622
2623         WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
2624
2625         udelay(50);
2626 }
2627
2628 /*
2629  * GPU scratch registers helpers function.
2630  */
2631 static void si_scratch_init(struct radeon_device *rdev)
2632 {
2633         int i;
2634
2635         rdev->scratch.num_reg = 7;
2636         rdev->scratch.reg_base = SCRATCH_REG0;
2637         for (i = 0; i < rdev->scratch.num_reg; i++) {
2638                 rdev->scratch.free[i] = true;
2639                 rdev->scratch.reg[i] = rdev->scratch.reg_base + (i * 4);
2640         }
2641 }
2642
2643 void si_fence_ring_emit(struct radeon_device *rdev,
2644                         struct radeon_fence *fence)
2645 {
2646         struct radeon_ring *ring = &rdev->ring[fence->ring];
2647         u64 addr = rdev->fence_drv[fence->ring].gpu_addr;
2648
2649         /* flush read cache over gart */
2650         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2651         radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2652         radeon_ring_write(ring, 0);
2653         radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2654         radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2655                           PACKET3_TC_ACTION_ENA |
2656                           PACKET3_SH_KCACHE_ACTION_ENA |
2657                           PACKET3_SH_ICACHE_ACTION_ENA);
2658         radeon_ring_write(ring, 0xFFFFFFFF);
2659         radeon_ring_write(ring, 0);
2660         radeon_ring_write(ring, 10); /* poll interval */
2661         /* EVENT_WRITE_EOP - flush caches, send int */
2662         radeon_ring_write(ring, PACKET3(PACKET3_EVENT_WRITE_EOP, 4));
2663         radeon_ring_write(ring, EVENT_TYPE(CACHE_FLUSH_AND_INV_TS_EVENT) | EVENT_INDEX(5));
2664         radeon_ring_write(ring, addr & 0xffffffff);
2665         radeon_ring_write(ring, (upper_32_bits(addr) & 0xff) | DATA_SEL(1) | INT_SEL(2));
2666         radeon_ring_write(ring, fence->seq);
2667         radeon_ring_write(ring, 0);
2668 }
2669
2670 /*
2671  * IB stuff
2672  */
2673 void si_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2674 {
2675         struct radeon_ring *ring = &rdev->ring[ib->ring];
2676         u32 header;
2677
2678         if (ib->is_const_ib) {
2679                 /* set switch buffer packet before const IB */
2680                 radeon_ring_write(ring, PACKET3(PACKET3_SWITCH_BUFFER, 0));
2681                 radeon_ring_write(ring, 0);
2682
2683                 header = PACKET3(PACKET3_INDIRECT_BUFFER_CONST, 2);
2684         } else {
2685                 u32 next_rptr;
2686                 if (ring->rptr_save_reg) {
2687                         next_rptr = ring->wptr + 3 + 4 + 8;
2688                         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2689                         radeon_ring_write(ring, ((ring->rptr_save_reg -
2690                                                   PACKET3_SET_CONFIG_REG_START) >> 2));
2691                         radeon_ring_write(ring, next_rptr);
2692                 } else if (rdev->wb.enabled) {
2693                         next_rptr = ring->wptr + 5 + 4 + 8;
2694                         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
2695                         radeon_ring_write(ring, (1 << 8));
2696                         radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2697                         radeon_ring_write(ring, upper_32_bits(ring->next_rptr_gpu_addr) & 0xffffffff);
2698                         radeon_ring_write(ring, next_rptr);
2699                 }
2700
2701                 header = PACKET3(PACKET3_INDIRECT_BUFFER, 2);
2702         }
2703
2704         radeon_ring_write(ring, header);
2705         radeon_ring_write(ring,
2706 #ifdef __BIG_ENDIAN
2707                           (2 << 0) |
2708 #endif
2709                           (ib->gpu_addr & 0xFFFFFFFC));
2710         radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFFFF);
2711         radeon_ring_write(ring, ib->length_dw |
2712                           (ib->vm ? (ib->vm->id << 24) : 0));
2713
2714         if (!ib->is_const_ib) {
2715                 /* flush read cache over gart for this vmid */
2716                 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2717                 radeon_ring_write(ring, (CP_COHER_CNTL2 - PACKET3_SET_CONFIG_REG_START) >> 2);
2718                 radeon_ring_write(ring, ib->vm ? ib->vm->id : 0);
2719                 radeon_ring_write(ring, PACKET3(PACKET3_SURFACE_SYNC, 3));
2720                 radeon_ring_write(ring, PACKET3_TCL1_ACTION_ENA |
2721                                   PACKET3_TC_ACTION_ENA |
2722                                   PACKET3_SH_KCACHE_ACTION_ENA |
2723                                   PACKET3_SH_ICACHE_ACTION_ENA);
2724                 radeon_ring_write(ring, 0xFFFFFFFF);
2725                 radeon_ring_write(ring, 0);
2726                 radeon_ring_write(ring, 10); /* poll interval */
2727         }
2728 }
2729
2730 /*
2731  * CP.
2732  */
2733 static void si_cp_enable(struct radeon_device *rdev, bool enable)
2734 {
2735         if (enable)
2736                 WREG32(CP_ME_CNTL, 0);
2737         else {
2738                 radeon_ttm_set_active_vram_size(rdev, rdev->mc.visible_vram_size);
2739                 WREG32(CP_ME_CNTL, (CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT));
2740                 WREG32(SCRATCH_UMSK, 0);
2741                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2742                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2743                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2744         }
2745         udelay(50);
2746 }
2747
2748 static int si_cp_load_microcode(struct radeon_device *rdev)
2749 {
2750         const __be32 *fw_data;
2751         int i;
2752
2753         if (!rdev->me_fw || !rdev->pfp_fw)
2754                 return -EINVAL;
2755
2756         si_cp_enable(rdev, false);
2757
2758         /* PFP */
2759         fw_data = (const __be32 *)rdev->pfp_fw->data;
2760         WREG32(CP_PFP_UCODE_ADDR, 0);
2761         for (i = 0; i < SI_PFP_UCODE_SIZE; i++)
2762                 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2763         WREG32(CP_PFP_UCODE_ADDR, 0);
2764
2765         /* CE */
2766         fw_data = (const __be32 *)rdev->ce_fw->data;
2767         WREG32(CP_CE_UCODE_ADDR, 0);
2768         for (i = 0; i < SI_CE_UCODE_SIZE; i++)
2769                 WREG32(CP_CE_UCODE_DATA, be32_to_cpup(fw_data++));
2770         WREG32(CP_CE_UCODE_ADDR, 0);
2771
2772         /* ME */
2773         fw_data = (const __be32 *)rdev->me_fw->data;
2774         WREG32(CP_ME_RAM_WADDR, 0);
2775         for (i = 0; i < SI_PM4_UCODE_SIZE; i++)
2776                 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2777         WREG32(CP_ME_RAM_WADDR, 0);
2778
2779         WREG32(CP_PFP_UCODE_ADDR, 0);
2780         WREG32(CP_CE_UCODE_ADDR, 0);
2781         WREG32(CP_ME_RAM_WADDR, 0);
2782         WREG32(CP_ME_RAM_RADDR, 0);
2783         return 0;
2784 }
2785
2786 static int si_cp_start(struct radeon_device *rdev)
2787 {
2788         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2789         int r, i;
2790
2791         r = radeon_ring_lock(rdev, ring, 7 + 4);
2792         if (r) {
2793                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2794                 return r;
2795         }
2796         /* init the CP */
2797         radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
2798         radeon_ring_write(ring, 0x1);
2799         radeon_ring_write(ring, 0x0);
2800         radeon_ring_write(ring, rdev->config.si.max_hw_contexts - 1);
2801         radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
2802         radeon_ring_write(ring, 0);
2803         radeon_ring_write(ring, 0);
2804
2805         /* init the CE partitions */
2806         radeon_ring_write(ring, PACKET3(PACKET3_SET_BASE, 2));
2807         radeon_ring_write(ring, PACKET3_BASE_INDEX(CE_PARTITION_BASE));
2808         radeon_ring_write(ring, 0xc000);
2809         radeon_ring_write(ring, 0xe000);
2810         radeon_ring_unlock_commit(rdev, ring);
2811
2812         si_cp_enable(rdev, true);
2813
2814         r = radeon_ring_lock(rdev, ring, si_default_size + 10);
2815         if (r) {
2816                 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
2817                 return r;
2818         }
2819
2820         /* setup clear context state */
2821         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2822         radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
2823
2824         for (i = 0; i < si_default_size; i++)
2825                 radeon_ring_write(ring, si_default_state[i]);
2826
2827         radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
2828         radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
2829
2830         /* set clear context state */
2831         radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
2832         radeon_ring_write(ring, 0);
2833
2834         radeon_ring_write(ring, PACKET3(PACKET3_SET_CONTEXT_REG, 2));
2835         radeon_ring_write(ring, 0x00000316);
2836         radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
2837         radeon_ring_write(ring, 0x00000010); /* VGT_OUT_DEALLOC_CNTL */
2838
2839         radeon_ring_unlock_commit(rdev, ring);
2840
2841         for (i = RADEON_RING_TYPE_GFX_INDEX; i <= CAYMAN_RING_TYPE_CP2_INDEX; ++i) {
2842                 ring = &rdev->ring[i];
2843                 r = radeon_ring_lock(rdev, ring, 2);
2844
2845                 /* clear the compute context state */
2846                 radeon_ring_write(ring, PACKET3_COMPUTE(PACKET3_CLEAR_STATE, 0));
2847                 radeon_ring_write(ring, 0);
2848
2849                 radeon_ring_unlock_commit(rdev, ring);
2850         }
2851
2852         return 0;
2853 }
2854
2855 static void si_cp_fini(struct radeon_device *rdev)
2856 {
2857         struct radeon_ring *ring;
2858         si_cp_enable(rdev, false);
2859
2860         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2861         radeon_ring_fini(rdev, ring);
2862         radeon_scratch_free(rdev, ring->rptr_save_reg);
2863
2864         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2865         radeon_ring_fini(rdev, ring);
2866         radeon_scratch_free(rdev, ring->rptr_save_reg);
2867
2868         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2869         radeon_ring_fini(rdev, ring);
2870         radeon_scratch_free(rdev, ring->rptr_save_reg);
2871 }
2872
2873 static int si_cp_resume(struct radeon_device *rdev)
2874 {
2875         struct radeon_ring *ring;
2876         u32 tmp;
2877         u32 rb_bufsz;
2878         int r;
2879
2880         /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
2881         WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
2882                                  SOFT_RESET_PA |
2883                                  SOFT_RESET_VGT |
2884                                  SOFT_RESET_SPI |
2885                                  SOFT_RESET_SX));
2886         RREG32(GRBM_SOFT_RESET);
2887         mdelay(15);
2888         WREG32(GRBM_SOFT_RESET, 0);
2889         RREG32(GRBM_SOFT_RESET);
2890
2891         WREG32(CP_SEM_WAIT_TIMER, 0x0);
2892         WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
2893
2894         /* Set the write pointer delay */
2895         WREG32(CP_RB_WPTR_DELAY, 0);
2896
2897         WREG32(CP_DEBUG, 0);
2898         WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
2899
2900         /* ring 0 - compute and gfx */
2901         /* Set ring buffer size */
2902         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
2903         rb_bufsz = drm_order(ring->ring_size / 8);
2904         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2905 #ifdef __BIG_ENDIAN
2906         tmp |= BUF_SWAP_32BIT;
2907 #endif
2908         WREG32(CP_RB0_CNTL, tmp);
2909
2910         /* Initialize the ring buffer's read and write pointers */
2911         WREG32(CP_RB0_CNTL, tmp | RB_RPTR_WR_ENA);
2912         ring->wptr = 0;
2913         WREG32(CP_RB0_WPTR, ring->wptr);
2914
2915         /* set the wb address whether it's enabled or not */
2916         WREG32(CP_RB0_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC);
2917         WREG32(CP_RB0_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
2918
2919         if (rdev->wb.enabled)
2920                 WREG32(SCRATCH_UMSK, 0xff);
2921         else {
2922                 tmp |= RB_NO_UPDATE;
2923                 WREG32(SCRATCH_UMSK, 0);
2924         }
2925
2926         mdelay(1);
2927         WREG32(CP_RB0_CNTL, tmp);
2928
2929         WREG32(CP_RB0_BASE, ring->gpu_addr >> 8);
2930
2931         ring->rptr = RREG32(CP_RB0_RPTR);
2932
2933         /* ring1  - compute only */
2934         /* Set ring buffer size */
2935         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
2936         rb_bufsz = drm_order(ring->ring_size / 8);
2937         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2938 #ifdef __BIG_ENDIAN
2939         tmp |= BUF_SWAP_32BIT;
2940 #endif
2941         WREG32(CP_RB1_CNTL, tmp);
2942
2943         /* Initialize the ring buffer's read and write pointers */
2944         WREG32(CP_RB1_CNTL, tmp | RB_RPTR_WR_ENA);
2945         ring->wptr = 0;
2946         WREG32(CP_RB1_WPTR, ring->wptr);
2947
2948         /* set the wb address whether it's enabled or not */
2949         WREG32(CP_RB1_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFFFFFFFC);
2950         WREG32(CP_RB1_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP1_RPTR_OFFSET) & 0xFF);
2951
2952         mdelay(1);
2953         WREG32(CP_RB1_CNTL, tmp);
2954
2955         WREG32(CP_RB1_BASE, ring->gpu_addr >> 8);
2956
2957         ring->rptr = RREG32(CP_RB1_RPTR);
2958
2959         /* ring2 - compute only */
2960         /* Set ring buffer size */
2961         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
2962         rb_bufsz = drm_order(ring->ring_size / 8);
2963         tmp = (drm_order(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
2964 #ifdef __BIG_ENDIAN
2965         tmp |= BUF_SWAP_32BIT;
2966 #endif
2967         WREG32(CP_RB2_CNTL, tmp);
2968
2969         /* Initialize the ring buffer's read and write pointers */
2970         WREG32(CP_RB2_CNTL, tmp | RB_RPTR_WR_ENA);
2971         ring->wptr = 0;
2972         WREG32(CP_RB2_WPTR, ring->wptr);
2973
2974         /* set the wb address whether it's enabled or not */
2975         WREG32(CP_RB2_RPTR_ADDR, (rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFFFFFFFC);
2976         WREG32(CP_RB2_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP2_RPTR_OFFSET) & 0xFF);
2977
2978         mdelay(1);
2979         WREG32(CP_RB2_CNTL, tmp);
2980
2981         WREG32(CP_RB2_BASE, ring->gpu_addr >> 8);
2982
2983         ring->rptr = RREG32(CP_RB2_RPTR);
2984
2985         /* start the rings */
2986         si_cp_start(rdev);
2987         rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = true;
2988         rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = true;
2989         rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = true;
2990         r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX]);
2991         if (r) {
2992                 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ready = false;
2993                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
2994                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
2995                 return r;
2996         }
2997         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP1_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX]);
2998         if (r) {
2999                 rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX].ready = false;
3000         }
3001         r = radeon_ring_test(rdev, CAYMAN_RING_TYPE_CP2_INDEX, &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX]);
3002         if (r) {
3003                 rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX].ready = false;
3004         }
3005
3006         return 0;
3007 }
3008
3009 static u32 si_gpu_check_soft_reset(struct radeon_device *rdev)
3010 {
3011         u32 reset_mask = 0;
3012         u32 tmp;
3013
3014         /* GRBM_STATUS */
3015         tmp = RREG32(GRBM_STATUS);
3016         if (tmp & (PA_BUSY | SC_BUSY |
3017                    BCI_BUSY | SX_BUSY |
3018                    TA_BUSY | VGT_BUSY |
3019                    DB_BUSY | CB_BUSY |
3020                    GDS_BUSY | SPI_BUSY |
3021                    IA_BUSY | IA_BUSY_NO_DMA))
3022                 reset_mask |= RADEON_RESET_GFX;
3023
3024         if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3025                    CP_BUSY | CP_COHERENCY_BUSY))
3026                 reset_mask |= RADEON_RESET_CP;
3027
3028         if (tmp & GRBM_EE_BUSY)
3029                 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3030
3031         /* GRBM_STATUS2 */
3032         tmp = RREG32(GRBM_STATUS2);
3033         if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3034                 reset_mask |= RADEON_RESET_RLC;
3035
3036         /* DMA_STATUS_REG 0 */
3037         tmp = RREG32(DMA_STATUS_REG + DMA0_REGISTER_OFFSET);
3038         if (!(tmp & DMA_IDLE))
3039                 reset_mask |= RADEON_RESET_DMA;
3040
3041         /* DMA_STATUS_REG 1 */
3042         tmp = RREG32(DMA_STATUS_REG + DMA1_REGISTER_OFFSET);
3043         if (!(tmp & DMA_IDLE))
3044                 reset_mask |= RADEON_RESET_DMA1;
3045
3046         /* SRBM_STATUS2 */
3047         tmp = RREG32(SRBM_STATUS2);
3048         if (tmp & DMA_BUSY)
3049                 reset_mask |= RADEON_RESET_DMA;
3050
3051         if (tmp & DMA1_BUSY)
3052                 reset_mask |= RADEON_RESET_DMA1;
3053
3054         /* SRBM_STATUS */
3055         tmp = RREG32(SRBM_STATUS);
3056
3057         if (tmp & IH_BUSY)
3058                 reset_mask |= RADEON_RESET_IH;
3059
3060         if (tmp & SEM_BUSY)
3061                 reset_mask |= RADEON_RESET_SEM;
3062
3063         if (tmp & GRBM_RQ_PENDING)
3064                 reset_mask |= RADEON_RESET_GRBM;
3065
3066         if (tmp & VMC_BUSY)
3067                 reset_mask |= RADEON_RESET_VMC;
3068
3069         if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3070                    MCC_BUSY | MCD_BUSY))
3071                 reset_mask |= RADEON_RESET_MC;
3072
3073         if (evergreen_is_display_hung(rdev))
3074                 reset_mask |= RADEON_RESET_DISPLAY;
3075
3076         /* VM_L2_STATUS */
3077         tmp = RREG32(VM_L2_STATUS);
3078         if (tmp & L2_BUSY)
3079                 reset_mask |= RADEON_RESET_VMC;
3080
3081         /* Skip MC reset as it's mostly likely not hung, just busy */
3082         if (reset_mask & RADEON_RESET_MC) {
3083                 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3084                 reset_mask &= ~RADEON_RESET_MC;
3085         }
3086
3087         return reset_mask;
3088 }
3089
3090 static void si_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3091 {
3092         struct evergreen_mc_save save;
3093         u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3094         u32 tmp;
3095
3096         if (reset_mask == 0)
3097                 return;
3098
3099         dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3100
3101         evergreen_print_gpu_status_regs(rdev);
3102         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
3103                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
3104         dev_info(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
3105                  RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
3106
3107         /* Disable CP parsing/prefetching */
3108         WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT | CP_CE_HALT);
3109
3110         if (reset_mask & RADEON_RESET_DMA) {
3111                 /* dma0 */
3112                 tmp = RREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET);
3113                 tmp &= ~DMA_RB_ENABLE;
3114                 WREG32(DMA_RB_CNTL + DMA0_REGISTER_OFFSET, tmp);
3115         }
3116         if (reset_mask & RADEON_RESET_DMA1) {
3117                 /* dma1 */
3118                 tmp = RREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET);
3119                 tmp &= ~DMA_RB_ENABLE;
3120                 WREG32(DMA_RB_CNTL + DMA1_REGISTER_OFFSET, tmp);
3121         }
3122
3123         udelay(50);
3124
3125         evergreen_mc_stop(rdev, &save);
3126         if (evergreen_mc_wait_for_idle(rdev)) {
3127                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3128         }
3129
3130         if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE | RADEON_RESET_CP)) {
3131                 grbm_soft_reset = SOFT_RESET_CB |
3132                         SOFT_RESET_DB |
3133                         SOFT_RESET_GDS |
3134                         SOFT_RESET_PA |
3135                         SOFT_RESET_SC |
3136                         SOFT_RESET_BCI |
3137                         SOFT_RESET_SPI |
3138                         SOFT_RESET_SX |
3139                         SOFT_RESET_TC |
3140                         SOFT_RESET_TA |
3141                         SOFT_RESET_VGT |
3142                         SOFT_RESET_IA;
3143         }
3144
3145         if (reset_mask & RADEON_RESET_CP) {
3146                 grbm_soft_reset |= SOFT_RESET_CP | SOFT_RESET_VGT;
3147
3148                 srbm_soft_reset |= SOFT_RESET_GRBM;
3149         }
3150
3151         if (reset_mask & RADEON_RESET_DMA)
3152                 srbm_soft_reset |= SOFT_RESET_DMA;
3153
3154         if (reset_mask & RADEON_RESET_DMA1)
3155                 srbm_soft_reset |= SOFT_RESET_DMA1;
3156
3157         if (reset_mask & RADEON_RESET_DISPLAY)
3158                 srbm_soft_reset |= SOFT_RESET_DC;
3159
3160         if (reset_mask & RADEON_RESET_RLC)
3161                 grbm_soft_reset |= SOFT_RESET_RLC;
3162
3163         if (reset_mask & RADEON_RESET_SEM)
3164                 srbm_soft_reset |= SOFT_RESET_SEM;
3165
3166         if (reset_mask & RADEON_RESET_IH)
3167                 srbm_soft_reset |= SOFT_RESET_IH;
3168
3169         if (reset_mask & RADEON_RESET_GRBM)
3170                 srbm_soft_reset |= SOFT_RESET_GRBM;
3171
3172         if (reset_mask & RADEON_RESET_VMC)
3173                 srbm_soft_reset |= SOFT_RESET_VMC;
3174
3175         if (reset_mask & RADEON_RESET_MC)
3176                 srbm_soft_reset |= SOFT_RESET_MC;
3177
3178         if (grbm_soft_reset) {
3179                 tmp = RREG32(GRBM_SOFT_RESET);
3180                 tmp |= grbm_soft_reset;
3181                 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3182                 WREG32(GRBM_SOFT_RESET, tmp);
3183                 tmp = RREG32(GRBM_SOFT_RESET);
3184
3185                 udelay(50);
3186
3187                 tmp &= ~grbm_soft_reset;
3188                 WREG32(GRBM_SOFT_RESET, tmp);
3189                 tmp = RREG32(GRBM_SOFT_RESET);
3190         }
3191
3192         if (srbm_soft_reset) {
3193                 tmp = RREG32(SRBM_SOFT_RESET);
3194                 tmp |= srbm_soft_reset;
3195                 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3196                 WREG32(SRBM_SOFT_RESET, tmp);
3197                 tmp = RREG32(SRBM_SOFT_RESET);
3198
3199                 udelay(50);
3200
3201                 tmp &= ~srbm_soft_reset;
3202                 WREG32(SRBM_SOFT_RESET, tmp);
3203                 tmp = RREG32(SRBM_SOFT_RESET);
3204         }
3205
3206         /* Wait a little for things to settle down */
3207         udelay(50);
3208
3209         evergreen_mc_resume(rdev, &save);
3210         udelay(50);
3211
3212         evergreen_print_gpu_status_regs(rdev);
3213 }
3214
3215 int si_asic_reset(struct radeon_device *rdev)
3216 {
3217         u32 reset_mask;
3218
3219         reset_mask = si_gpu_check_soft_reset(rdev);
3220
3221         if (reset_mask)
3222                 r600_set_bios_scratch_engine_hung(rdev, true);
3223
3224         si_gpu_soft_reset(rdev, reset_mask);
3225
3226         reset_mask = si_gpu_check_soft_reset(rdev);
3227
3228         if (!reset_mask)
3229                 r600_set_bios_scratch_engine_hung(rdev, false);
3230
3231         return 0;
3232 }
3233
3234 /**
3235  * si_gfx_is_lockup - Check if the GFX engine is locked up
3236  *
3237  * @rdev: radeon_device pointer
3238  * @ring: radeon_ring structure holding ring information
3239  *
3240  * Check if the GFX engine is locked up.
3241  * Returns true if the engine appears to be locked up, false if not.
3242  */
3243 bool si_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3244 {
3245         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3246
3247         if (!(reset_mask & (RADEON_RESET_GFX |
3248                             RADEON_RESET_COMPUTE |
3249                             RADEON_RESET_CP))) {
3250                 radeon_ring_lockup_update(ring);
3251                 return false;
3252         }
3253         /* force CP activities */
3254         radeon_ring_force_activity(rdev, ring);
3255         return radeon_ring_test_lockup(rdev, ring);
3256 }
3257
3258 /**
3259  * si_dma_is_lockup - Check if the DMA engine is locked up
3260  *
3261  * @rdev: radeon_device pointer
3262  * @ring: radeon_ring structure holding ring information
3263  *
3264  * Check if the async DMA engine is locked up.
3265  * Returns true if the engine appears to be locked up, false if not.
3266  */
3267 bool si_dma_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
3268 {
3269         u32 reset_mask = si_gpu_check_soft_reset(rdev);
3270         u32 mask;
3271
3272         if (ring->idx == R600_RING_TYPE_DMA_INDEX)
3273                 mask = RADEON_RESET_DMA;
3274         else
3275                 mask = RADEON_RESET_DMA1;
3276
3277         if (!(reset_mask & mask)) {
3278                 radeon_ring_lockup_update(ring);
3279                 return false;
3280         }
3281         /* force ring activities */
3282         radeon_ring_force_activity(rdev, ring);
3283         return radeon_ring_test_lockup(rdev, ring);
3284 }
3285
3286 /* MC */
3287 static void si_mc_program(struct radeon_device *rdev)
3288 {
3289         struct evergreen_mc_save save;
3290         u32 tmp;
3291         int i, j;
3292
3293         /* Initialize HDP */
3294         for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3295                 WREG32((0x2c14 + j), 0x00000000);
3296                 WREG32((0x2c18 + j), 0x00000000);
3297                 WREG32((0x2c1c + j), 0x00000000);
3298                 WREG32((0x2c20 + j), 0x00000000);
3299                 WREG32((0x2c24 + j), 0x00000000);
3300         }
3301         WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
3302
3303         evergreen_mc_stop(rdev, &save);
3304         if (radeon_mc_wait_for_idle(rdev)) {
3305                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3306         }
3307         /* Lockout access through VGA aperture*/
3308         WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
3309         /* Update configuration */
3310         WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
3311                rdev->mc.vram_start >> 12);
3312         WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
3313                rdev->mc.vram_end >> 12);
3314         WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR,
3315                rdev->vram_scratch.gpu_addr >> 12);
3316         tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
3317         tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
3318         WREG32(MC_VM_FB_LOCATION, tmp);
3319         /* XXX double check these! */
3320         WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
3321         WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
3322         WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
3323         WREG32(MC_VM_AGP_BASE, 0);
3324         WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
3325         WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
3326         if (radeon_mc_wait_for_idle(rdev)) {
3327                 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3328         }
3329         evergreen_mc_resume(rdev, &save);
3330         /* we need to own VRAM, so turn off the VGA renderer here
3331          * to stop it overwriting our objects */
3332         rv515_vga_render_disable(rdev);
3333 }
3334
3335 static void si_vram_gtt_location(struct radeon_device *rdev,
3336                                  struct radeon_mc *mc)
3337 {
3338         if (mc->mc_vram_size > 0xFFC0000000ULL) {
3339                 /* leave room for at least 1024M GTT */
3340                 dev_warn(rdev->dev, "limiting VRAM\n");
3341                 mc->real_vram_size = 0xFFC0000000ULL;
3342                 mc->mc_vram_size = 0xFFC0000000ULL;
3343         }
3344         radeon_vram_location(rdev, &rdev->mc, 0);
3345         rdev->mc.gtt_base_align = 0;
3346         radeon_gtt_location(rdev, mc);
3347 }
3348
3349 static int si_mc_init(struct radeon_device *rdev)
3350 {
3351         u32 tmp;
3352         int chansize, numchan;
3353
3354         /* Get VRAM informations */
3355         rdev->mc.vram_is_ddr = true;
3356         tmp = RREG32(MC_ARB_RAMCFG);
3357         if (tmp & CHANSIZE_OVERRIDE) {
3358                 chansize = 16;
3359         } else if (tmp & CHANSIZE_MASK) {
3360                 chansize = 64;
3361         } else {
3362                 chansize = 32;
3363         }
3364         tmp = RREG32(MC_SHARED_CHMAP);
3365         switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3366         case 0:
3367         default:
3368                 numchan = 1;
3369                 break;
3370         case 1:
3371                 numchan = 2;
3372                 break;
3373         case 2:
3374                 numchan = 4;
3375                 break;
3376         case 3:
3377                 numchan = 8;
3378                 break;
3379         case 4:
3380                 numchan = 3;
3381                 break;
3382         case 5:
3383                 numchan = 6;
3384                 break;
3385         case 6:
3386                 numchan = 10;
3387                 break;
3388         case 7:
3389                 numchan = 12;
3390                 break;
3391         case 8:
3392                 numchan = 16;
3393                 break;
3394         }
3395         rdev->mc.vram_width = numchan * chansize;
3396         /* Could aper size report 0 ? */
3397         rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3398         rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3399         /* size in MB on si */
3400         rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3401         rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024 * 1024;
3402         rdev->mc.visible_vram_size = rdev->mc.aper_size;
3403         si_vram_gtt_location(rdev, &rdev->mc);
3404         radeon_update_bandwidth_info(rdev);
3405
3406         return 0;
3407 }
3408
3409 /*
3410  * GART
3411  */
3412 void si_pcie_gart_tlb_flush(struct radeon_device *rdev)
3413 {
3414         /* flush hdp cache */
3415         WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
3416
3417         /* bits 0-15 are the VM contexts0-15 */
3418         WREG32(VM_INVALIDATE_REQUEST, 1);
3419 }
3420
3421 static int si_pcie_gart_enable(struct radeon_device *rdev)
3422 {
3423         int r, i;
3424
3425         if (rdev->gart.robj == NULL) {
3426                 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
3427                 return -EINVAL;
3428         }
3429         r = radeon_gart_table_vram_pin(rdev);
3430         if (r)
3431                 return r;
3432         radeon_gart_restore(rdev);
3433         /* Setup TLB control */
3434         WREG32(MC_VM_MX_L1_TLB_CNTL,
3435                (0xA << 7) |
3436                ENABLE_L1_TLB |
3437                SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3438                ENABLE_ADVANCED_DRIVER_MODEL |
3439                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3440         /* Setup L2 cache */
3441         WREG32(VM_L2_CNTL, ENABLE_L2_CACHE |
3442                ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3443                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3444                EFFECTIVE_L2_QUEUE_SIZE(7) |
3445                CONTEXT1_IDENTITY_ACCESS_MODE(1));
3446         WREG32(VM_L2_CNTL2, INVALIDATE_ALL_L1_TLBS | INVALIDATE_L2_CACHE);
3447         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3448                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3449         /* setup context0 */
3450         WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
3451         WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
3452         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
3453         WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
3454                         (u32)(rdev->dummy_page.addr >> 12));
3455         WREG32(VM_CONTEXT0_CNTL2, 0);
3456         WREG32(VM_CONTEXT0_CNTL, (ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
3457                                   RANGE_PROTECTION_FAULT_ENABLE_DEFAULT));
3458
3459         WREG32(0x15D4, 0);
3460         WREG32(0x15D8, 0);
3461         WREG32(0x15DC, 0);
3462
3463         /* empty context1-15 */
3464         /* set vm size, must be a multiple of 4 */
3465         WREG32(VM_CONTEXT1_PAGE_TABLE_START_ADDR, 0);
3466         WREG32(VM_CONTEXT1_PAGE_TABLE_END_ADDR, rdev->vm_manager.max_pfn);
3467         /* Assign the pt base to something valid for now; the pts used for
3468          * the VMs are determined by the application and setup and assigned
3469          * on the fly in the vm part of radeon_gart.c
3470          */
3471         for (i = 1; i < 16; i++) {
3472                 if (i < 8)
3473                         WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (i << 2),
3474                                rdev->gart.table_addr >> 12);
3475                 else
3476                         WREG32(VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((i - 8) << 2),
3477                                rdev->gart.table_addr >> 12);
3478         }
3479
3480         /* enable context1-15 */
3481         WREG32(VM_CONTEXT1_PROTECTION_FAULT_DEFAULT_ADDR,
3482                (u32)(rdev->dummy_page.addr >> 12));
3483         WREG32(VM_CONTEXT1_CNTL2, 4);
3484         WREG32(VM_CONTEXT1_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(1) |
3485                                 RANGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3486                                 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3487                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3488                                 DUMMY_PAGE_PROTECTION_FAULT_ENABLE_DEFAULT |
3489                                 PDE0_PROTECTION_FAULT_ENABLE_INTERRUPT |
3490                                 PDE0_PROTECTION_FAULT_ENABLE_DEFAULT |
3491                                 VALID_PROTECTION_FAULT_ENABLE_INTERRUPT |
3492                                 VALID_PROTECTION_FAULT_ENABLE_DEFAULT |
3493                                 READ_PROTECTION_FAULT_ENABLE_INTERRUPT |
3494                                 READ_PROTECTION_FAULT_ENABLE_DEFAULT |
3495                                 WRITE_PROTECTION_FAULT_ENABLE_INTERRUPT |
3496                                 WRITE_PROTECTION_FAULT_ENABLE_DEFAULT);
3497
3498         si_pcie_gart_tlb_flush(rdev);
3499         DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
3500                  (unsigned)(rdev->mc.gtt_size >> 20),
3501                  (unsigned long long)rdev->gart.table_addr);
3502         rdev->gart.ready = true;
3503         return 0;
3504 }
3505
3506 static void si_pcie_gart_disable(struct radeon_device *rdev)
3507 {
3508         /* Disable all tables */
3509         WREG32(VM_CONTEXT0_CNTL, 0);
3510         WREG32(VM_CONTEXT1_CNTL, 0);
3511         /* Setup TLB control */
3512         WREG32(MC_VM_MX_L1_TLB_CNTL, SYSTEM_ACCESS_MODE_NOT_IN_SYS |
3513                SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU);
3514         /* Setup L2 cache */
3515         WREG32(VM_L2_CNTL, ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
3516                ENABLE_L2_PDE0_CACHE_LRU_UPDATE_BY_WRITE |
3517                EFFECTIVE_L2_QUEUE_SIZE(7) |
3518                CONTEXT1_IDENTITY_ACCESS_MODE(1));
3519         WREG32(VM_L2_CNTL2, 0);
3520         WREG32(VM_L2_CNTL3, L2_CACHE_BIGK_ASSOCIATIVITY |
3521                L2_CACHE_BIGK_FRAGMENT_SIZE(0));
3522         radeon_gart_table_vram_unpin(rdev);
3523 }
3524
3525 static void si_pcie_gart_fini(struct radeon_device *rdev)
3526 {
3527         si_pcie_gart_disable(rdev);
3528         radeon_gart_table_vram_free(rdev);
3529         radeon_gart_fini(rdev);
3530 }
3531
3532 /* vm parser */
3533 static bool si_vm_reg_valid(u32 reg)
3534 {
3535         /* context regs are fine */
3536         if (reg >= 0x28000)
3537                 return true;
3538
3539         /* check config regs */
3540         switch (reg) {
3541         case GRBM_GFX_INDEX:
3542         case CP_STRMOUT_CNTL:
3543         case VGT_VTX_VECT_EJECT_REG:
3544         case VGT_CACHE_INVALIDATION:
3545         case VGT_ESGS_RING_SIZE:
3546         case VGT_GSVS_RING_SIZE:
3547         case VGT_GS_VERTEX_REUSE:
3548         case VGT_PRIMITIVE_TYPE:
3549         case VGT_INDEX_TYPE:
3550         case VGT_NUM_INDICES:
3551         case VGT_NUM_INSTANCES:
3552         case VGT_TF_RING_SIZE:
3553         case VGT_HS_OFFCHIP_PARAM:
3554         case VGT_TF_MEMORY_BASE:
3555         case PA_CL_ENHANCE:
3556         case PA_SU_LINE_STIPPLE_VALUE:
3557         case PA_SC_LINE_STIPPLE_STATE:
3558         case PA_SC_ENHANCE:
3559         case SQC_CACHES:
3560         case SPI_STATIC_THREAD_MGMT_1:
3561         case SPI_STATIC_THREAD_MGMT_2:
3562         case SPI_STATIC_THREAD_MGMT_3:
3563         case SPI_PS_MAX_WAVE_ID:
3564         case SPI_CONFIG_CNTL:
3565         case SPI_CONFIG_CNTL_1:
3566         case TA_CNTL_AUX:
3567                 return true;
3568         default:
3569                 DRM_ERROR("Invalid register 0x%x in CS\n", reg);
3570                 return false;
3571         }
3572 }
3573
3574 static int si_vm_packet3_ce_check(struct radeon_device *rdev,
3575                                   u32 *ib, struct radeon_cs_packet *pkt)
3576 {
3577         switch (pkt->opcode) {
3578         case PACKET3_NOP:
3579         case PACKET3_SET_BASE:
3580         case PACKET3_SET_CE_DE_COUNTERS:
3581         case PACKET3_LOAD_CONST_RAM:
3582         case PACKET3_WRITE_CONST_RAM:
3583         case PACKET3_WRITE_CONST_RAM_OFFSET:
3584         case PACKET3_DUMP_CONST_RAM:
3585         case PACKET3_INCREMENT_CE_COUNTER:
3586         case PACKET3_WAIT_ON_DE_COUNTER:
3587         case PACKET3_CE_WRITE:
3588                 break;
3589         default:
3590                 DRM_ERROR("Invalid CE packet3: 0x%x\n", pkt->opcode);
3591                 return -EINVAL;
3592         }
3593         return 0;
3594 }
3595
3596 static int si_vm_packet3_gfx_check(struct radeon_device *rdev,
3597                                    u32 *ib, struct radeon_cs_packet *pkt)
3598 {
3599         u32 idx = pkt->idx + 1;
3600         u32 idx_value = ib[idx];
3601         u32 start_reg, end_reg, reg, i;
3602         u32 command, info;
3603
3604         switch (pkt->opcode) {
3605         case PACKET3_NOP:
3606         case PACKET3_SET_BASE:
3607         case PACKET3_CLEAR_STATE:
3608         case PACKET3_INDEX_BUFFER_SIZE:
3609         case PACKET3_DISPATCH_DIRECT:
3610         case PACKET3_DISPATCH_INDIRECT:
3611         case PACKET3_ALLOC_GDS:
3612         case PACKET3_WRITE_GDS_RAM:
3613         case PACKET3_ATOMIC_GDS:
3614         case PACKET3_ATOMIC:
3615         case PACKET3_OCCLUSION_QUERY:
3616         case PACKET3_SET_PREDICATION:
3617         case PACKET3_COND_EXEC:
3618         case PACKET3_PRED_EXEC:
3619         case PACKET3_DRAW_INDIRECT:
3620         case PACKET3_DRAW_INDEX_INDIRECT:
3621         case PACKET3_INDEX_BASE:
3622         case PACKET3_DRAW_INDEX_2:
3623         case PACKET3_CONTEXT_CONTROL:
3624         case PACKET3_INDEX_TYPE:
3625         case PACKET3_DRAW_INDIRECT_MULTI:
3626         case PACKET3_DRAW_INDEX_AUTO:
3627         case PACKET3_DRAW_INDEX_IMMD:
3628         case PACKET3_NUM_INSTANCES:
3629         case PACKET3_DRAW_INDEX_MULTI_AUTO:
3630         case PACKET3_STRMOUT_BUFFER_UPDATE:
3631         case PACKET3_DRAW_INDEX_OFFSET_2:
3632         case PACKET3_DRAW_INDEX_MULTI_ELEMENT:
3633         case PACKET3_DRAW_INDEX_INDIRECT_MULTI:
3634         case PACKET3_MPEG_INDEX:
3635         case PACKET3_WAIT_REG_MEM:
3636         case PACKET3_MEM_WRITE:
3637         case PACKET3_PFP_SYNC_ME:
3638         case PACKET3_SURFACE_SYNC:
3639         case PACKET3_EVENT_WRITE:
3640         case PACKET3_EVENT_WRITE_EOP:
3641         case PACKET3_EVENT_WRITE_EOS:
3642         case PACKET3_SET_CONTEXT_REG:
3643         case PACKET3_SET_CONTEXT_REG_INDIRECT:
3644         case PACKET3_SET_SH_REG:
3645         case PACKET3_SET_SH_REG_OFFSET:
3646         case PACKET3_INCREMENT_DE_COUNTER:
3647         case PACKET3_WAIT_ON_CE_COUNTER:
3648         case PACKET3_WAIT_ON_AVAIL_BUFFER:
3649         case PACKET3_ME_WRITE:
3650                 break;
3651         case PACKET3_COPY_DATA:
3652                 if ((idx_value & 0xf00) == 0) {
3653                         reg = ib[idx + 3] * 4;
3654                         if (!si_vm_reg_valid(reg))
3655                                 return -EINVAL;
3656                 }
3657                 break;
3658         case PACKET3_WRITE_DATA:
3659                 if ((idx_value & 0xf00) == 0) {
3660                         start_reg = ib[idx + 1] * 4;
3661                         if (idx_value & 0x10000) {
3662                                 if (!si_vm_reg_valid(start_reg))
3663                                         return -EINVAL;
3664                         } else {
3665                                 for (i = 0; i < (pkt->count - 2); i++) {
3666                                         reg = start_reg + (4 * i);
3667                                         if (!si_vm_reg_valid(reg))
3668                                                 return -EINVAL;
3669                                 }
3670                         }
3671                 }
3672                 break;
3673         case PACKET3_COND_WRITE:
3674                 if (idx_value & 0x100) {
3675                         reg = ib[idx + 5] * 4;
3676                         if (!si_vm_reg_valid(reg))
3677                                 return -EINVAL;
3678                 }
3679                 break;
3680         case PACKET3_COPY_DW:
3681                 if (idx_value & 0x2) {
3682                         reg = ib[idx + 3] * 4;
3683                         if (!si_vm_reg_valid(reg))
3684                                 return -EINVAL;
3685                 }
3686                 break;
3687         case PACKET3_SET_CONFIG_REG:
3688                 start_reg = (idx_value << 2) + PACKET3_SET_CONFIG_REG_START;
3689                 end_reg = 4 * pkt->count + start_reg - 4;
3690                 if ((start_reg < PACKET3_SET_CONFIG_REG_START) ||
3691                     (start_reg >= PACKET3_SET_CONFIG_REG_END) ||
3692                     (end_reg >= PACKET3_SET_CONFIG_REG_END)) {
3693                         DRM_ERROR("bad PACKET3_SET_CONFIG_REG\n");
3694                         return -EINVAL;
3695                 }
3696                 for (i = 0; i < pkt->count; i++) {
3697                         reg = start_reg + (4 * i);
3698                         if (!si_vm_reg_valid(reg))
3699                                 return -EINVAL;
3700                 }
3701                 break;
3702         case PACKET3_CP_DMA:
3703                 command = ib[idx + 4];
3704                 info = ib[idx + 1];
3705                 if (command & PACKET3_CP_DMA_CMD_SAS) {
3706                         /* src address space is register */
3707                         if (((info & 0x60000000) >> 29) == 0) {
3708                                 start_reg = idx_value << 2;
3709                                 if (command & PACKET3_CP_DMA_CMD_SAIC) {
3710                                         reg = start_reg;
3711                                         if (!si_vm_reg_valid(reg)) {
3712                                                 DRM_ERROR("CP DMA Bad SRC register\n");
3713                                                 return -EINVAL;
3714                                         }
3715                                 } else {
3716                                         for (i = 0; i < (command & 0x1fffff); i++) {
3717                                                 reg = start_reg + (4 * i);
3718                                                 if (!si_vm_reg_valid(reg)) {
3719                                                         DRM_ERROR("CP DMA Bad SRC register\n");
3720                                                         return -EINVAL;
3721                                                 }
3722                                         }
3723                                 }
3724                         }
3725                 }
3726                 if (command & PACKET3_CP_DMA_CMD_DAS) {
3727                         /* dst address space is register */
3728                         if (((info & 0x00300000) >> 20) == 0) {
3729                                 start_reg = ib[idx + 2];
3730                                 if (command & PACKET3_CP_DMA_CMD_DAIC) {
3731                                         reg = start_reg;
3732                                         if (!si_vm_reg_valid(reg)) {
3733                                                 DRM_ERROR("CP DMA Bad DST register\n");
3734                                                 return -EINVAL;
3735                                         }
3736                                 } else {
3737                                         for (i = 0; i < (command & 0x1fffff); i++) {
3738                                                 reg = start_reg + (4 * i);
3739                                                 if (!si_vm_reg_valid(reg)) {
3740                                                         DRM_ERROR("CP DMA Bad DST register\n");
3741                                                         return -EINVAL;
3742                                                 }
3743                                         }
3744                                 }
3745                         }
3746                 }
3747                 break;
3748         default:
3749                 DRM_ERROR("Invalid GFX packet3: 0x%x\n", pkt->opcode);
3750                 return -EINVAL;
3751         }
3752         return 0;
3753 }
3754
3755 static int si_vm_packet3_compute_check(struct radeon_device *rdev,
3756                                        u32 *ib, struct radeon_cs_packet *pkt)
3757 {
3758         u32 idx = pkt->idx + 1;
3759         u32 idx_value = ib[idx];
3760         u32 start_reg, reg, i;
3761
3762         switch (pkt->opcode) {
3763         case PACKET3_NOP:
3764         case PACKET3_SET_BASE:
3765         case PACKET3_CLEAR_STATE:
3766         case PACKET3_DISPATCH_DIRECT:
3767         case PACKET3_DISPATCH_INDIRECT:
3768         case PACKET3_ALLOC_GDS:
3769         case PACKET3_WRITE_GDS_RAM:
3770         case PACKET3_ATOMIC_GDS:
3771         case PACKET3_ATOMIC:
3772         case PACKET3_OCCLUSION_QUERY:
3773         case PACKET3_SET_PREDICATION:
3774         case PACKET3_COND_EXEC:
3775         case PACKET3_PRED_EXEC:
3776         case PACKET3_CONTEXT_CONTROL:
3777         case PACKET3_STRMOUT_BUFFER_UPDATE:
3778         case PACKET3_WAIT_REG_MEM:
3779         case PACKET3_MEM_WRITE:
3780         case PACKET3_PFP_SYNC_ME:
3781         case PACKET3_SURFACE_SYNC:
3782         case PACKET3_EVENT_WRITE:
3783         case PACKET3_EVENT_WRITE_EOP:
3784         case PACKET3_EVENT_WRITE_EOS:
3785         case PACKET3_SET_CONTEXT_REG:
3786         case PACKET3_SET_CONTEXT_REG_INDIRECT:
3787         case PACKET3_SET_SH_REG:
3788         case PACKET3_SET_SH_REG_OFFSET:
3789         case PACKET3_INCREMENT_DE_COUNTER:
3790         case PACKET3_WAIT_ON_CE_COUNTER:
3791         case PACKET3_WAIT_ON_AVAIL_BUFFER:
3792         case PACKET3_ME_WRITE:
3793                 break;
3794         case PACKET3_COPY_DATA:
3795                 if ((idx_value & 0xf00) == 0) {
3796                         reg = ib[idx + 3] * 4;
3797                         if (!si_vm_reg_valid(reg))
3798                                 return -EINVAL;
3799                 }
3800                 break;
3801         case PACKET3_WRITE_DATA:
3802                 if ((idx_value & 0xf00) == 0) {
3803                         start_reg = ib[idx + 1] * 4;
3804                         if (idx_value & 0x10000) {
3805                                 if (!si_vm_reg_valid(start_reg))
3806                                         return -EINVAL;
3807                         } else {
3808                                 for (i = 0; i < (pkt->count - 2); i++) {
3809                                         reg = start_reg + (4 * i);
3810                                         if (!si_vm_reg_valid(reg))
3811                                                 return -EINVAL;
3812                                 }
3813                         }
3814                 }
3815                 break;
3816         case PACKET3_COND_WRITE:
3817                 if (idx_value & 0x100) {
3818                         reg = ib[idx + 5] * 4;
3819                         if (!si_vm_reg_valid(reg))
3820                                 return -EINVAL;
3821                 }
3822                 break;
3823         case PACKET3_COPY_DW:
3824                 if (idx_value & 0x2) {
3825                         reg = ib[idx + 3] * 4;
3826                         if (!si_vm_reg_valid(reg))
3827                                 return -EINVAL;
3828                 }
3829                 break;
3830         default:
3831                 DRM_ERROR("Invalid Compute packet3: 0x%x\n", pkt->opcode);
3832                 return -EINVAL;
3833         }
3834         return 0;
3835 }
3836
3837 int si_ib_parse(struct radeon_device *rdev, struct radeon_ib *ib)
3838 {
3839         int ret = 0;
3840         u32 idx = 0;
3841         struct radeon_cs_packet pkt;
3842
3843         do {
3844                 pkt.idx = idx;
3845                 pkt.type = RADEON_CP_PACKET_GET_TYPE(ib->ptr[idx]);
3846                 pkt.count = RADEON_CP_PACKET_GET_COUNT(ib->ptr[idx]);
3847                 pkt.one_reg_wr = 0;
3848                 switch (pkt.type) {
3849                 case RADEON_PACKET_TYPE0:
3850                         dev_err(rdev->dev, "Packet0 not allowed!\n");
3851                         ret = -EINVAL;
3852                         break;
3853                 case RADEON_PACKET_TYPE2:
3854                         idx += 1;
3855                         break;
3856                 case RADEON_PACKET_TYPE3:
3857                         pkt.opcode = RADEON_CP_PACKET3_GET_OPCODE(ib->ptr[idx]);
3858                         if (ib->is_const_ib)
3859                                 ret = si_vm_packet3_ce_check(rdev, ib->ptr, &pkt);
3860                         else {
3861                                 switch (ib->ring) {
3862                                 case RADEON_RING_TYPE_GFX_INDEX:
3863                                         ret = si_vm_packet3_gfx_check(rdev, ib->ptr, &pkt);
3864                                         break;
3865                                 case CAYMAN_RING_TYPE_CP1_INDEX:
3866                                 case CAYMAN_RING_TYPE_CP2_INDEX:
3867                                         ret = si_vm_packet3_compute_check(rdev, ib->ptr, &pkt);
3868                                         break;
3869                                 default:
3870                                         dev_err(rdev->dev, "Non-PM4 ring %d !\n", ib->ring);
3871                                         ret = -EINVAL;
3872                                         break;
3873                                 }
3874                         }
3875                         idx += pkt.count + 2;
3876                         break;
3877                 default:
3878                         dev_err(rdev->dev, "Unknown packet type %d !\n", pkt.type);
3879                         ret = -EINVAL;
3880                         break;
3881                 }
3882                 if (ret)
3883                         break;
3884         } while (idx < ib->length_dw);
3885
3886         return ret;
3887 }
3888
3889 /*
3890  * vm
3891  */
3892 int si_vm_init(struct radeon_device *rdev)
3893 {
3894         /* number of VMs */
3895         rdev->vm_manager.nvm = 16;
3896         /* base offset of vram pages */
3897         rdev->vm_manager.vram_base_offset = 0;
3898
3899         return 0;
3900 }
3901
3902 void si_vm_fini(struct radeon_device *rdev)
3903 {
3904 }
3905
3906 /**
3907  * si_vm_set_page - update the page tables using the CP
3908  *
3909  * @rdev: radeon_device pointer
3910  * @ib: indirect buffer to fill with commands
3911  * @pe: addr of the page entry
3912  * @addr: dst addr to write into pe
3913  * @count: number of page entries to update
3914  * @incr: increase next addr by incr bytes
3915  * @flags: access flags
3916  *
3917  * Update the page tables using the CP (SI).
3918  */
3919 void si_vm_set_page(struct radeon_device *rdev,
3920                     struct radeon_ib *ib,
3921                     uint64_t pe,
3922                     uint64_t addr, unsigned count,
3923                     uint32_t incr, uint32_t flags)
3924 {
3925         uint32_t r600_flags = cayman_vm_page_flags(rdev, flags);
3926         uint64_t value;
3927         unsigned ndw;
3928
3929         if (rdev->asic->vm.pt_ring_index == RADEON_RING_TYPE_GFX_INDEX) {
3930                 while (count) {
3931                         ndw = 2 + count * 2;
3932                         if (ndw > 0x3FFE)
3933                                 ndw = 0x3FFE;
3934
3935                         ib->ptr[ib->length_dw++] = PACKET3(PACKET3_WRITE_DATA, ndw);
3936                         ib->ptr[ib->length_dw++] = (WRITE_DATA_ENGINE_SEL(0) |
3937                                         WRITE_DATA_DST_SEL(1));
3938                         ib->ptr[ib->length_dw++] = pe;
3939                         ib->ptr[ib->length_dw++] = upper_32_bits(pe);
3940                         for (; ndw > 2; ndw -= 2, --count, pe += 8) {
3941                                 if (flags & RADEON_VM_PAGE_SYSTEM) {
3942                                         value = radeon_vm_map_gart(rdev, addr);
3943                                         value &= 0xFFFFFFFFFFFFF000ULL;
3944                                 } else if (flags & RADEON_VM_PAGE_VALID) {
3945                                         value = addr;
3946                                 } else {
3947                                         value = 0;
3948                                 }
3949                                 addr += incr;
3950                                 value |= r600_flags;
3951                                 ib->ptr[ib->length_dw++] = value;
3952                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
3953                         }
3954                 }
3955         } else {
3956                 /* DMA */
3957                 if (flags & RADEON_VM_PAGE_SYSTEM) {
3958                         while (count) {
3959                                 ndw = count * 2;
3960                                 if (ndw > 0xFFFFE)
3961                                         ndw = 0xFFFFE;
3962
3963                                 /* for non-physically contiguous pages (system) */
3964                                 ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_WRITE, 0, 0, 0, ndw);
3965                                 ib->ptr[ib->length_dw++] = pe;
3966                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3967                                 for (; ndw > 0; ndw -= 2, --count, pe += 8) {
3968                                         if (flags & RADEON_VM_PAGE_SYSTEM) {
3969                                                 value = radeon_vm_map_gart(rdev, addr);
3970                                                 value &= 0xFFFFFFFFFFFFF000ULL;
3971                                         } else if (flags & RADEON_VM_PAGE_VALID) {
3972                                                 value = addr;
3973                                         } else {
3974                                                 value = 0;
3975                                         }
3976                                         addr += incr;
3977                                         value |= r600_flags;
3978                                         ib->ptr[ib->length_dw++] = value;
3979                                         ib->ptr[ib->length_dw++] = upper_32_bits(value);
3980                                 }
3981                         }
3982                 } else {
3983                         while (count) {
3984                                 ndw = count * 2;
3985                                 if (ndw > 0xFFFFE)
3986                                         ndw = 0xFFFFE;
3987
3988                                 if (flags & RADEON_VM_PAGE_VALID)
3989                                         value = addr;
3990                                 else
3991                                         value = 0;
3992                                 /* for physically contiguous pages (vram) */
3993                                 ib->ptr[ib->length_dw++] = DMA_PTE_PDE_PACKET(ndw);
3994                                 ib->ptr[ib->length_dw++] = pe; /* dst addr */
3995                                 ib->ptr[ib->length_dw++] = upper_32_bits(pe) & 0xff;
3996                                 ib->ptr[ib->length_dw++] = r600_flags; /* mask */
3997                                 ib->ptr[ib->length_dw++] = 0;
3998                                 ib->ptr[ib->length_dw++] = value; /* value */
3999                                 ib->ptr[ib->length_dw++] = upper_32_bits(value);
4000                                 ib->ptr[ib->length_dw++] = incr; /* increment size */
4001                                 ib->ptr[ib->length_dw++] = 0;
4002                                 pe += ndw * 4;
4003                                 addr += (ndw / 2) * incr;
4004                                 count -= ndw / 2;
4005                         }
4006                 }
4007                 while (ib->length_dw & 0x7)
4008                         ib->ptr[ib->length_dw++] = DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0);
4009         }
4010 }
4011
4012 void si_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4013 {
4014         struct radeon_ring *ring = &rdev->ring[ridx];
4015
4016         if (vm == NULL)
4017                 return;
4018
4019         /* write new base address */
4020         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4021         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4022                                  WRITE_DATA_DST_SEL(0)));
4023
4024         if (vm->id < 8) {
4025                 radeon_ring_write(ring,
4026                                   (VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2);
4027         } else {
4028                 radeon_ring_write(ring,
4029                                   (VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2);
4030         }
4031         radeon_ring_write(ring, 0);
4032         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4033
4034         /* flush hdp cache */
4035         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4036         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4037                                  WRITE_DATA_DST_SEL(0)));
4038         radeon_ring_write(ring, HDP_MEM_COHERENCY_FLUSH_CNTL >> 2);
4039         radeon_ring_write(ring, 0);
4040         radeon_ring_write(ring, 0x1);
4041
4042         /* bits 0-15 are the VM contexts0-15 */
4043         radeon_ring_write(ring, PACKET3(PACKET3_WRITE_DATA, 3));
4044         radeon_ring_write(ring, (WRITE_DATA_ENGINE_SEL(0) |
4045                                  WRITE_DATA_DST_SEL(0)));
4046         radeon_ring_write(ring, VM_INVALIDATE_REQUEST >> 2);
4047         radeon_ring_write(ring, 0);
4048         radeon_ring_write(ring, 1 << vm->id);
4049
4050         /* sync PFP to ME, otherwise we might get invalid PFP reads */
4051         radeon_ring_write(ring, PACKET3(PACKET3_PFP_SYNC_ME, 0));
4052         radeon_ring_write(ring, 0x0);
4053 }
4054
4055 void si_dma_vm_flush(struct radeon_device *rdev, int ridx, struct radeon_vm *vm)
4056 {
4057         struct radeon_ring *ring = &rdev->ring[ridx];
4058
4059         if (vm == NULL)
4060                 return;
4061
4062         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4063         if (vm->id < 8) {
4064                 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT0_PAGE_TABLE_BASE_ADDR + (vm->id << 2)) >> 2));
4065         } else {
4066                 radeon_ring_write(ring, (0xf << 16) | ((VM_CONTEXT8_PAGE_TABLE_BASE_ADDR + ((vm->id - 8) << 2)) >> 2));
4067         }
4068         radeon_ring_write(ring, vm->pd_gpu_addr >> 12);
4069
4070         /* flush hdp cache */
4071         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4072         radeon_ring_write(ring, (0xf << 16) | (HDP_MEM_COHERENCY_FLUSH_CNTL >> 2));
4073         radeon_ring_write(ring, 1);
4074
4075         /* bits 0-7 are the VM contexts0-7 */
4076         radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_SRBM_WRITE, 0, 0, 0, 0));
4077         radeon_ring_write(ring, (0xf << 16) | (VM_INVALIDATE_REQUEST >> 2));
4078         radeon_ring_write(ring, 1 << vm->id);
4079 }
4080
4081 /*
4082  * RLC
4083  */
4084 void si_rlc_fini(struct radeon_device *rdev)
4085 {
4086         int r;
4087
4088         /* save restore block */
4089         if (rdev->rlc.save_restore_obj) {
4090                 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4091                 if (unlikely(r != 0))
4092                         dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4093                 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4094                 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4095
4096                 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4097                 rdev->rlc.save_restore_obj = NULL;
4098         }
4099
4100         /* clear state block */
4101         if (rdev->rlc.clear_state_obj) {
4102                 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4103                 if (unlikely(r != 0))
4104                         dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4105                 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4106                 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4107
4108                 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4109                 rdev->rlc.clear_state_obj = NULL;
4110         }
4111 }
4112
4113 int si_rlc_init(struct radeon_device *rdev)
4114 {
4115         int r;
4116
4117         /* save restore block */
4118         if (rdev->rlc.save_restore_obj == NULL) {
4119                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4120                                      RADEON_GEM_DOMAIN_VRAM, NULL,
4121                                      &rdev->rlc.save_restore_obj);
4122                 if (r) {
4123                         dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4124                         return r;
4125                 }
4126         }
4127
4128         r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4129         if (unlikely(r != 0)) {
4130                 si_rlc_fini(rdev);
4131                 return r;
4132         }
4133         r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4134                           &rdev->rlc.save_restore_gpu_addr);
4135         radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4136         if (r) {
4137                 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4138                 si_rlc_fini(rdev);
4139                 return r;
4140         }
4141
4142         /* clear state block */
4143         if (rdev->rlc.clear_state_obj == NULL) {
4144                 r = radeon_bo_create(rdev, RADEON_GPU_PAGE_SIZE, PAGE_SIZE, true,
4145                                      RADEON_GEM_DOMAIN_VRAM, NULL,
4146                                      &rdev->rlc.clear_state_obj);
4147                 if (r) {
4148                         dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4149                         si_rlc_fini(rdev);
4150                         return r;
4151                 }
4152         }
4153         r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4154         if (unlikely(r != 0)) {
4155                 si_rlc_fini(rdev);
4156                 return r;
4157         }
4158         r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4159                           &rdev->rlc.clear_state_gpu_addr);
4160         radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4161         if (r) {
4162                 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4163                 si_rlc_fini(rdev);
4164                 return r;
4165         }
4166
4167         return 0;
4168 }
4169
4170 static void si_rlc_stop(struct radeon_device *rdev)
4171 {
4172         WREG32(RLC_CNTL, 0);
4173 }
4174
4175 static void si_rlc_start(struct radeon_device *rdev)
4176 {
4177         WREG32(RLC_CNTL, RLC_ENABLE);
4178 }
4179
4180 static int si_rlc_resume(struct radeon_device *rdev)
4181 {
4182         u32 i;
4183         const __be32 *fw_data;
4184
4185         if (!rdev->rlc_fw)
4186                 return -EINVAL;
4187
4188         si_rlc_stop(rdev);
4189
4190         WREG32(RLC_RL_BASE, 0);
4191         WREG32(RLC_RL_SIZE, 0);
4192         WREG32(RLC_LB_CNTL, 0);
4193         WREG32(RLC_LB_CNTR_MAX, 0xffffffff);
4194         WREG32(RLC_LB_CNTR_INIT, 0);
4195
4196         WREG32(RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4197         WREG32(RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4198
4199         WREG32(RLC_MC_CNTL, 0);
4200         WREG32(RLC_UCODE_CNTL, 0);
4201
4202         fw_data = (const __be32 *)rdev->rlc_fw->data;
4203         for (i = 0; i < SI_RLC_UCODE_SIZE; i++) {
4204                 WREG32(RLC_UCODE_ADDR, i);
4205                 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4206         }
4207         WREG32(RLC_UCODE_ADDR, 0);
4208
4209         si_rlc_start(rdev);
4210
4211         return 0;
4212 }
4213
4214 static void si_enable_interrupts(struct radeon_device *rdev)
4215 {
4216         u32 ih_cntl = RREG32(IH_CNTL);
4217         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4218
4219         ih_cntl |= ENABLE_INTR;
4220         ih_rb_cntl |= IH_RB_ENABLE;
4221         WREG32(IH_CNTL, ih_cntl);
4222         WREG32(IH_RB_CNTL, ih_rb_cntl);
4223         rdev->ih.enabled = true;
4224 }
4225
4226 static void si_disable_interrupts(struct radeon_device *rdev)
4227 {
4228         u32 ih_rb_cntl = RREG32(IH_RB_CNTL);
4229         u32 ih_cntl = RREG32(IH_CNTL);
4230
4231         ih_rb_cntl &= ~IH_RB_ENABLE;
4232         ih_cntl &= ~ENABLE_INTR;
4233         WREG32(IH_RB_CNTL, ih_rb_cntl);
4234         WREG32(IH_CNTL, ih_cntl);
4235         /* set rptr, wptr to 0 */
4236         WREG32(IH_RB_RPTR, 0);
4237         WREG32(IH_RB_WPTR, 0);
4238         rdev->ih.enabled = false;
4239         rdev->ih.rptr = 0;
4240 }
4241
4242 static void si_disable_interrupt_state(struct radeon_device *rdev)
4243 {
4244         u32 tmp;
4245
4246         WREG32(CP_INT_CNTL_RING0, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4247         WREG32(CP_INT_CNTL_RING1, 0);
4248         WREG32(CP_INT_CNTL_RING2, 0);
4249         tmp = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4250         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, tmp);
4251         tmp = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4252         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, tmp);
4253         WREG32(GRBM_INT_CNTL, 0);
4254         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4255         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4256         if (rdev->num_crtc >= 4) {
4257                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4258                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4259         }
4260         if (rdev->num_crtc >= 6) {
4261                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4262                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4263         }
4264
4265         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, 0);
4266         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, 0);
4267         if (rdev->num_crtc >= 4) {
4268                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, 0);
4269                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, 0);
4270         }
4271         if (rdev->num_crtc >= 6) {
4272                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, 0);
4273                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, 0);
4274         }
4275
4276         WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4277
4278         tmp = RREG32(DC_HPD1_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4279         WREG32(DC_HPD1_INT_CONTROL, tmp);
4280         tmp = RREG32(DC_HPD2_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4281         WREG32(DC_HPD2_INT_CONTROL, tmp);
4282         tmp = RREG32(DC_HPD3_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4283         WREG32(DC_HPD3_INT_CONTROL, tmp);
4284         tmp = RREG32(DC_HPD4_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4285         WREG32(DC_HPD4_INT_CONTROL, tmp);
4286         tmp = RREG32(DC_HPD5_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4287         WREG32(DC_HPD5_INT_CONTROL, tmp);
4288         tmp = RREG32(DC_HPD6_INT_CONTROL) & DC_HPDx_INT_POLARITY;
4289         WREG32(DC_HPD6_INT_CONTROL, tmp);
4290
4291 }
4292
4293 static int si_irq_init(struct radeon_device *rdev)
4294 {
4295         int ret = 0;
4296         int rb_bufsz;
4297         u32 interrupt_cntl, ih_cntl, ih_rb_cntl;
4298
4299         /* allocate ring */
4300         ret = r600_ih_ring_alloc(rdev);
4301         if (ret)
4302                 return ret;
4303
4304         /* disable irqs */
4305         si_disable_interrupts(rdev);
4306
4307         /* init rlc */
4308         ret = si_rlc_resume(rdev);
4309         if (ret) {
4310                 r600_ih_ring_fini(rdev);
4311                 return ret;
4312         }
4313
4314         /* setup interrupt control */
4315         /* set dummy read address to ring address */
4316         WREG32(INTERRUPT_CNTL2, rdev->ih.gpu_addr >> 8);
4317         interrupt_cntl = RREG32(INTERRUPT_CNTL);
4318         /* IH_DUMMY_RD_OVERRIDE=0 - dummy read disabled with msi, enabled without msi
4319          * IH_DUMMY_RD_OVERRIDE=1 - dummy read controlled by IH_DUMMY_RD_EN
4320          */
4321         interrupt_cntl &= ~IH_DUMMY_RD_OVERRIDE;
4322         /* IH_REQ_NONSNOOP_EN=1 if ring is in non-cacheable memory, e.g., vram */
4323         interrupt_cntl &= ~IH_REQ_NONSNOOP_EN;
4324         WREG32(INTERRUPT_CNTL, interrupt_cntl);
4325
4326         WREG32(IH_RB_BASE, rdev->ih.gpu_addr >> 8);
4327         rb_bufsz = drm_order(rdev->ih.ring_size / 4);
4328
4329         ih_rb_cntl = (IH_WPTR_OVERFLOW_ENABLE |
4330                       IH_WPTR_OVERFLOW_CLEAR |
4331                       (rb_bufsz << 1));
4332
4333         if (rdev->wb.enabled)
4334                 ih_rb_cntl |= IH_WPTR_WRITEBACK_ENABLE;
4335
4336         /* set the writeback address whether it's enabled or not */
4337         WREG32(IH_RB_WPTR_ADDR_LO, (rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFFFFFFFC);
4338         WREG32(IH_RB_WPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + R600_WB_IH_WPTR_OFFSET) & 0xFF);
4339
4340         WREG32(IH_RB_CNTL, ih_rb_cntl);
4341
4342         /* set rptr, wptr to 0 */
4343         WREG32(IH_RB_RPTR, 0);
4344         WREG32(IH_RB_WPTR, 0);
4345
4346         /* Default settings for IH_CNTL (disabled at first) */
4347         ih_cntl = MC_WRREQ_CREDIT(0x10) | MC_WR_CLEAN_CNT(0x10) | MC_VMID(0);
4348         /* RPTR_REARM only works if msi's are enabled */
4349         if (rdev->msi_enabled)
4350                 ih_cntl |= RPTR_REARM;
4351         WREG32(IH_CNTL, ih_cntl);
4352
4353         /* force the active interrupt state to all disabled */
4354         si_disable_interrupt_state(rdev);
4355
4356         pci_set_master(rdev->pdev);
4357
4358         /* enable irqs */
4359         si_enable_interrupts(rdev);
4360
4361         return ret;
4362 }
4363
4364 int si_irq_set(struct radeon_device *rdev)
4365 {
4366         u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4367         u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4368         u32 crtc1 = 0, crtc2 = 0, crtc3 = 0, crtc4 = 0, crtc5 = 0, crtc6 = 0;
4369         u32 hpd1, hpd2, hpd3, hpd4, hpd5, hpd6;
4370         u32 grbm_int_cntl = 0;
4371         u32 grph1 = 0, grph2 = 0, grph3 = 0, grph4 = 0, grph5 = 0, grph6 = 0;
4372         u32 dma_cntl, dma_cntl1;
4373
4374         if (!rdev->irq.installed) {
4375                 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4376                 return -EINVAL;
4377         }
4378         /* don't enable anything if the ih is disabled */
4379         if (!rdev->ih.enabled) {
4380                 si_disable_interrupts(rdev);
4381                 /* force the active interrupt state to all disabled */
4382                 si_disable_interrupt_state(rdev);
4383                 return 0;
4384         }
4385
4386         hpd1 = RREG32(DC_HPD1_INT_CONTROL) & ~DC_HPDx_INT_EN;
4387         hpd2 = RREG32(DC_HPD2_INT_CONTROL) & ~DC_HPDx_INT_EN;
4388         hpd3 = RREG32(DC_HPD3_INT_CONTROL) & ~DC_HPDx_INT_EN;
4389         hpd4 = RREG32(DC_HPD4_INT_CONTROL) & ~DC_HPDx_INT_EN;
4390         hpd5 = RREG32(DC_HPD5_INT_CONTROL) & ~DC_HPDx_INT_EN;
4391         hpd6 = RREG32(DC_HPD6_INT_CONTROL) & ~DC_HPDx_INT_EN;
4392
4393         dma_cntl = RREG32(DMA_CNTL + DMA0_REGISTER_OFFSET) & ~TRAP_ENABLE;
4394         dma_cntl1 = RREG32(DMA_CNTL + DMA1_REGISTER_OFFSET) & ~TRAP_ENABLE;
4395
4396         /* enable CP interrupts on all rings */
4397         if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4398                 DRM_DEBUG("si_irq_set: sw int gfx\n");
4399                 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4400         }
4401         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4402                 DRM_DEBUG("si_irq_set: sw int cp1\n");
4403                 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4404         }
4405         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4406                 DRM_DEBUG("si_irq_set: sw int cp2\n");
4407                 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4408         }
4409         if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4410                 DRM_DEBUG("si_irq_set: sw int dma\n");
4411                 dma_cntl |= TRAP_ENABLE;
4412         }
4413
4414         if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4415                 DRM_DEBUG("si_irq_set: sw int dma1\n");
4416                 dma_cntl1 |= TRAP_ENABLE;
4417         }
4418         if (rdev->irq.crtc_vblank_int[0] ||
4419             atomic_read(&rdev->irq.pflip[0])) {
4420                 DRM_DEBUG("si_irq_set: vblank 0\n");
4421                 crtc1 |= VBLANK_INT_MASK;
4422         }
4423         if (rdev->irq.crtc_vblank_int[1] ||
4424             atomic_read(&rdev->irq.pflip[1])) {
4425                 DRM_DEBUG("si_irq_set: vblank 1\n");
4426                 crtc2 |= VBLANK_INT_MASK;
4427         }
4428         if (rdev->irq.crtc_vblank_int[2] ||
4429             atomic_read(&rdev->irq.pflip[2])) {
4430                 DRM_DEBUG("si_irq_set: vblank 2\n");
4431                 crtc3 |= VBLANK_INT_MASK;
4432         }
4433         if (rdev->irq.crtc_vblank_int[3] ||
4434             atomic_read(&rdev->irq.pflip[3])) {
4435                 DRM_DEBUG("si_irq_set: vblank 3\n");
4436                 crtc4 |= VBLANK_INT_MASK;
4437         }
4438         if (rdev->irq.crtc_vblank_int[4] ||
4439             atomic_read(&rdev->irq.pflip[4])) {
4440                 DRM_DEBUG("si_irq_set: vblank 4\n");
4441                 crtc5 |= VBLANK_INT_MASK;
4442         }
4443         if (rdev->irq.crtc_vblank_int[5] ||
4444             atomic_read(&rdev->irq.pflip[5])) {
4445                 DRM_DEBUG("si_irq_set: vblank 5\n");
4446                 crtc6 |= VBLANK_INT_MASK;
4447         }
4448         if (rdev->irq.hpd[0]) {
4449                 DRM_DEBUG("si_irq_set: hpd 1\n");
4450                 hpd1 |= DC_HPDx_INT_EN;
4451         }
4452         if (rdev->irq.hpd[1]) {
4453                 DRM_DEBUG("si_irq_set: hpd 2\n");
4454                 hpd2 |= DC_HPDx_INT_EN;
4455         }
4456         if (rdev->irq.hpd[2]) {
4457                 DRM_DEBUG("si_irq_set: hpd 3\n");
4458                 hpd3 |= DC_HPDx_INT_EN;
4459         }
4460         if (rdev->irq.hpd[3]) {
4461                 DRM_DEBUG("si_irq_set: hpd 4\n");
4462                 hpd4 |= DC_HPDx_INT_EN;
4463         }
4464         if (rdev->irq.hpd[4]) {
4465                 DRM_DEBUG("si_irq_set: hpd 5\n");
4466                 hpd5 |= DC_HPDx_INT_EN;
4467         }
4468         if (rdev->irq.hpd[5]) {
4469                 DRM_DEBUG("si_irq_set: hpd 6\n");
4470                 hpd6 |= DC_HPDx_INT_EN;
4471         }
4472
4473         WREG32(CP_INT_CNTL_RING0, cp_int_cntl);
4474         WREG32(CP_INT_CNTL_RING1, cp_int_cntl1);
4475         WREG32(CP_INT_CNTL_RING2, cp_int_cntl2);
4476
4477         WREG32(DMA_CNTL + DMA0_REGISTER_OFFSET, dma_cntl);
4478         WREG32(DMA_CNTL + DMA1_REGISTER_OFFSET, dma_cntl1);
4479
4480         WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4481
4482         WREG32(INT_MASK + EVERGREEN_CRTC0_REGISTER_OFFSET, crtc1);
4483         WREG32(INT_MASK + EVERGREEN_CRTC1_REGISTER_OFFSET, crtc2);
4484         if (rdev->num_crtc >= 4) {
4485                 WREG32(INT_MASK + EVERGREEN_CRTC2_REGISTER_OFFSET, crtc3);
4486                 WREG32(INT_MASK + EVERGREEN_CRTC3_REGISTER_OFFSET, crtc4);
4487         }
4488         if (rdev->num_crtc >= 6) {
4489                 WREG32(INT_MASK + EVERGREEN_CRTC4_REGISTER_OFFSET, crtc5);
4490                 WREG32(INT_MASK + EVERGREEN_CRTC5_REGISTER_OFFSET, crtc6);
4491         }
4492
4493         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC0_REGISTER_OFFSET, grph1);
4494         WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC1_REGISTER_OFFSET, grph2);
4495         if (rdev->num_crtc >= 4) {
4496                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC2_REGISTER_OFFSET, grph3);
4497                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC3_REGISTER_OFFSET, grph4);
4498         }
4499         if (rdev->num_crtc >= 6) {
4500                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC4_REGISTER_OFFSET, grph5);
4501                 WREG32(GRPH_INT_CONTROL + EVERGREEN_CRTC5_REGISTER_OFFSET, grph6);
4502         }
4503
4504         WREG32(DC_HPD1_INT_CONTROL, hpd1);
4505         WREG32(DC_HPD2_INT_CONTROL, hpd2);
4506         WREG32(DC_HPD3_INT_CONTROL, hpd3);
4507         WREG32(DC_HPD4_INT_CONTROL, hpd4);
4508         WREG32(DC_HPD5_INT_CONTROL, hpd5);
4509         WREG32(DC_HPD6_INT_CONTROL, hpd6);
4510
4511         return 0;
4512 }
4513
4514 static inline void si_irq_ack(struct radeon_device *rdev)
4515 {
4516         u32 tmp;
4517
4518         rdev->irq.stat_regs.evergreen.disp_int = RREG32(DISP_INTERRUPT_STATUS);
4519         rdev->irq.stat_regs.evergreen.disp_int_cont = RREG32(DISP_INTERRUPT_STATUS_CONTINUE);
4520         rdev->irq.stat_regs.evergreen.disp_int_cont2 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE2);
4521         rdev->irq.stat_regs.evergreen.disp_int_cont3 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE3);
4522         rdev->irq.stat_regs.evergreen.disp_int_cont4 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE4);
4523         rdev->irq.stat_regs.evergreen.disp_int_cont5 = RREG32(DISP_INTERRUPT_STATUS_CONTINUE5);
4524         rdev->irq.stat_regs.evergreen.d1grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET);
4525         rdev->irq.stat_regs.evergreen.d2grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET);
4526         if (rdev->num_crtc >= 4) {
4527                 rdev->irq.stat_regs.evergreen.d3grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET);
4528                 rdev->irq.stat_regs.evergreen.d4grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET);
4529         }
4530         if (rdev->num_crtc >= 6) {
4531                 rdev->irq.stat_regs.evergreen.d5grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET);
4532                 rdev->irq.stat_regs.evergreen.d6grph_int = RREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET);
4533         }
4534
4535         if (rdev->irq.stat_regs.evergreen.d1grph_int & GRPH_PFLIP_INT_OCCURRED)
4536                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4537         if (rdev->irq.stat_regs.evergreen.d2grph_int & GRPH_PFLIP_INT_OCCURRED)
4538                 WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4539         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT)
4540                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VBLANK_ACK);
4541         if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT)
4542                 WREG32(VLINE_STATUS + EVERGREEN_CRTC0_REGISTER_OFFSET, VLINE_ACK);
4543         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT)
4544                 WREG32(VBLANK_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VBLANK_ACK);
4545         if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT)
4546                 WREG32(VLINE_STATUS + EVERGREEN_CRTC1_REGISTER_OFFSET, VLINE_ACK);
4547
4548         if (rdev->num_crtc >= 4) {
4549                 if (rdev->irq.stat_regs.evergreen.d3grph_int & GRPH_PFLIP_INT_OCCURRED)
4550                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4551                 if (rdev->irq.stat_regs.evergreen.d4grph_int & GRPH_PFLIP_INT_OCCURRED)
4552                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4553                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT)
4554                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VBLANK_ACK);
4555                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT)
4556                         WREG32(VLINE_STATUS + EVERGREEN_CRTC2_REGISTER_OFFSET, VLINE_ACK);
4557                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT)
4558                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VBLANK_ACK);
4559                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT)
4560                         WREG32(VLINE_STATUS + EVERGREEN_CRTC3_REGISTER_OFFSET, VLINE_ACK);
4561         }
4562
4563         if (rdev->num_crtc >= 6) {
4564                 if (rdev->irq.stat_regs.evergreen.d5grph_int & GRPH_PFLIP_INT_OCCURRED)
4565                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4566                 if (rdev->irq.stat_regs.evergreen.d6grph_int & GRPH_PFLIP_INT_OCCURRED)
4567                         WREG32(GRPH_INT_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, GRPH_PFLIP_INT_CLEAR);
4568                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT)
4569                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VBLANK_ACK);
4570                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT)
4571                         WREG32(VLINE_STATUS + EVERGREEN_CRTC4_REGISTER_OFFSET, VLINE_ACK);
4572                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT)
4573                         WREG32(VBLANK_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VBLANK_ACK);
4574                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT)
4575                         WREG32(VLINE_STATUS + EVERGREEN_CRTC5_REGISTER_OFFSET, VLINE_ACK);
4576         }
4577
4578         if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4579                 tmp = RREG32(DC_HPD1_INT_CONTROL);
4580                 tmp |= DC_HPDx_INT_ACK;
4581                 WREG32(DC_HPD1_INT_CONTROL, tmp);
4582         }
4583         if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4584                 tmp = RREG32(DC_HPD2_INT_CONTROL);
4585                 tmp |= DC_HPDx_INT_ACK;
4586                 WREG32(DC_HPD2_INT_CONTROL, tmp);
4587         }
4588         if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4589                 tmp = RREG32(DC_HPD3_INT_CONTROL);
4590                 tmp |= DC_HPDx_INT_ACK;
4591                 WREG32(DC_HPD3_INT_CONTROL, tmp);
4592         }
4593         if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4594                 tmp = RREG32(DC_HPD4_INT_CONTROL);
4595                 tmp |= DC_HPDx_INT_ACK;
4596                 WREG32(DC_HPD4_INT_CONTROL, tmp);
4597         }
4598         if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4599                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4600                 tmp |= DC_HPDx_INT_ACK;
4601                 WREG32(DC_HPD5_INT_CONTROL, tmp);
4602         }
4603         if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4604                 tmp = RREG32(DC_HPD5_INT_CONTROL);
4605                 tmp |= DC_HPDx_INT_ACK;
4606                 WREG32(DC_HPD6_INT_CONTROL, tmp);
4607         }
4608 }
4609
4610 static void si_irq_disable(struct radeon_device *rdev)
4611 {
4612         si_disable_interrupts(rdev);
4613         /* Wait and acknowledge irq */
4614         mdelay(1);
4615         si_irq_ack(rdev);
4616         si_disable_interrupt_state(rdev);
4617 }
4618
4619 static void si_irq_suspend(struct radeon_device *rdev)
4620 {
4621         si_irq_disable(rdev);
4622         si_rlc_stop(rdev);
4623 }
4624
4625 static void si_irq_fini(struct radeon_device *rdev)
4626 {
4627         si_irq_suspend(rdev);
4628         r600_ih_ring_fini(rdev);
4629 }
4630
4631 static inline u32 si_get_ih_wptr(struct radeon_device *rdev)
4632 {
4633         u32 wptr, tmp;
4634
4635         if (rdev->wb.enabled)
4636                 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4637         else
4638                 wptr = RREG32(IH_RB_WPTR);
4639
4640         if (wptr & RB_OVERFLOW) {
4641                 /* When a ring buffer overflow happen start parsing interrupt
4642                  * from the last not overwritten vector (wptr + 16). Hopefully
4643                  * this should allow us to catchup.
4644                  */
4645                 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, %d, %d)\n",
4646                         wptr, rdev->ih.rptr, (wptr + 16) + rdev->ih.ptr_mask);
4647                 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4648                 tmp = RREG32(IH_RB_CNTL);
4649                 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4650                 WREG32(IH_RB_CNTL, tmp);
4651         }
4652         return (wptr & rdev->ih.ptr_mask);
4653 }
4654
4655 /*        SI IV Ring
4656  * Each IV ring entry is 128 bits:
4657  * [7:0]    - interrupt source id
4658  * [31:8]   - reserved
4659  * [59:32]  - interrupt source data
4660  * [63:60]  - reserved
4661  * [71:64]  - RINGID
4662  * [79:72]  - VMID
4663  * [127:80] - reserved
4664  */
4665 int si_irq_process(struct radeon_device *rdev)
4666 {
4667         u32 wptr;
4668         u32 rptr;
4669         u32 src_id, src_data, ring_id;
4670         u32 ring_index;
4671         bool queue_hotplug = false;
4672
4673         if (!rdev->ih.enabled || rdev->shutdown)
4674                 return IRQ_NONE;
4675
4676         wptr = si_get_ih_wptr(rdev);
4677
4678 restart_ih:
4679         /* is somebody else already processing irqs? */
4680         if (atomic_xchg(&rdev->ih.lock, 1))
4681                 return IRQ_NONE;
4682
4683         rptr = rdev->ih.rptr;
4684         DRM_DEBUG("si_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4685
4686         /* Order reading of wptr vs. reading of IH ring data */
4687         rmb();
4688
4689         /* display interrupts */
4690         si_irq_ack(rdev);
4691
4692         while (rptr != wptr) {
4693                 /* wptr/rptr are in bytes! */
4694                 ring_index = rptr / 4;
4695                 src_id =  le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4696                 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4697                 ring_id = le32_to_cpu(rdev->ih.ring[ring_index + 2]) & 0xff;
4698
4699                 switch (src_id) {
4700                 case 1: /* D1 vblank/vline */
4701                         switch (src_data) {
4702                         case 0: /* D1 vblank */
4703                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VBLANK_INTERRUPT) {
4704                                         if (rdev->irq.crtc_vblank_int[0]) {
4705                                                 drm_handle_vblank(rdev->ddev, 0);
4706                                                 rdev->pm.vblank_sync = true;
4707                                                 wake_up(&rdev->irq.vblank_queue);
4708                                         }
4709                                         if (atomic_read(&rdev->irq.pflip[0]))
4710                                                 radeon_crtc_handle_flip(rdev, 0);
4711                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VBLANK_INTERRUPT;
4712                                         DRM_DEBUG("IH: D1 vblank\n");
4713                                 }
4714                                 break;
4715                         case 1: /* D1 vline */
4716                                 if (rdev->irq.stat_regs.evergreen.disp_int & LB_D1_VLINE_INTERRUPT) {
4717                                         rdev->irq.stat_regs.evergreen.disp_int &= ~LB_D1_VLINE_INTERRUPT;
4718                                         DRM_DEBUG("IH: D1 vline\n");
4719                                 }
4720                                 break;
4721                         default:
4722                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4723                                 break;
4724                         }
4725                         break;
4726                 case 2: /* D2 vblank/vline */
4727                         switch (src_data) {
4728                         case 0: /* D2 vblank */
4729                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VBLANK_INTERRUPT) {
4730                                         if (rdev->irq.crtc_vblank_int[1]) {
4731                                                 drm_handle_vblank(rdev->ddev, 1);
4732                                                 rdev->pm.vblank_sync = true;
4733                                                 wake_up(&rdev->irq.vblank_queue);
4734                                         }
4735                                         if (atomic_read(&rdev->irq.pflip[1]))
4736                                                 radeon_crtc_handle_flip(rdev, 1);
4737                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VBLANK_INTERRUPT;
4738                                         DRM_DEBUG("IH: D2 vblank\n");
4739                                 }
4740                                 break;
4741                         case 1: /* D2 vline */
4742                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & LB_D2_VLINE_INTERRUPT) {
4743                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~LB_D2_VLINE_INTERRUPT;
4744                                         DRM_DEBUG("IH: D2 vline\n");
4745                                 }
4746                                 break;
4747                         default:
4748                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4749                                 break;
4750                         }
4751                         break;
4752                 case 3: /* D3 vblank/vline */
4753                         switch (src_data) {
4754                         case 0: /* D3 vblank */
4755                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VBLANK_INTERRUPT) {
4756                                         if (rdev->irq.crtc_vblank_int[2]) {
4757                                                 drm_handle_vblank(rdev->ddev, 2);
4758                                                 rdev->pm.vblank_sync = true;
4759                                                 wake_up(&rdev->irq.vblank_queue);
4760                                         }
4761                                         if (atomic_read(&rdev->irq.pflip[2]))
4762                                                 radeon_crtc_handle_flip(rdev, 2);
4763                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VBLANK_INTERRUPT;
4764                                         DRM_DEBUG("IH: D3 vblank\n");
4765                                 }
4766                                 break;
4767                         case 1: /* D3 vline */
4768                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & LB_D3_VLINE_INTERRUPT) {
4769                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~LB_D3_VLINE_INTERRUPT;
4770                                         DRM_DEBUG("IH: D3 vline\n");
4771                                 }
4772                                 break;
4773                         default:
4774                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4775                                 break;
4776                         }
4777                         break;
4778                 case 4: /* D4 vblank/vline */
4779                         switch (src_data) {
4780                         case 0: /* D4 vblank */
4781                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VBLANK_INTERRUPT) {
4782                                         if (rdev->irq.crtc_vblank_int[3]) {
4783                                                 drm_handle_vblank(rdev->ddev, 3);
4784                                                 rdev->pm.vblank_sync = true;
4785                                                 wake_up(&rdev->irq.vblank_queue);
4786                                         }
4787                                         if (atomic_read(&rdev->irq.pflip[3]))
4788                                                 radeon_crtc_handle_flip(rdev, 3);
4789                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VBLANK_INTERRUPT;
4790                                         DRM_DEBUG("IH: D4 vblank\n");
4791                                 }
4792                                 break;
4793                         case 1: /* D4 vline */
4794                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & LB_D4_VLINE_INTERRUPT) {
4795                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~LB_D4_VLINE_INTERRUPT;
4796                                         DRM_DEBUG("IH: D4 vline\n");
4797                                 }
4798                                 break;
4799                         default:
4800                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4801                                 break;
4802                         }
4803                         break;
4804                 case 5: /* D5 vblank/vline */
4805                         switch (src_data) {
4806                         case 0: /* D5 vblank */
4807                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VBLANK_INTERRUPT) {
4808                                         if (rdev->irq.crtc_vblank_int[4]) {
4809                                                 drm_handle_vblank(rdev->ddev, 4);
4810                                                 rdev->pm.vblank_sync = true;
4811                                                 wake_up(&rdev->irq.vblank_queue);
4812                                         }
4813                                         if (atomic_read(&rdev->irq.pflip[4]))
4814                                                 radeon_crtc_handle_flip(rdev, 4);
4815                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VBLANK_INTERRUPT;
4816                                         DRM_DEBUG("IH: D5 vblank\n");
4817                                 }
4818                                 break;
4819                         case 1: /* D5 vline */
4820                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & LB_D5_VLINE_INTERRUPT) {
4821                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~LB_D5_VLINE_INTERRUPT;
4822                                         DRM_DEBUG("IH: D5 vline\n");
4823                                 }
4824                                 break;
4825                         default:
4826                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4827                                 break;
4828                         }
4829                         break;
4830                 case 6: /* D6 vblank/vline */
4831                         switch (src_data) {
4832                         case 0: /* D6 vblank */
4833                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VBLANK_INTERRUPT) {
4834                                         if (rdev->irq.crtc_vblank_int[5]) {
4835                                                 drm_handle_vblank(rdev->ddev, 5);
4836                                                 rdev->pm.vblank_sync = true;
4837                                                 wake_up(&rdev->irq.vblank_queue);
4838                                         }
4839                                         if (atomic_read(&rdev->irq.pflip[5]))
4840                                                 radeon_crtc_handle_flip(rdev, 5);
4841                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VBLANK_INTERRUPT;
4842                                         DRM_DEBUG("IH: D6 vblank\n");
4843                                 }
4844                                 break;
4845                         case 1: /* D6 vline */
4846                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & LB_D6_VLINE_INTERRUPT) {
4847                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~LB_D6_VLINE_INTERRUPT;
4848                                         DRM_DEBUG("IH: D6 vline\n");
4849                                 }
4850                                 break;
4851                         default:
4852                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4853                                 break;
4854                         }
4855                         break;
4856                 case 42: /* HPD hotplug */
4857                         switch (src_data) {
4858                         case 0:
4859                                 if (rdev->irq.stat_regs.evergreen.disp_int & DC_HPD1_INTERRUPT) {
4860                                         rdev->irq.stat_regs.evergreen.disp_int &= ~DC_HPD1_INTERRUPT;
4861                                         queue_hotplug = true;
4862                                         DRM_DEBUG("IH: HPD1\n");
4863                                 }
4864                                 break;
4865                         case 1:
4866                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont & DC_HPD2_INTERRUPT) {
4867                                         rdev->irq.stat_regs.evergreen.disp_int_cont &= ~DC_HPD2_INTERRUPT;
4868                                         queue_hotplug = true;
4869                                         DRM_DEBUG("IH: HPD2\n");
4870                                 }
4871                                 break;
4872                         case 2:
4873                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont2 & DC_HPD3_INTERRUPT) {
4874                                         rdev->irq.stat_regs.evergreen.disp_int_cont2 &= ~DC_HPD3_INTERRUPT;
4875                                         queue_hotplug = true;
4876                                         DRM_DEBUG("IH: HPD3\n");
4877                                 }
4878                                 break;
4879                         case 3:
4880                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont3 & DC_HPD4_INTERRUPT) {
4881                                         rdev->irq.stat_regs.evergreen.disp_int_cont3 &= ~DC_HPD4_INTERRUPT;
4882                                         queue_hotplug = true;
4883                                         DRM_DEBUG("IH: HPD4\n");
4884                                 }
4885                                 break;
4886                         case 4:
4887                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont4 & DC_HPD5_INTERRUPT) {
4888                                         rdev->irq.stat_regs.evergreen.disp_int_cont4 &= ~DC_HPD5_INTERRUPT;
4889                                         queue_hotplug = true;
4890                                         DRM_DEBUG("IH: HPD5\n");
4891                                 }
4892                                 break;
4893                         case 5:
4894                                 if (rdev->irq.stat_regs.evergreen.disp_int_cont5 & DC_HPD6_INTERRUPT) {
4895                                         rdev->irq.stat_regs.evergreen.disp_int_cont5 &= ~DC_HPD6_INTERRUPT;
4896                                         queue_hotplug = true;
4897                                         DRM_DEBUG("IH: HPD6\n");
4898                                 }
4899                                 break;
4900                         default:
4901                                 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4902                                 break;
4903                         }
4904                         break;
4905                 case 146:
4906                 case 147:
4907                         dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4908                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_ADDR   0x%08X\n",
4909                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR));
4910                         dev_err(rdev->dev, "  VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4911                                 RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS));
4912                         /* reset addr and status */
4913                         WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4914                         break;
4915                 case 176: /* RINGID0 CP_INT */
4916                         radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4917                         break;
4918                 case 177: /* RINGID1 CP_INT */
4919                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4920                         break;
4921                 case 178: /* RINGID2 CP_INT */
4922                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4923                         break;
4924                 case 181: /* CP EOP event */
4925                         DRM_DEBUG("IH: CP EOP\n");
4926                         switch (ring_id) {
4927                         case 0:
4928                                 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4929                                 break;
4930                         case 1:
4931                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4932                                 break;
4933                         case 2:
4934                                 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4935                                 break;
4936                         }
4937                         break;
4938                 case 224: /* DMA trap event */
4939                         DRM_DEBUG("IH: DMA trap\n");
4940                         radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4941                         break;
4942                 case 233: /* GUI IDLE */
4943                         DRM_DEBUG("IH: GUI idle\n");
4944                         break;
4945                 case 244: /* DMA trap event */
4946                         DRM_DEBUG("IH: DMA1 trap\n");
4947                         radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4948                         break;
4949                 default:
4950                         DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4951                         break;
4952                 }
4953
4954                 /* wptr/rptr are in bytes! */
4955                 rptr += 16;
4956                 rptr &= rdev->ih.ptr_mask;
4957         }
4958         if (queue_hotplug)
4959                 schedule_work(&rdev->hotplug_work);
4960         rdev->ih.rptr = rptr;
4961         WREG32(IH_RB_RPTR, rdev->ih.rptr);
4962         atomic_set(&rdev->ih.lock, 0);
4963
4964         /* make sure wptr hasn't changed while processing */
4965         wptr = si_get_ih_wptr(rdev);
4966         if (wptr != rptr)
4967                 goto restart_ih;
4968
4969         return IRQ_HANDLED;
4970 }
4971
4972 /**
4973  * si_copy_dma - copy pages using the DMA engine
4974  *
4975  * @rdev: radeon_device pointer
4976  * @src_offset: src GPU address
4977  * @dst_offset: dst GPU address
4978  * @num_gpu_pages: number of GPU pages to xfer
4979  * @fence: radeon fence object
4980  *
4981  * Copy GPU paging using the DMA engine (SI).
4982  * Used by the radeon ttm implementation to move pages if
4983  * registered as the asic copy callback.
4984  */
4985 int si_copy_dma(struct radeon_device *rdev,
4986                 uint64_t src_offset, uint64_t dst_offset,
4987                 unsigned num_gpu_pages,
4988                 struct radeon_fence **fence)
4989 {
4990         struct radeon_semaphore *sem = NULL;
4991         int ring_index = rdev->asic->copy.dma_ring_index;
4992         struct radeon_ring *ring = &rdev->ring[ring_index];
4993         u32 size_in_bytes, cur_size_in_bytes;
4994         int i, num_loops;
4995         int r = 0;
4996
4997         r = radeon_semaphore_create(rdev, &sem);
4998         if (r) {
4999                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5000                 return r;
5001         }
5002
5003         size_in_bytes = (num_gpu_pages << RADEON_GPU_PAGE_SHIFT);
5004         num_loops = DIV_ROUND_UP(size_in_bytes, 0xfffff);
5005         r = radeon_ring_lock(rdev, ring, num_loops * 5 + 11);
5006         if (r) {
5007                 DRM_ERROR("radeon: moving bo (%d).\n", r);
5008                 radeon_semaphore_free(rdev, &sem, NULL);
5009                 return r;
5010         }
5011
5012         if (radeon_fence_need_sync(*fence, ring->idx)) {
5013                 radeon_semaphore_sync_rings(rdev, sem, (*fence)->ring,
5014                                             ring->idx);
5015                 radeon_fence_note_sync(*fence, ring->idx);
5016         } else {
5017                 radeon_semaphore_free(rdev, &sem, NULL);
5018         }
5019
5020         for (i = 0; i < num_loops; i++) {
5021                 cur_size_in_bytes = size_in_bytes;
5022                 if (cur_size_in_bytes > 0xFFFFF)
5023                         cur_size_in_bytes = 0xFFFFF;
5024                 size_in_bytes -= cur_size_in_bytes;
5025                 radeon_ring_write(ring, DMA_PACKET(DMA_PACKET_COPY, 1, 0, 0, cur_size_in_bytes));
5026                 radeon_ring_write(ring, dst_offset & 0xffffffff);
5027                 radeon_ring_write(ring, src_offset & 0xffffffff);
5028                 radeon_ring_write(ring, upper_32_bits(dst_offset) & 0xff);
5029                 radeon_ring_write(ring, upper_32_bits(src_offset) & 0xff);
5030                 src_offset += cur_size_in_bytes;
5031                 dst_offset += cur_size_in_bytes;
5032         }
5033
5034         r = radeon_fence_emit(rdev, fence, ring->idx);
5035         if (r) {
5036                 radeon_ring_unlock_undo(rdev, ring);
5037                 return r;
5038         }
5039
5040         radeon_ring_unlock_commit(rdev, ring);
5041         radeon_semaphore_free(rdev, &sem, *fence);
5042
5043         return r;
5044 }
5045
5046 /*
5047  * startup/shutdown callbacks
5048  */
5049 static int si_startup(struct radeon_device *rdev)
5050 {
5051         struct radeon_ring *ring;
5052         int r;
5053
5054         if (!rdev->me_fw || !rdev->pfp_fw || !rdev->ce_fw ||
5055             !rdev->rlc_fw || !rdev->mc_fw) {
5056                 r = si_init_microcode(rdev);
5057                 if (r) {
5058                         DRM_ERROR("Failed to load firmware!\n");
5059                         return r;
5060                 }
5061         }
5062
5063         r = si_mc_load_microcode(rdev);
5064         if (r) {
5065                 DRM_ERROR("Failed to load MC firmware!\n");
5066                 return r;
5067         }
5068
5069         r = r600_vram_scratch_init(rdev);
5070         if (r)
5071                 return r;
5072
5073         si_mc_program(rdev);
5074         r = si_pcie_gart_enable(rdev);
5075         if (r)
5076                 return r;
5077         si_gpu_init(rdev);
5078
5079         /* allocate rlc buffers */
5080         r = si_rlc_init(rdev);
5081         if (r) {
5082                 DRM_ERROR("Failed to init rlc BOs!\n");
5083                 return r;
5084         }
5085
5086         /* allocate wb buffer */
5087         r = radeon_wb_init(rdev);
5088         if (r)
5089                 return r;
5090
5091         r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5092         if (r) {
5093                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5094                 return r;
5095         }
5096
5097         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
5098         if (r) {
5099                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5100                 return r;
5101         }
5102
5103         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
5104         if (r) {
5105                 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5106                 return r;
5107         }
5108
5109         r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5110         if (r) {
5111                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5112                 return r;
5113         }
5114
5115         r = radeon_fence_driver_start_ring(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
5116         if (r) {
5117                 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5118                 return r;
5119         }
5120
5121         r = rv770_uvd_resume(rdev);
5122         if (!r) {
5123                 r = radeon_fence_driver_start_ring(rdev,
5124                                                    R600_RING_TYPE_UVD_INDEX);
5125                 if (r)
5126                         dev_err(rdev->dev, "UVD fences init error (%d).\n", r);
5127         }
5128         if (r)
5129                 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
5130
5131         /* Enable IRQ */
5132         r = si_irq_init(rdev);
5133         if (r) {
5134                 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5135                 radeon_irq_kms_fini(rdev);
5136                 return r;
5137         }
5138         si_irq_set(rdev);
5139
5140         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5141         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5142                              CP_RB0_RPTR, CP_RB0_WPTR,
5143                              0, 0xfffff, RADEON_CP_PACKET2);
5144         if (r)
5145                 return r;
5146
5147         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5148         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP1_RPTR_OFFSET,
5149                              CP_RB1_RPTR, CP_RB1_WPTR,
5150                              0, 0xfffff, RADEON_CP_PACKET2);
5151         if (r)
5152                 return r;
5153
5154         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5155         r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP2_RPTR_OFFSET,
5156                              CP_RB2_RPTR, CP_RB2_WPTR,
5157                              0, 0xfffff, RADEON_CP_PACKET2);
5158         if (r)
5159                 return r;
5160
5161         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5162         r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5163                              DMA_RB_RPTR + DMA0_REGISTER_OFFSET,
5164                              DMA_RB_WPTR + DMA0_REGISTER_OFFSET,
5165                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5166         if (r)
5167                 return r;
5168
5169         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5170         r = radeon_ring_init(rdev, ring, ring->ring_size, CAYMAN_WB_DMA1_RPTR_OFFSET,
5171                              DMA_RB_RPTR + DMA1_REGISTER_OFFSET,
5172                              DMA_RB_WPTR + DMA1_REGISTER_OFFSET,
5173                              2, 0x3fffc, DMA_PACKET(DMA_PACKET_NOP, 0, 0, 0, 0));
5174         if (r)
5175                 return r;
5176
5177         r = si_cp_load_microcode(rdev);
5178         if (r)
5179                 return r;
5180         r = si_cp_resume(rdev);
5181         if (r)
5182                 return r;
5183
5184         r = cayman_dma_resume(rdev);
5185         if (r)
5186                 return r;
5187
5188         ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5189         if (ring->ring_size) {
5190                 r = radeon_ring_init(rdev, ring, ring->ring_size,
5191                                      R600_WB_UVD_RPTR_OFFSET,
5192                                      UVD_RBC_RB_RPTR, UVD_RBC_RB_WPTR,
5193                                      0, 0xfffff, RADEON_CP_PACKET2);
5194                 if (!r)
5195                         r = r600_uvd_init(rdev);
5196                 if (r)
5197                         DRM_ERROR("radeon: failed initializing UVD (%d).\n", r);
5198         }
5199
5200         r = radeon_ib_pool_init(rdev);
5201         if (r) {
5202                 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5203                 return r;
5204         }
5205
5206         r = radeon_vm_manager_init(rdev);
5207         if (r) {
5208                 dev_err(rdev->dev, "vm manager initialization failed (%d).\n", r);
5209                 return r;
5210         }
5211
5212         return 0;
5213 }
5214
5215 int si_resume(struct radeon_device *rdev)
5216 {
5217         int r;
5218
5219         /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5220          * posting will perform necessary task to bring back GPU into good
5221          * shape.
5222          */
5223         /* post card */
5224         atom_asic_init(rdev->mode_info.atom_context);
5225
5226         /* init golden registers */
5227         si_init_golden_registers(rdev);
5228
5229         rdev->accel_working = true;
5230         r = si_startup(rdev);
5231         if (r) {
5232                 DRM_ERROR("si startup failed on resume\n");
5233                 rdev->accel_working = false;
5234                 return r;
5235         }
5236
5237         return r;
5238
5239 }
5240
5241 int si_suspend(struct radeon_device *rdev)
5242 {
5243         radeon_vm_manager_fini(rdev);
5244         si_cp_enable(rdev, false);
5245         cayman_dma_stop(rdev);
5246         r600_uvd_rbc_stop(rdev);
5247         radeon_uvd_suspend(rdev);
5248         si_irq_suspend(rdev);
5249         radeon_wb_disable(rdev);
5250         si_pcie_gart_disable(rdev);
5251         return 0;
5252 }
5253
5254 /* Plan is to move initialization in that function and use
5255  * helper function so that radeon_device_init pretty much
5256  * do nothing more than calling asic specific function. This
5257  * should also allow to remove a bunch of callback function
5258  * like vram_info.
5259  */
5260 int si_init(struct radeon_device *rdev)
5261 {
5262         struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5263         int r;
5264
5265         /* Read BIOS */
5266         if (!radeon_get_bios(rdev)) {
5267                 if (ASIC_IS_AVIVO(rdev))
5268                         return -EINVAL;
5269         }
5270         /* Must be an ATOMBIOS */
5271         if (!rdev->is_atom_bios) {
5272                 dev_err(rdev->dev, "Expecting atombios for cayman GPU\n");
5273                 return -EINVAL;
5274         }
5275         r = radeon_atombios_init(rdev);
5276         if (r)
5277                 return r;
5278
5279         /* Post card if necessary */
5280         if (!radeon_card_posted(rdev)) {
5281                 if (!rdev->bios) {
5282                         dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5283                         return -EINVAL;
5284                 }
5285                 DRM_INFO("GPU not posted. posting now...\n");
5286                 atom_asic_init(rdev->mode_info.atom_context);
5287         }
5288         /* init golden registers */
5289         si_init_golden_registers(rdev);
5290         /* Initialize scratch registers */
5291         si_scratch_init(rdev);
5292         /* Initialize surface registers */
5293         radeon_surface_init(rdev);
5294         /* Initialize clocks */
5295         radeon_get_clock_info(rdev->ddev);
5296
5297         /* Fence driver */
5298         r = radeon_fence_driver_init(rdev);
5299         if (r)
5300                 return r;
5301
5302         /* initialize memory controller */
5303         r = si_mc_init(rdev);
5304         if (r)
5305                 return r;
5306         /* Memory manager */
5307         r = radeon_bo_init(rdev);
5308         if (r)
5309                 return r;
5310
5311         r = radeon_irq_kms_init(rdev);
5312         if (r)
5313                 return r;
5314
5315         ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5316         ring->ring_obj = NULL;
5317         r600_ring_init(rdev, ring, 1024 * 1024);
5318
5319         ring = &rdev->ring[CAYMAN_RING_TYPE_CP1_INDEX];
5320         ring->ring_obj = NULL;
5321         r600_ring_init(rdev, ring, 1024 * 1024);
5322
5323         ring = &rdev->ring[CAYMAN_RING_TYPE_CP2_INDEX];
5324         ring->ring_obj = NULL;
5325         r600_ring_init(rdev, ring, 1024 * 1024);
5326
5327         ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5328         ring->ring_obj = NULL;
5329         r600_ring_init(rdev, ring, 64 * 1024);
5330
5331         ring = &rdev->ring[CAYMAN_RING_TYPE_DMA1_INDEX];
5332         ring->ring_obj = NULL;
5333         r600_ring_init(rdev, ring, 64 * 1024);
5334
5335         r = radeon_uvd_init(rdev);
5336         if (!r) {
5337                 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
5338                 ring->ring_obj = NULL;
5339                 r600_ring_init(rdev, ring, 4096);
5340         }
5341
5342         rdev->ih.ring_obj = NULL;
5343         r600_ih_ring_init(rdev, 64 * 1024);
5344
5345         r = r600_pcie_gart_init(rdev);
5346         if (r)
5347                 return r;
5348
5349         rdev->accel_working = true;
5350         r = si_startup(rdev);
5351         if (r) {
5352                 dev_err(rdev->dev, "disabling GPU acceleration\n");
5353                 si_cp_fini(rdev);
5354                 cayman_dma_fini(rdev);
5355                 si_irq_fini(rdev);
5356                 si_rlc_fini(rdev);
5357                 radeon_wb_fini(rdev);
5358                 radeon_ib_pool_fini(rdev);
5359                 radeon_vm_manager_fini(rdev);
5360                 radeon_irq_kms_fini(rdev);
5361                 si_pcie_gart_fini(rdev);
5362                 rdev->accel_working = false;
5363         }
5364
5365         /* Don't start up if the MC ucode is missing.
5366          * The default clocks and voltages before the MC ucode
5367          * is loaded are not suffient for advanced operations.
5368          */
5369         if (!rdev->mc_fw) {
5370                 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5371                 return -EINVAL;
5372         }
5373
5374         return 0;
5375 }
5376
5377 void si_fini(struct radeon_device *rdev)
5378 {
5379         si_cp_fini(rdev);
5380         cayman_dma_fini(rdev);
5381         si_irq_fini(rdev);
5382         si_rlc_fini(rdev);
5383         radeon_wb_fini(rdev);
5384         radeon_vm_manager_fini(rdev);
5385         radeon_ib_pool_fini(rdev);
5386         radeon_irq_kms_fini(rdev);
5387         radeon_uvd_fini(rdev);
5388         si_pcie_gart_fini(rdev);
5389         r600_vram_scratch_fini(rdev);
5390         radeon_gem_fini(rdev);
5391         radeon_fence_driver_fini(rdev);
5392         radeon_bo_fini(rdev);
5393         radeon_atombios_fini(rdev);
5394         kfree(rdev->bios);
5395         rdev->bios = NULL;
5396 }
5397
5398 /**
5399  * si_get_gpu_clock_counter - return GPU clock counter snapshot
5400  *
5401  * @rdev: radeon_device pointer
5402  *
5403  * Fetches a GPU clock counter snapshot (SI).
5404  * Returns the 64 bit clock counter snapshot.
5405  */
5406 uint64_t si_get_gpu_clock_counter(struct radeon_device *rdev)
5407 {
5408         uint64_t clock;
5409
5410         mutex_lock(&rdev->gpu_clock_mutex);
5411         WREG32(RLC_CAPTURE_GPU_CLOCK_COUNT, 1);
5412         clock = (uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_LSB) |
5413                 ((uint64_t)RREG32(RLC_GPU_CLOCK_COUNT_MSB) << 32ULL);
5414         mutex_unlock(&rdev->gpu_clock_mutex);
5415         return clock;
5416 }
5417
5418 int si_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
5419 {
5420         unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
5421         int r;
5422
5423         /* bypass vclk and dclk with bclk */
5424         WREG32_P(CG_UPLL_FUNC_CNTL_2,
5425                 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
5426                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5427
5428         /* put PLL in bypass mode */
5429         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
5430
5431         if (!vclk || !dclk) {
5432                 /* keep the Bypass mode, put PLL to sleep */
5433                 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5434                 return 0;
5435         }
5436
5437         r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
5438                                           16384, 0x03FFFFFF, 0, 128, 5,
5439                                           &fb_div, &vclk_div, &dclk_div);
5440         if (r)
5441                 return r;
5442
5443         /* set RESET_ANTI_MUX to 0 */
5444         WREG32_P(CG_UPLL_FUNC_CNTL_5, 0, ~RESET_ANTI_MUX_MASK);
5445
5446         /* set VCO_MODE to 1 */
5447         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
5448
5449         /* toggle UPLL_SLEEP to 1 then back to 0 */
5450         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
5451         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
5452
5453         /* deassert UPLL_RESET */
5454         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5455
5456         mdelay(1);
5457
5458         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5459         if (r)
5460                 return r;
5461
5462         /* assert UPLL_RESET again */
5463         WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
5464
5465         /* disable spread spectrum. */
5466         WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
5467
5468         /* set feedback divider */
5469         WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
5470
5471         /* set ref divider to 0 */
5472         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
5473
5474         if (fb_div < 307200)
5475                 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
5476         else
5477                 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
5478
5479         /* set PDIV_A and PDIV_B */
5480         WREG32_P(CG_UPLL_FUNC_CNTL_2,
5481                 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
5482                 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
5483
5484         /* give the PLL some time to settle */
5485         mdelay(15);
5486
5487         /* deassert PLL_RESET */
5488         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
5489
5490         mdelay(15);
5491
5492         /* switch from bypass mode to normal mode */
5493         WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
5494
5495         r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
5496         if (r)
5497                 return r;
5498
5499         /* switch VCLK and DCLK selection */
5500         WREG32_P(CG_UPLL_FUNC_CNTL_2,
5501                 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
5502                 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
5503
5504         mdelay(100);
5505
5506         return 0;
5507 }