2 * Copyright 2010 Advanced Micro Devices, Inc.
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
11 * The above copyright notice and this permission notice shall be included in
12 * all copies or substantial portions of the Software.
14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
20 * OTHER DEALINGS IN THE SOFTWARE.
22 * Authors: Alex Deucher
25 #include <linux/firmware.h>
26 #include <linux/pci.h>
27 #include <linux/slab.h>
29 #include <drm/drm_edid.h>
30 #include <drm/drm_vblank.h>
31 #include <drm/radeon_drm.h>
32 #include <drm/drm_fourcc.h>
33 #include <drm/drm_framebuffer.h>
40 #include "evergreen.h"
41 #include "evergreen_blit_shaders.h"
42 #include "evergreen_reg.h"
43 #include "evergreend.h"
45 #include "radeon_asic.h"
46 #include "radeon_audio.h"
47 #include "radeon_ucode.h"
50 #define DC_HPDx_CONTROL(x) (DC_HPD1_CONTROL + (x * 0xc))
51 #define DC_HPDx_INT_CONTROL(x) (DC_HPD1_INT_CONTROL + (x * 0xc))
52 #define DC_HPDx_INT_STATUS_REG(x) (DC_HPD1_INT_STATUS + (x * 0xc))
55 * Indirect registers accessor
57 u32 eg_cg_rreg(struct radeon_device *rdev, u32 reg)
62 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
63 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
64 r = RREG32(EVERGREEN_CG_IND_DATA);
65 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
69 void eg_cg_wreg(struct radeon_device *rdev, u32 reg, u32 v)
73 spin_lock_irqsave(&rdev->cg_idx_lock, flags);
74 WREG32(EVERGREEN_CG_IND_ADDR, ((reg) & 0xffff));
75 WREG32(EVERGREEN_CG_IND_DATA, (v));
76 spin_unlock_irqrestore(&rdev->cg_idx_lock, flags);
79 u32 eg_pif_phy0_rreg(struct radeon_device *rdev, u32 reg)
84 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
85 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
86 r = RREG32(EVERGREEN_PIF_PHY0_DATA);
87 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
91 void eg_pif_phy0_wreg(struct radeon_device *rdev, u32 reg, u32 v)
95 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
96 WREG32(EVERGREEN_PIF_PHY0_INDEX, ((reg) & 0xffff));
97 WREG32(EVERGREEN_PIF_PHY0_DATA, (v));
98 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
101 u32 eg_pif_phy1_rreg(struct radeon_device *rdev, u32 reg)
106 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
107 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
108 r = RREG32(EVERGREEN_PIF_PHY1_DATA);
109 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
113 void eg_pif_phy1_wreg(struct radeon_device *rdev, u32 reg, u32 v)
117 spin_lock_irqsave(&rdev->pif_idx_lock, flags);
118 WREG32(EVERGREEN_PIF_PHY1_INDEX, ((reg) & 0xffff));
119 WREG32(EVERGREEN_PIF_PHY1_DATA, (v));
120 spin_unlock_irqrestore(&rdev->pif_idx_lock, flags);
123 static const u32 crtc_offsets[6] =
125 EVERGREEN_CRTC0_REGISTER_OFFSET,
126 EVERGREEN_CRTC1_REGISTER_OFFSET,
127 EVERGREEN_CRTC2_REGISTER_OFFSET,
128 EVERGREEN_CRTC3_REGISTER_OFFSET,
129 EVERGREEN_CRTC4_REGISTER_OFFSET,
130 EVERGREEN_CRTC5_REGISTER_OFFSET
133 #include "clearstate_evergreen.h"
135 static const u32 sumo_rlc_save_restore_register_list[] =
220 static void evergreen_gpu_init(struct radeon_device *rdev);
221 void evergreen_fini(struct radeon_device *rdev);
222 void evergreen_pcie_gen2_enable(struct radeon_device *rdev);
223 void evergreen_program_aspm(struct radeon_device *rdev);
225 static const u32 evergreen_golden_registers[] =
227 0x3f90, 0xffff0000, 0xff000000,
228 0x9148, 0xffff0000, 0xff000000,
229 0x3f94, 0xffff0000, 0xff000000,
230 0x914c, 0xffff0000, 0xff000000,
231 0x9b7c, 0xffffffff, 0x00000000,
232 0x8a14, 0xffffffff, 0x00000007,
233 0x8b10, 0xffffffff, 0x00000000,
234 0x960c, 0xffffffff, 0x54763210,
235 0x88c4, 0xffffffff, 0x000000c2,
236 0x88d4, 0xffffffff, 0x00000010,
237 0x8974, 0xffffffff, 0x00000000,
238 0xc78, 0x00000080, 0x00000080,
239 0x5eb4, 0xffffffff, 0x00000002,
240 0x5e78, 0xffffffff, 0x001000f0,
241 0x6104, 0x01000300, 0x00000000,
242 0x5bc0, 0x00300000, 0x00000000,
243 0x7030, 0xffffffff, 0x00000011,
244 0x7c30, 0xffffffff, 0x00000011,
245 0x10830, 0xffffffff, 0x00000011,
246 0x11430, 0xffffffff, 0x00000011,
247 0x12030, 0xffffffff, 0x00000011,
248 0x12c30, 0xffffffff, 0x00000011,
249 0xd02c, 0xffffffff, 0x08421000,
250 0x240c, 0xffffffff, 0x00000380,
251 0x8b24, 0xffffffff, 0x00ff0fff,
252 0x28a4c, 0x06000000, 0x06000000,
253 0x10c, 0x00000001, 0x00000001,
254 0x8d00, 0xffffffff, 0x100e4848,
255 0x8d04, 0xffffffff, 0x00164745,
256 0x8c00, 0xffffffff, 0xe4000003,
257 0x8c04, 0xffffffff, 0x40600060,
258 0x8c08, 0xffffffff, 0x001c001c,
259 0x8cf0, 0xffffffff, 0x08e00620,
260 0x8c20, 0xffffffff, 0x00800080,
261 0x8c24, 0xffffffff, 0x00800080,
262 0x8c18, 0xffffffff, 0x20202078,
263 0x8c1c, 0xffffffff, 0x00001010,
264 0x28350, 0xffffffff, 0x00000000,
265 0xa008, 0xffffffff, 0x00010000,
266 0x5c4, 0xffffffff, 0x00000001,
267 0x9508, 0xffffffff, 0x00000002,
268 0x913c, 0x0000000f, 0x0000000a
271 static const u32 evergreen_golden_registers2[] =
273 0x2f4c, 0xffffffff, 0x00000000,
274 0x54f4, 0xffffffff, 0x00000000,
275 0x54f0, 0xffffffff, 0x00000000,
276 0x5498, 0xffffffff, 0x00000000,
277 0x549c, 0xffffffff, 0x00000000,
278 0x5494, 0xffffffff, 0x00000000,
279 0x53cc, 0xffffffff, 0x00000000,
280 0x53c8, 0xffffffff, 0x00000000,
281 0x53c4, 0xffffffff, 0x00000000,
282 0x53c0, 0xffffffff, 0x00000000,
283 0x53bc, 0xffffffff, 0x00000000,
284 0x53b8, 0xffffffff, 0x00000000,
285 0x53b4, 0xffffffff, 0x00000000,
286 0x53b0, 0xffffffff, 0x00000000
289 static const u32 cypress_mgcg_init[] =
291 0x802c, 0xffffffff, 0xc0000000,
292 0x5448, 0xffffffff, 0x00000100,
293 0x55e4, 0xffffffff, 0x00000100,
294 0x160c, 0xffffffff, 0x00000100,
295 0x5644, 0xffffffff, 0x00000100,
296 0xc164, 0xffffffff, 0x00000100,
297 0x8a18, 0xffffffff, 0x00000100,
298 0x897c, 0xffffffff, 0x06000100,
299 0x8b28, 0xffffffff, 0x00000100,
300 0x9144, 0xffffffff, 0x00000100,
301 0x9a60, 0xffffffff, 0x00000100,
302 0x9868, 0xffffffff, 0x00000100,
303 0x8d58, 0xffffffff, 0x00000100,
304 0x9510, 0xffffffff, 0x00000100,
305 0x949c, 0xffffffff, 0x00000100,
306 0x9654, 0xffffffff, 0x00000100,
307 0x9030, 0xffffffff, 0x00000100,
308 0x9034, 0xffffffff, 0x00000100,
309 0x9038, 0xffffffff, 0x00000100,
310 0x903c, 0xffffffff, 0x00000100,
311 0x9040, 0xffffffff, 0x00000100,
312 0xa200, 0xffffffff, 0x00000100,
313 0xa204, 0xffffffff, 0x00000100,
314 0xa208, 0xffffffff, 0x00000100,
315 0xa20c, 0xffffffff, 0x00000100,
316 0x971c, 0xffffffff, 0x00000100,
317 0x977c, 0xffffffff, 0x00000100,
318 0x3f80, 0xffffffff, 0x00000100,
319 0xa210, 0xffffffff, 0x00000100,
320 0xa214, 0xffffffff, 0x00000100,
321 0x4d8, 0xffffffff, 0x00000100,
322 0x9784, 0xffffffff, 0x00000100,
323 0x9698, 0xffffffff, 0x00000100,
324 0x4d4, 0xffffffff, 0x00000200,
325 0x30cc, 0xffffffff, 0x00000100,
326 0xd0c0, 0xffffffff, 0xff000100,
327 0x802c, 0xffffffff, 0x40000000,
328 0x915c, 0xffffffff, 0x00010000,
329 0x9160, 0xffffffff, 0x00030002,
330 0x9178, 0xffffffff, 0x00070000,
331 0x917c, 0xffffffff, 0x00030002,
332 0x9180, 0xffffffff, 0x00050004,
333 0x918c, 0xffffffff, 0x00010006,
334 0x9190, 0xffffffff, 0x00090008,
335 0x9194, 0xffffffff, 0x00070000,
336 0x9198, 0xffffffff, 0x00030002,
337 0x919c, 0xffffffff, 0x00050004,
338 0x91a8, 0xffffffff, 0x00010006,
339 0x91ac, 0xffffffff, 0x00090008,
340 0x91b0, 0xffffffff, 0x00070000,
341 0x91b4, 0xffffffff, 0x00030002,
342 0x91b8, 0xffffffff, 0x00050004,
343 0x91c4, 0xffffffff, 0x00010006,
344 0x91c8, 0xffffffff, 0x00090008,
345 0x91cc, 0xffffffff, 0x00070000,
346 0x91d0, 0xffffffff, 0x00030002,
347 0x91d4, 0xffffffff, 0x00050004,
348 0x91e0, 0xffffffff, 0x00010006,
349 0x91e4, 0xffffffff, 0x00090008,
350 0x91e8, 0xffffffff, 0x00000000,
351 0x91ec, 0xffffffff, 0x00070000,
352 0x91f0, 0xffffffff, 0x00030002,
353 0x91f4, 0xffffffff, 0x00050004,
354 0x9200, 0xffffffff, 0x00010006,
355 0x9204, 0xffffffff, 0x00090008,
356 0x9208, 0xffffffff, 0x00070000,
357 0x920c, 0xffffffff, 0x00030002,
358 0x9210, 0xffffffff, 0x00050004,
359 0x921c, 0xffffffff, 0x00010006,
360 0x9220, 0xffffffff, 0x00090008,
361 0x9224, 0xffffffff, 0x00070000,
362 0x9228, 0xffffffff, 0x00030002,
363 0x922c, 0xffffffff, 0x00050004,
364 0x9238, 0xffffffff, 0x00010006,
365 0x923c, 0xffffffff, 0x00090008,
366 0x9240, 0xffffffff, 0x00070000,
367 0x9244, 0xffffffff, 0x00030002,
368 0x9248, 0xffffffff, 0x00050004,
369 0x9254, 0xffffffff, 0x00010006,
370 0x9258, 0xffffffff, 0x00090008,
371 0x925c, 0xffffffff, 0x00070000,
372 0x9260, 0xffffffff, 0x00030002,
373 0x9264, 0xffffffff, 0x00050004,
374 0x9270, 0xffffffff, 0x00010006,
375 0x9274, 0xffffffff, 0x00090008,
376 0x9278, 0xffffffff, 0x00070000,
377 0x927c, 0xffffffff, 0x00030002,
378 0x9280, 0xffffffff, 0x00050004,
379 0x928c, 0xffffffff, 0x00010006,
380 0x9290, 0xffffffff, 0x00090008,
381 0x9294, 0xffffffff, 0x00000000,
382 0x929c, 0xffffffff, 0x00000001,
383 0x802c, 0xffffffff, 0x40010000,
384 0x915c, 0xffffffff, 0x00010000,
385 0x9160, 0xffffffff, 0x00030002,
386 0x9178, 0xffffffff, 0x00070000,
387 0x917c, 0xffffffff, 0x00030002,
388 0x9180, 0xffffffff, 0x00050004,
389 0x918c, 0xffffffff, 0x00010006,
390 0x9190, 0xffffffff, 0x00090008,
391 0x9194, 0xffffffff, 0x00070000,
392 0x9198, 0xffffffff, 0x00030002,
393 0x919c, 0xffffffff, 0x00050004,
394 0x91a8, 0xffffffff, 0x00010006,
395 0x91ac, 0xffffffff, 0x00090008,
396 0x91b0, 0xffffffff, 0x00070000,
397 0x91b4, 0xffffffff, 0x00030002,
398 0x91b8, 0xffffffff, 0x00050004,
399 0x91c4, 0xffffffff, 0x00010006,
400 0x91c8, 0xffffffff, 0x00090008,
401 0x91cc, 0xffffffff, 0x00070000,
402 0x91d0, 0xffffffff, 0x00030002,
403 0x91d4, 0xffffffff, 0x00050004,
404 0x91e0, 0xffffffff, 0x00010006,
405 0x91e4, 0xffffffff, 0x00090008,
406 0x91e8, 0xffffffff, 0x00000000,
407 0x91ec, 0xffffffff, 0x00070000,
408 0x91f0, 0xffffffff, 0x00030002,
409 0x91f4, 0xffffffff, 0x00050004,
410 0x9200, 0xffffffff, 0x00010006,
411 0x9204, 0xffffffff, 0x00090008,
412 0x9208, 0xffffffff, 0x00070000,
413 0x920c, 0xffffffff, 0x00030002,
414 0x9210, 0xffffffff, 0x00050004,
415 0x921c, 0xffffffff, 0x00010006,
416 0x9220, 0xffffffff, 0x00090008,
417 0x9224, 0xffffffff, 0x00070000,
418 0x9228, 0xffffffff, 0x00030002,
419 0x922c, 0xffffffff, 0x00050004,
420 0x9238, 0xffffffff, 0x00010006,
421 0x923c, 0xffffffff, 0x00090008,
422 0x9240, 0xffffffff, 0x00070000,
423 0x9244, 0xffffffff, 0x00030002,
424 0x9248, 0xffffffff, 0x00050004,
425 0x9254, 0xffffffff, 0x00010006,
426 0x9258, 0xffffffff, 0x00090008,
427 0x925c, 0xffffffff, 0x00070000,
428 0x9260, 0xffffffff, 0x00030002,
429 0x9264, 0xffffffff, 0x00050004,
430 0x9270, 0xffffffff, 0x00010006,
431 0x9274, 0xffffffff, 0x00090008,
432 0x9278, 0xffffffff, 0x00070000,
433 0x927c, 0xffffffff, 0x00030002,
434 0x9280, 0xffffffff, 0x00050004,
435 0x928c, 0xffffffff, 0x00010006,
436 0x9290, 0xffffffff, 0x00090008,
437 0x9294, 0xffffffff, 0x00000000,
438 0x929c, 0xffffffff, 0x00000001,
439 0x802c, 0xffffffff, 0xc0000000
442 static const u32 redwood_mgcg_init[] =
444 0x802c, 0xffffffff, 0xc0000000,
445 0x5448, 0xffffffff, 0x00000100,
446 0x55e4, 0xffffffff, 0x00000100,
447 0x160c, 0xffffffff, 0x00000100,
448 0x5644, 0xffffffff, 0x00000100,
449 0xc164, 0xffffffff, 0x00000100,
450 0x8a18, 0xffffffff, 0x00000100,
451 0x897c, 0xffffffff, 0x06000100,
452 0x8b28, 0xffffffff, 0x00000100,
453 0x9144, 0xffffffff, 0x00000100,
454 0x9a60, 0xffffffff, 0x00000100,
455 0x9868, 0xffffffff, 0x00000100,
456 0x8d58, 0xffffffff, 0x00000100,
457 0x9510, 0xffffffff, 0x00000100,
458 0x949c, 0xffffffff, 0x00000100,
459 0x9654, 0xffffffff, 0x00000100,
460 0x9030, 0xffffffff, 0x00000100,
461 0x9034, 0xffffffff, 0x00000100,
462 0x9038, 0xffffffff, 0x00000100,
463 0x903c, 0xffffffff, 0x00000100,
464 0x9040, 0xffffffff, 0x00000100,
465 0xa200, 0xffffffff, 0x00000100,
466 0xa204, 0xffffffff, 0x00000100,
467 0xa208, 0xffffffff, 0x00000100,
468 0xa20c, 0xffffffff, 0x00000100,
469 0x971c, 0xffffffff, 0x00000100,
470 0x977c, 0xffffffff, 0x00000100,
471 0x3f80, 0xffffffff, 0x00000100,
472 0xa210, 0xffffffff, 0x00000100,
473 0xa214, 0xffffffff, 0x00000100,
474 0x4d8, 0xffffffff, 0x00000100,
475 0x9784, 0xffffffff, 0x00000100,
476 0x9698, 0xffffffff, 0x00000100,
477 0x4d4, 0xffffffff, 0x00000200,
478 0x30cc, 0xffffffff, 0x00000100,
479 0xd0c0, 0xffffffff, 0xff000100,
480 0x802c, 0xffffffff, 0x40000000,
481 0x915c, 0xffffffff, 0x00010000,
482 0x9160, 0xffffffff, 0x00030002,
483 0x9178, 0xffffffff, 0x00070000,
484 0x917c, 0xffffffff, 0x00030002,
485 0x9180, 0xffffffff, 0x00050004,
486 0x918c, 0xffffffff, 0x00010006,
487 0x9190, 0xffffffff, 0x00090008,
488 0x9194, 0xffffffff, 0x00070000,
489 0x9198, 0xffffffff, 0x00030002,
490 0x919c, 0xffffffff, 0x00050004,
491 0x91a8, 0xffffffff, 0x00010006,
492 0x91ac, 0xffffffff, 0x00090008,
493 0x91b0, 0xffffffff, 0x00070000,
494 0x91b4, 0xffffffff, 0x00030002,
495 0x91b8, 0xffffffff, 0x00050004,
496 0x91c4, 0xffffffff, 0x00010006,
497 0x91c8, 0xffffffff, 0x00090008,
498 0x91cc, 0xffffffff, 0x00070000,
499 0x91d0, 0xffffffff, 0x00030002,
500 0x91d4, 0xffffffff, 0x00050004,
501 0x91e0, 0xffffffff, 0x00010006,
502 0x91e4, 0xffffffff, 0x00090008,
503 0x91e8, 0xffffffff, 0x00000000,
504 0x91ec, 0xffffffff, 0x00070000,
505 0x91f0, 0xffffffff, 0x00030002,
506 0x91f4, 0xffffffff, 0x00050004,
507 0x9200, 0xffffffff, 0x00010006,
508 0x9204, 0xffffffff, 0x00090008,
509 0x9294, 0xffffffff, 0x00000000,
510 0x929c, 0xffffffff, 0x00000001,
511 0x802c, 0xffffffff, 0xc0000000
514 static const u32 cedar_golden_registers[] =
516 0x3f90, 0xffff0000, 0xff000000,
517 0x9148, 0xffff0000, 0xff000000,
518 0x3f94, 0xffff0000, 0xff000000,
519 0x914c, 0xffff0000, 0xff000000,
520 0x9b7c, 0xffffffff, 0x00000000,
521 0x8a14, 0xffffffff, 0x00000007,
522 0x8b10, 0xffffffff, 0x00000000,
523 0x960c, 0xffffffff, 0x54763210,
524 0x88c4, 0xffffffff, 0x000000c2,
525 0x88d4, 0xffffffff, 0x00000000,
526 0x8974, 0xffffffff, 0x00000000,
527 0xc78, 0x00000080, 0x00000080,
528 0x5eb4, 0xffffffff, 0x00000002,
529 0x5e78, 0xffffffff, 0x001000f0,
530 0x6104, 0x01000300, 0x00000000,
531 0x5bc0, 0x00300000, 0x00000000,
532 0x7030, 0xffffffff, 0x00000011,
533 0x7c30, 0xffffffff, 0x00000011,
534 0x10830, 0xffffffff, 0x00000011,
535 0x11430, 0xffffffff, 0x00000011,
536 0xd02c, 0xffffffff, 0x08421000,
537 0x240c, 0xffffffff, 0x00000380,
538 0x8b24, 0xffffffff, 0x00ff0fff,
539 0x28a4c, 0x06000000, 0x06000000,
540 0x10c, 0x00000001, 0x00000001,
541 0x8d00, 0xffffffff, 0x100e4848,
542 0x8d04, 0xffffffff, 0x00164745,
543 0x8c00, 0xffffffff, 0xe4000003,
544 0x8c04, 0xffffffff, 0x40600060,
545 0x8c08, 0xffffffff, 0x001c001c,
546 0x8cf0, 0xffffffff, 0x08e00410,
547 0x8c20, 0xffffffff, 0x00800080,
548 0x8c24, 0xffffffff, 0x00800080,
549 0x8c18, 0xffffffff, 0x20202078,
550 0x8c1c, 0xffffffff, 0x00001010,
551 0x28350, 0xffffffff, 0x00000000,
552 0xa008, 0xffffffff, 0x00010000,
553 0x5c4, 0xffffffff, 0x00000001,
554 0x9508, 0xffffffff, 0x00000002
557 static const u32 cedar_mgcg_init[] =
559 0x802c, 0xffffffff, 0xc0000000,
560 0x5448, 0xffffffff, 0x00000100,
561 0x55e4, 0xffffffff, 0x00000100,
562 0x160c, 0xffffffff, 0x00000100,
563 0x5644, 0xffffffff, 0x00000100,
564 0xc164, 0xffffffff, 0x00000100,
565 0x8a18, 0xffffffff, 0x00000100,
566 0x897c, 0xffffffff, 0x06000100,
567 0x8b28, 0xffffffff, 0x00000100,
568 0x9144, 0xffffffff, 0x00000100,
569 0x9a60, 0xffffffff, 0x00000100,
570 0x9868, 0xffffffff, 0x00000100,
571 0x8d58, 0xffffffff, 0x00000100,
572 0x9510, 0xffffffff, 0x00000100,
573 0x949c, 0xffffffff, 0x00000100,
574 0x9654, 0xffffffff, 0x00000100,
575 0x9030, 0xffffffff, 0x00000100,
576 0x9034, 0xffffffff, 0x00000100,
577 0x9038, 0xffffffff, 0x00000100,
578 0x903c, 0xffffffff, 0x00000100,
579 0x9040, 0xffffffff, 0x00000100,
580 0xa200, 0xffffffff, 0x00000100,
581 0xa204, 0xffffffff, 0x00000100,
582 0xa208, 0xffffffff, 0x00000100,
583 0xa20c, 0xffffffff, 0x00000100,
584 0x971c, 0xffffffff, 0x00000100,
585 0x977c, 0xffffffff, 0x00000100,
586 0x3f80, 0xffffffff, 0x00000100,
587 0xa210, 0xffffffff, 0x00000100,
588 0xa214, 0xffffffff, 0x00000100,
589 0x4d8, 0xffffffff, 0x00000100,
590 0x9784, 0xffffffff, 0x00000100,
591 0x9698, 0xffffffff, 0x00000100,
592 0x4d4, 0xffffffff, 0x00000200,
593 0x30cc, 0xffffffff, 0x00000100,
594 0xd0c0, 0xffffffff, 0xff000100,
595 0x802c, 0xffffffff, 0x40000000,
596 0x915c, 0xffffffff, 0x00010000,
597 0x9178, 0xffffffff, 0x00050000,
598 0x917c, 0xffffffff, 0x00030002,
599 0x918c, 0xffffffff, 0x00010004,
600 0x9190, 0xffffffff, 0x00070006,
601 0x9194, 0xffffffff, 0x00050000,
602 0x9198, 0xffffffff, 0x00030002,
603 0x91a8, 0xffffffff, 0x00010004,
604 0x91ac, 0xffffffff, 0x00070006,
605 0x91e8, 0xffffffff, 0x00000000,
606 0x9294, 0xffffffff, 0x00000000,
607 0x929c, 0xffffffff, 0x00000001,
608 0x802c, 0xffffffff, 0xc0000000
611 static const u32 juniper_mgcg_init[] =
613 0x802c, 0xffffffff, 0xc0000000,
614 0x5448, 0xffffffff, 0x00000100,
615 0x55e4, 0xffffffff, 0x00000100,
616 0x160c, 0xffffffff, 0x00000100,
617 0x5644, 0xffffffff, 0x00000100,
618 0xc164, 0xffffffff, 0x00000100,
619 0x8a18, 0xffffffff, 0x00000100,
620 0x897c, 0xffffffff, 0x06000100,
621 0x8b28, 0xffffffff, 0x00000100,
622 0x9144, 0xffffffff, 0x00000100,
623 0x9a60, 0xffffffff, 0x00000100,
624 0x9868, 0xffffffff, 0x00000100,
625 0x8d58, 0xffffffff, 0x00000100,
626 0x9510, 0xffffffff, 0x00000100,
627 0x949c, 0xffffffff, 0x00000100,
628 0x9654, 0xffffffff, 0x00000100,
629 0x9030, 0xffffffff, 0x00000100,
630 0x9034, 0xffffffff, 0x00000100,
631 0x9038, 0xffffffff, 0x00000100,
632 0x903c, 0xffffffff, 0x00000100,
633 0x9040, 0xffffffff, 0x00000100,
634 0xa200, 0xffffffff, 0x00000100,
635 0xa204, 0xffffffff, 0x00000100,
636 0xa208, 0xffffffff, 0x00000100,
637 0xa20c, 0xffffffff, 0x00000100,
638 0x971c, 0xffffffff, 0x00000100,
639 0xd0c0, 0xffffffff, 0xff000100,
640 0x802c, 0xffffffff, 0x40000000,
641 0x915c, 0xffffffff, 0x00010000,
642 0x9160, 0xffffffff, 0x00030002,
643 0x9178, 0xffffffff, 0x00070000,
644 0x917c, 0xffffffff, 0x00030002,
645 0x9180, 0xffffffff, 0x00050004,
646 0x918c, 0xffffffff, 0x00010006,
647 0x9190, 0xffffffff, 0x00090008,
648 0x9194, 0xffffffff, 0x00070000,
649 0x9198, 0xffffffff, 0x00030002,
650 0x919c, 0xffffffff, 0x00050004,
651 0x91a8, 0xffffffff, 0x00010006,
652 0x91ac, 0xffffffff, 0x00090008,
653 0x91b0, 0xffffffff, 0x00070000,
654 0x91b4, 0xffffffff, 0x00030002,
655 0x91b8, 0xffffffff, 0x00050004,
656 0x91c4, 0xffffffff, 0x00010006,
657 0x91c8, 0xffffffff, 0x00090008,
658 0x91cc, 0xffffffff, 0x00070000,
659 0x91d0, 0xffffffff, 0x00030002,
660 0x91d4, 0xffffffff, 0x00050004,
661 0x91e0, 0xffffffff, 0x00010006,
662 0x91e4, 0xffffffff, 0x00090008,
663 0x91e8, 0xffffffff, 0x00000000,
664 0x91ec, 0xffffffff, 0x00070000,
665 0x91f0, 0xffffffff, 0x00030002,
666 0x91f4, 0xffffffff, 0x00050004,
667 0x9200, 0xffffffff, 0x00010006,
668 0x9204, 0xffffffff, 0x00090008,
669 0x9208, 0xffffffff, 0x00070000,
670 0x920c, 0xffffffff, 0x00030002,
671 0x9210, 0xffffffff, 0x00050004,
672 0x921c, 0xffffffff, 0x00010006,
673 0x9220, 0xffffffff, 0x00090008,
674 0x9224, 0xffffffff, 0x00070000,
675 0x9228, 0xffffffff, 0x00030002,
676 0x922c, 0xffffffff, 0x00050004,
677 0x9238, 0xffffffff, 0x00010006,
678 0x923c, 0xffffffff, 0x00090008,
679 0x9240, 0xffffffff, 0x00070000,
680 0x9244, 0xffffffff, 0x00030002,
681 0x9248, 0xffffffff, 0x00050004,
682 0x9254, 0xffffffff, 0x00010006,
683 0x9258, 0xffffffff, 0x00090008,
684 0x925c, 0xffffffff, 0x00070000,
685 0x9260, 0xffffffff, 0x00030002,
686 0x9264, 0xffffffff, 0x00050004,
687 0x9270, 0xffffffff, 0x00010006,
688 0x9274, 0xffffffff, 0x00090008,
689 0x9278, 0xffffffff, 0x00070000,
690 0x927c, 0xffffffff, 0x00030002,
691 0x9280, 0xffffffff, 0x00050004,
692 0x928c, 0xffffffff, 0x00010006,
693 0x9290, 0xffffffff, 0x00090008,
694 0x9294, 0xffffffff, 0x00000000,
695 0x929c, 0xffffffff, 0x00000001,
696 0x802c, 0xffffffff, 0xc0000000,
697 0x977c, 0xffffffff, 0x00000100,
698 0x3f80, 0xffffffff, 0x00000100,
699 0xa210, 0xffffffff, 0x00000100,
700 0xa214, 0xffffffff, 0x00000100,
701 0x4d8, 0xffffffff, 0x00000100,
702 0x9784, 0xffffffff, 0x00000100,
703 0x9698, 0xffffffff, 0x00000100,
704 0x4d4, 0xffffffff, 0x00000200,
705 0x30cc, 0xffffffff, 0x00000100,
706 0x802c, 0xffffffff, 0xc0000000
709 static const u32 supersumo_golden_registers[] =
711 0x5eb4, 0xffffffff, 0x00000002,
712 0x5c4, 0xffffffff, 0x00000001,
713 0x7030, 0xffffffff, 0x00000011,
714 0x7c30, 0xffffffff, 0x00000011,
715 0x6104, 0x01000300, 0x00000000,
716 0x5bc0, 0x00300000, 0x00000000,
717 0x8c04, 0xffffffff, 0x40600060,
718 0x8c08, 0xffffffff, 0x001c001c,
719 0x8c20, 0xffffffff, 0x00800080,
720 0x8c24, 0xffffffff, 0x00800080,
721 0x8c18, 0xffffffff, 0x20202078,
722 0x8c1c, 0xffffffff, 0x00001010,
723 0x918c, 0xffffffff, 0x00010006,
724 0x91a8, 0xffffffff, 0x00010006,
725 0x91c4, 0xffffffff, 0x00010006,
726 0x91e0, 0xffffffff, 0x00010006,
727 0x9200, 0xffffffff, 0x00010006,
728 0x9150, 0xffffffff, 0x6e944040,
729 0x917c, 0xffffffff, 0x00030002,
730 0x9180, 0xffffffff, 0x00050004,
731 0x9198, 0xffffffff, 0x00030002,
732 0x919c, 0xffffffff, 0x00050004,
733 0x91b4, 0xffffffff, 0x00030002,
734 0x91b8, 0xffffffff, 0x00050004,
735 0x91d0, 0xffffffff, 0x00030002,
736 0x91d4, 0xffffffff, 0x00050004,
737 0x91f0, 0xffffffff, 0x00030002,
738 0x91f4, 0xffffffff, 0x00050004,
739 0x915c, 0xffffffff, 0x00010000,
740 0x9160, 0xffffffff, 0x00030002,
741 0x3f90, 0xffff0000, 0xff000000,
742 0x9178, 0xffffffff, 0x00070000,
743 0x9194, 0xffffffff, 0x00070000,
744 0x91b0, 0xffffffff, 0x00070000,
745 0x91cc, 0xffffffff, 0x00070000,
746 0x91ec, 0xffffffff, 0x00070000,
747 0x9148, 0xffff0000, 0xff000000,
748 0x9190, 0xffffffff, 0x00090008,
749 0x91ac, 0xffffffff, 0x00090008,
750 0x91c8, 0xffffffff, 0x00090008,
751 0x91e4, 0xffffffff, 0x00090008,
752 0x9204, 0xffffffff, 0x00090008,
753 0x3f94, 0xffff0000, 0xff000000,
754 0x914c, 0xffff0000, 0xff000000,
755 0x929c, 0xffffffff, 0x00000001,
756 0x8a18, 0xffffffff, 0x00000100,
757 0x8b28, 0xffffffff, 0x00000100,
758 0x9144, 0xffffffff, 0x00000100,
759 0x5644, 0xffffffff, 0x00000100,
760 0x9b7c, 0xffffffff, 0x00000000,
761 0x8030, 0xffffffff, 0x0000100a,
762 0x8a14, 0xffffffff, 0x00000007,
763 0x8b24, 0xffffffff, 0x00ff0fff,
764 0x8b10, 0xffffffff, 0x00000000,
765 0x28a4c, 0x06000000, 0x06000000,
766 0x4d8, 0xffffffff, 0x00000100,
767 0x913c, 0xffff000f, 0x0100000a,
768 0x960c, 0xffffffff, 0x54763210,
769 0x88c4, 0xffffffff, 0x000000c2,
770 0x88d4, 0xffffffff, 0x00000010,
771 0x8974, 0xffffffff, 0x00000000,
772 0xc78, 0x00000080, 0x00000080,
773 0x5e78, 0xffffffff, 0x001000f0,
774 0xd02c, 0xffffffff, 0x08421000,
775 0xa008, 0xffffffff, 0x00010000,
776 0x8d00, 0xffffffff, 0x100e4848,
777 0x8d04, 0xffffffff, 0x00164745,
778 0x8c00, 0xffffffff, 0xe4000003,
779 0x8cf0, 0x1fffffff, 0x08e00620,
780 0x28350, 0xffffffff, 0x00000000,
781 0x9508, 0xffffffff, 0x00000002
784 static const u32 sumo_golden_registers[] =
786 0x900c, 0x00ffffff, 0x0017071f,
787 0x8c18, 0xffffffff, 0x10101060,
788 0x8c1c, 0xffffffff, 0x00001010,
789 0x8c30, 0x0000000f, 0x00000005,
790 0x9688, 0x0000000f, 0x00000007
793 static const u32 wrestler_golden_registers[] =
795 0x5eb4, 0xffffffff, 0x00000002,
796 0x5c4, 0xffffffff, 0x00000001,
797 0x7030, 0xffffffff, 0x00000011,
798 0x7c30, 0xffffffff, 0x00000011,
799 0x6104, 0x01000300, 0x00000000,
800 0x5bc0, 0x00300000, 0x00000000,
801 0x918c, 0xffffffff, 0x00010006,
802 0x91a8, 0xffffffff, 0x00010006,
803 0x9150, 0xffffffff, 0x6e944040,
804 0x917c, 0xffffffff, 0x00030002,
805 0x9198, 0xffffffff, 0x00030002,
806 0x915c, 0xffffffff, 0x00010000,
807 0x3f90, 0xffff0000, 0xff000000,
808 0x9178, 0xffffffff, 0x00070000,
809 0x9194, 0xffffffff, 0x00070000,
810 0x9148, 0xffff0000, 0xff000000,
811 0x9190, 0xffffffff, 0x00090008,
812 0x91ac, 0xffffffff, 0x00090008,
813 0x3f94, 0xffff0000, 0xff000000,
814 0x914c, 0xffff0000, 0xff000000,
815 0x929c, 0xffffffff, 0x00000001,
816 0x8a18, 0xffffffff, 0x00000100,
817 0x8b28, 0xffffffff, 0x00000100,
818 0x9144, 0xffffffff, 0x00000100,
819 0x9b7c, 0xffffffff, 0x00000000,
820 0x8030, 0xffffffff, 0x0000100a,
821 0x8a14, 0xffffffff, 0x00000001,
822 0x8b24, 0xffffffff, 0x00ff0fff,
823 0x8b10, 0xffffffff, 0x00000000,
824 0x28a4c, 0x06000000, 0x06000000,
825 0x4d8, 0xffffffff, 0x00000100,
826 0x913c, 0xffff000f, 0x0100000a,
827 0x960c, 0xffffffff, 0x54763210,
828 0x88c4, 0xffffffff, 0x000000c2,
829 0x88d4, 0xffffffff, 0x00000010,
830 0x8974, 0xffffffff, 0x00000000,
831 0xc78, 0x00000080, 0x00000080,
832 0x5e78, 0xffffffff, 0x001000f0,
833 0xd02c, 0xffffffff, 0x08421000,
834 0xa008, 0xffffffff, 0x00010000,
835 0x8d00, 0xffffffff, 0x100e4848,
836 0x8d04, 0xffffffff, 0x00164745,
837 0x8c00, 0xffffffff, 0xe4000003,
838 0x8cf0, 0x1fffffff, 0x08e00410,
839 0x28350, 0xffffffff, 0x00000000,
840 0x9508, 0xffffffff, 0x00000002,
841 0x900c, 0xffffffff, 0x0017071f,
842 0x8c18, 0xffffffff, 0x10101060,
843 0x8c1c, 0xffffffff, 0x00001010
846 static const u32 barts_golden_registers[] =
848 0x5eb4, 0xffffffff, 0x00000002,
849 0x5e78, 0x8f311ff1, 0x001000f0,
850 0x3f90, 0xffff0000, 0xff000000,
851 0x9148, 0xffff0000, 0xff000000,
852 0x3f94, 0xffff0000, 0xff000000,
853 0x914c, 0xffff0000, 0xff000000,
854 0xc78, 0x00000080, 0x00000080,
855 0xbd4, 0x70073777, 0x00010001,
856 0xd02c, 0xbfffff1f, 0x08421000,
857 0xd0b8, 0x03773777, 0x02011003,
858 0x5bc0, 0x00200000, 0x50100000,
859 0x98f8, 0x33773777, 0x02011003,
860 0x98fc, 0xffffffff, 0x76543210,
861 0x7030, 0x31000311, 0x00000011,
862 0x2f48, 0x00000007, 0x02011003,
863 0x6b28, 0x00000010, 0x00000012,
864 0x7728, 0x00000010, 0x00000012,
865 0x10328, 0x00000010, 0x00000012,
866 0x10f28, 0x00000010, 0x00000012,
867 0x11b28, 0x00000010, 0x00000012,
868 0x12728, 0x00000010, 0x00000012,
869 0x240c, 0x000007ff, 0x00000380,
870 0x8a14, 0xf000001f, 0x00000007,
871 0x8b24, 0x3fff3fff, 0x00ff0fff,
872 0x8b10, 0x0000ff0f, 0x00000000,
873 0x28a4c, 0x07ffffff, 0x06000000,
874 0x10c, 0x00000001, 0x00010003,
875 0xa02c, 0xffffffff, 0x0000009b,
876 0x913c, 0x0000000f, 0x0100000a,
877 0x8d00, 0xffff7f7f, 0x100e4848,
878 0x8d04, 0x00ffffff, 0x00164745,
879 0x8c00, 0xfffc0003, 0xe4000003,
880 0x8c04, 0xf8ff00ff, 0x40600060,
881 0x8c08, 0x00ff00ff, 0x001c001c,
882 0x8cf0, 0x1fff1fff, 0x08e00620,
883 0x8c20, 0x0fff0fff, 0x00800080,
884 0x8c24, 0x0fff0fff, 0x00800080,
885 0x8c18, 0xffffffff, 0x20202078,
886 0x8c1c, 0x0000ffff, 0x00001010,
887 0x28350, 0x00000f01, 0x00000000,
888 0x9508, 0x3700001f, 0x00000002,
889 0x960c, 0xffffffff, 0x54763210,
890 0x88c4, 0x001f3ae3, 0x000000c2,
891 0x88d4, 0x0000001f, 0x00000010,
892 0x8974, 0xffffffff, 0x00000000
895 static const u32 turks_golden_registers[] =
897 0x5eb4, 0xffffffff, 0x00000002,
898 0x5e78, 0x8f311ff1, 0x001000f0,
899 0x8c8, 0x00003000, 0x00001070,
900 0x8cc, 0x000fffff, 0x00040035,
901 0x3f90, 0xffff0000, 0xfff00000,
902 0x9148, 0xffff0000, 0xfff00000,
903 0x3f94, 0xffff0000, 0xfff00000,
904 0x914c, 0xffff0000, 0xfff00000,
905 0xc78, 0x00000080, 0x00000080,
906 0xbd4, 0x00073007, 0x00010002,
907 0xd02c, 0xbfffff1f, 0x08421000,
908 0xd0b8, 0x03773777, 0x02010002,
909 0x5bc0, 0x00200000, 0x50100000,
910 0x98f8, 0x33773777, 0x00010002,
911 0x98fc, 0xffffffff, 0x33221100,
912 0x7030, 0x31000311, 0x00000011,
913 0x2f48, 0x33773777, 0x00010002,
914 0x6b28, 0x00000010, 0x00000012,
915 0x7728, 0x00000010, 0x00000012,
916 0x10328, 0x00000010, 0x00000012,
917 0x10f28, 0x00000010, 0x00000012,
918 0x11b28, 0x00000010, 0x00000012,
919 0x12728, 0x00000010, 0x00000012,
920 0x240c, 0x000007ff, 0x00000380,
921 0x8a14, 0xf000001f, 0x00000007,
922 0x8b24, 0x3fff3fff, 0x00ff0fff,
923 0x8b10, 0x0000ff0f, 0x00000000,
924 0x28a4c, 0x07ffffff, 0x06000000,
925 0x10c, 0x00000001, 0x00010003,
926 0xa02c, 0xffffffff, 0x0000009b,
927 0x913c, 0x0000000f, 0x0100000a,
928 0x8d00, 0xffff7f7f, 0x100e4848,
929 0x8d04, 0x00ffffff, 0x00164745,
930 0x8c00, 0xfffc0003, 0xe4000003,
931 0x8c04, 0xf8ff00ff, 0x40600060,
932 0x8c08, 0x00ff00ff, 0x001c001c,
933 0x8cf0, 0x1fff1fff, 0x08e00410,
934 0x8c20, 0x0fff0fff, 0x00800080,
935 0x8c24, 0x0fff0fff, 0x00800080,
936 0x8c18, 0xffffffff, 0x20202078,
937 0x8c1c, 0x0000ffff, 0x00001010,
938 0x28350, 0x00000f01, 0x00000000,
939 0x9508, 0x3700001f, 0x00000002,
940 0x960c, 0xffffffff, 0x54763210,
941 0x88c4, 0x001f3ae3, 0x000000c2,
942 0x88d4, 0x0000001f, 0x00000010,
943 0x8974, 0xffffffff, 0x00000000
946 static const u32 caicos_golden_registers[] =
948 0x5eb4, 0xffffffff, 0x00000002,
949 0x5e78, 0x8f311ff1, 0x001000f0,
950 0x8c8, 0x00003420, 0x00001450,
951 0x8cc, 0x000fffff, 0x00040035,
952 0x3f90, 0xffff0000, 0xfffc0000,
953 0x9148, 0xffff0000, 0xfffc0000,
954 0x3f94, 0xffff0000, 0xfffc0000,
955 0x914c, 0xffff0000, 0xfffc0000,
956 0xc78, 0x00000080, 0x00000080,
957 0xbd4, 0x00073007, 0x00010001,
958 0xd02c, 0xbfffff1f, 0x08421000,
959 0xd0b8, 0x03773777, 0x02010001,
960 0x5bc0, 0x00200000, 0x50100000,
961 0x98f8, 0x33773777, 0x02010001,
962 0x98fc, 0xffffffff, 0x33221100,
963 0x7030, 0x31000311, 0x00000011,
964 0x2f48, 0x33773777, 0x02010001,
965 0x6b28, 0x00000010, 0x00000012,
966 0x7728, 0x00000010, 0x00000012,
967 0x10328, 0x00000010, 0x00000012,
968 0x10f28, 0x00000010, 0x00000012,
969 0x11b28, 0x00000010, 0x00000012,
970 0x12728, 0x00000010, 0x00000012,
971 0x240c, 0x000007ff, 0x00000380,
972 0x8a14, 0xf000001f, 0x00000001,
973 0x8b24, 0x3fff3fff, 0x00ff0fff,
974 0x8b10, 0x0000ff0f, 0x00000000,
975 0x28a4c, 0x07ffffff, 0x06000000,
976 0x10c, 0x00000001, 0x00010003,
977 0xa02c, 0xffffffff, 0x0000009b,
978 0x913c, 0x0000000f, 0x0100000a,
979 0x8d00, 0xffff7f7f, 0x100e4848,
980 0x8d04, 0x00ffffff, 0x00164745,
981 0x8c00, 0xfffc0003, 0xe4000003,
982 0x8c04, 0xf8ff00ff, 0x40600060,
983 0x8c08, 0x00ff00ff, 0x001c001c,
984 0x8cf0, 0x1fff1fff, 0x08e00410,
985 0x8c20, 0x0fff0fff, 0x00800080,
986 0x8c24, 0x0fff0fff, 0x00800080,
987 0x8c18, 0xffffffff, 0x20202078,
988 0x8c1c, 0x0000ffff, 0x00001010,
989 0x28350, 0x00000f01, 0x00000000,
990 0x9508, 0x3700001f, 0x00000002,
991 0x960c, 0xffffffff, 0x54763210,
992 0x88c4, 0x001f3ae3, 0x000000c2,
993 0x88d4, 0x0000001f, 0x00000010,
994 0x8974, 0xffffffff, 0x00000000
997 static void evergreen_init_golden_registers(struct radeon_device *rdev)
999 switch (rdev->family) {
1002 radeon_program_register_sequence(rdev,
1003 evergreen_golden_registers,
1004 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1005 radeon_program_register_sequence(rdev,
1006 evergreen_golden_registers2,
1007 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1008 radeon_program_register_sequence(rdev,
1010 (const u32)ARRAY_SIZE(cypress_mgcg_init));
1013 radeon_program_register_sequence(rdev,
1014 evergreen_golden_registers,
1015 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1016 radeon_program_register_sequence(rdev,
1017 evergreen_golden_registers2,
1018 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1019 radeon_program_register_sequence(rdev,
1021 (const u32)ARRAY_SIZE(juniper_mgcg_init));
1024 radeon_program_register_sequence(rdev,
1025 evergreen_golden_registers,
1026 (const u32)ARRAY_SIZE(evergreen_golden_registers));
1027 radeon_program_register_sequence(rdev,
1028 evergreen_golden_registers2,
1029 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1030 radeon_program_register_sequence(rdev,
1032 (const u32)ARRAY_SIZE(redwood_mgcg_init));
1035 radeon_program_register_sequence(rdev,
1036 cedar_golden_registers,
1037 (const u32)ARRAY_SIZE(cedar_golden_registers));
1038 radeon_program_register_sequence(rdev,
1039 evergreen_golden_registers2,
1040 (const u32)ARRAY_SIZE(evergreen_golden_registers2));
1041 radeon_program_register_sequence(rdev,
1043 (const u32)ARRAY_SIZE(cedar_mgcg_init));
1046 radeon_program_register_sequence(rdev,
1047 wrestler_golden_registers,
1048 (const u32)ARRAY_SIZE(wrestler_golden_registers));
1051 radeon_program_register_sequence(rdev,
1052 supersumo_golden_registers,
1053 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1056 radeon_program_register_sequence(rdev,
1057 supersumo_golden_registers,
1058 (const u32)ARRAY_SIZE(supersumo_golden_registers));
1059 radeon_program_register_sequence(rdev,
1060 sumo_golden_registers,
1061 (const u32)ARRAY_SIZE(sumo_golden_registers));
1064 radeon_program_register_sequence(rdev,
1065 barts_golden_registers,
1066 (const u32)ARRAY_SIZE(barts_golden_registers));
1069 radeon_program_register_sequence(rdev,
1070 turks_golden_registers,
1071 (const u32)ARRAY_SIZE(turks_golden_registers));
1074 radeon_program_register_sequence(rdev,
1075 caicos_golden_registers,
1076 (const u32)ARRAY_SIZE(caicos_golden_registers));
1084 * evergreen_get_allowed_info_register - fetch the register for the info ioctl
1086 * @rdev: radeon_device pointer
1087 * @reg: register offset in bytes
1088 * @val: register value
1090 * Returns 0 for success or -EINVAL for an invalid register
1093 int evergreen_get_allowed_info_register(struct radeon_device *rdev,
1098 case GRBM_STATUS_SE0:
1099 case GRBM_STATUS_SE1:
1102 case DMA_STATUS_REG:
1111 void evergreen_tiling_fields(unsigned tiling_flags, unsigned *bankw,
1112 unsigned *bankh, unsigned *mtaspect,
1113 unsigned *tile_split)
1115 *bankw = (tiling_flags >> RADEON_TILING_EG_BANKW_SHIFT) & RADEON_TILING_EG_BANKW_MASK;
1116 *bankh = (tiling_flags >> RADEON_TILING_EG_BANKH_SHIFT) & RADEON_TILING_EG_BANKH_MASK;
1117 *mtaspect = (tiling_flags >> RADEON_TILING_EG_MACRO_TILE_ASPECT_SHIFT) & RADEON_TILING_EG_MACRO_TILE_ASPECT_MASK;
1118 *tile_split = (tiling_flags >> RADEON_TILING_EG_TILE_SPLIT_SHIFT) & RADEON_TILING_EG_TILE_SPLIT_MASK;
1121 case 1: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_1; break;
1122 case 2: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_2; break;
1123 case 4: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_4; break;
1124 case 8: *bankw = EVERGREEN_ADDR_SURF_BANK_WIDTH_8; break;
1128 case 1: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_1; break;
1129 case 2: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_2; break;
1130 case 4: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_4; break;
1131 case 8: *bankh = EVERGREEN_ADDR_SURF_BANK_HEIGHT_8; break;
1133 switch (*mtaspect) {
1135 case 1: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_1; break;
1136 case 2: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_2; break;
1137 case 4: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_4; break;
1138 case 8: *mtaspect = EVERGREEN_ADDR_SURF_MACRO_TILE_ASPECT_8; break;
1142 static int sumo_set_uvd_clock(struct radeon_device *rdev, u32 clock,
1143 u32 cntl_reg, u32 status_reg)
1146 struct atom_clock_dividers dividers;
1148 r = radeon_atom_get_clock_dividers(rdev, COMPUTE_ENGINE_PLL_PARAM,
1149 clock, false, ÷rs);
1153 WREG32_P(cntl_reg, dividers.post_div, ~(DCLK_DIR_CNTL_EN|DCLK_DIVIDER_MASK));
1155 for (i = 0; i < 100; i++) {
1156 if (RREG32(status_reg) & DCLK_STATUS)
1166 int sumo_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1169 u32 cg_scratch = RREG32(CG_SCRATCH1);
1171 r = sumo_set_uvd_clock(rdev, vclk, CG_VCLK_CNTL, CG_VCLK_STATUS);
1174 cg_scratch &= 0xffff0000;
1175 cg_scratch |= vclk / 100; /* Mhz */
1177 r = sumo_set_uvd_clock(rdev, dclk, CG_DCLK_CNTL, CG_DCLK_STATUS);
1180 cg_scratch &= 0x0000ffff;
1181 cg_scratch |= (dclk / 100) << 16; /* Mhz */
1184 WREG32(CG_SCRATCH1, cg_scratch);
1189 int evergreen_set_uvd_clocks(struct radeon_device *rdev, u32 vclk, u32 dclk)
1191 /* start off with something large */
1192 unsigned fb_div = 0, vclk_div = 0, dclk_div = 0;
1195 /* bypass vclk and dclk with bclk */
1196 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1197 VCLK_SRC_SEL(1) | DCLK_SRC_SEL(1),
1198 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1200 /* put PLL in bypass mode */
1201 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_BYPASS_EN_MASK, ~UPLL_BYPASS_EN_MASK);
1203 if (!vclk || !dclk) {
1204 /* keep the Bypass mode, put PLL to sleep */
1205 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1209 r = radeon_uvd_calc_upll_dividers(rdev, vclk, dclk, 125000, 250000,
1210 16384, 0x03FFFFFF, 0, 128, 5,
1211 &fb_div, &vclk_div, &dclk_div);
1215 /* set VCO_MODE to 1 */
1216 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_VCO_MODE_MASK, ~UPLL_VCO_MODE_MASK);
1218 /* toggle UPLL_SLEEP to 1 then back to 0 */
1219 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_SLEEP_MASK, ~UPLL_SLEEP_MASK);
1220 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_SLEEP_MASK);
1222 /* deassert UPLL_RESET */
1223 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1227 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1231 /* assert UPLL_RESET again */
1232 WREG32_P(CG_UPLL_FUNC_CNTL, UPLL_RESET_MASK, ~UPLL_RESET_MASK);
1234 /* disable spread spectrum. */
1235 WREG32_P(CG_UPLL_SPREAD_SPECTRUM, 0, ~SSEN_MASK);
1237 /* set feedback divider */
1238 WREG32_P(CG_UPLL_FUNC_CNTL_3, UPLL_FB_DIV(fb_div), ~UPLL_FB_DIV_MASK);
1240 /* set ref divider to 0 */
1241 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_REF_DIV_MASK);
1243 if (fb_div < 307200)
1244 WREG32_P(CG_UPLL_FUNC_CNTL_4, 0, ~UPLL_SPARE_ISPARE9);
1246 WREG32_P(CG_UPLL_FUNC_CNTL_4, UPLL_SPARE_ISPARE9, ~UPLL_SPARE_ISPARE9);
1248 /* set PDIV_A and PDIV_B */
1249 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1250 UPLL_PDIV_A(vclk_div) | UPLL_PDIV_B(dclk_div),
1251 ~(UPLL_PDIV_A_MASK | UPLL_PDIV_B_MASK));
1253 /* give the PLL some time to settle */
1256 /* deassert PLL_RESET */
1257 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_RESET_MASK);
1261 /* switch from bypass mode to normal mode */
1262 WREG32_P(CG_UPLL_FUNC_CNTL, 0, ~UPLL_BYPASS_EN_MASK);
1264 r = radeon_uvd_send_upll_ctlreq(rdev, CG_UPLL_FUNC_CNTL);
1268 /* switch VCLK and DCLK selection */
1269 WREG32_P(CG_UPLL_FUNC_CNTL_2,
1270 VCLK_SRC_SEL(2) | DCLK_SRC_SEL(2),
1271 ~(VCLK_SRC_SEL_MASK | DCLK_SRC_SEL_MASK));
1278 void evergreen_fix_pci_max_read_req_size(struct radeon_device *rdev)
1283 readrq = pcie_get_readrq(rdev->pdev);
1284 v = ffs(readrq) - 8;
1285 /* if bios or OS sets MAX_READ_REQUEST_SIZE to an invalid value, fix it
1286 * to avoid hangs or perfomance issues
1288 if ((v == 0) || (v == 6) || (v == 7))
1289 pcie_set_readrq(rdev->pdev, 512);
1292 void dce4_program_fmt(struct drm_encoder *encoder)
1294 struct drm_device *dev = encoder->dev;
1295 struct radeon_device *rdev = dev->dev_private;
1296 struct radeon_encoder *radeon_encoder = to_radeon_encoder(encoder);
1297 struct radeon_crtc *radeon_crtc = to_radeon_crtc(encoder->crtc);
1298 struct drm_connector *connector = radeon_get_connector_for_encoder(encoder);
1301 enum radeon_connector_dither dither = RADEON_FMT_DITHER_DISABLE;
1304 struct radeon_connector *radeon_connector = to_radeon_connector(connector);
1305 bpc = radeon_get_monitor_bpc(connector);
1306 dither = radeon_connector->dither;
1309 /* LVDS/eDP FMT is set up by atom */
1310 if (radeon_encoder->devices & ATOM_DEVICE_LCD_SUPPORT)
1313 /* not needed for analog */
1314 if ((radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC1) ||
1315 (radeon_encoder->encoder_id == ENCODER_OBJECT_ID_INTERNAL_KLDSCP_DAC2))
1323 if (dither == RADEON_FMT_DITHER_ENABLE)
1324 /* XXX sort out optimal dither settings */
1325 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1326 FMT_SPATIAL_DITHER_EN);
1328 tmp |= FMT_TRUNCATE_EN;
1331 if (dither == RADEON_FMT_DITHER_ENABLE)
1332 /* XXX sort out optimal dither settings */
1333 tmp |= (FMT_FRAME_RANDOM_ENABLE | FMT_HIGHPASS_RANDOM_ENABLE |
1334 FMT_RGB_RANDOM_ENABLE |
1335 FMT_SPATIAL_DITHER_EN | FMT_SPATIAL_DITHER_DEPTH);
1337 tmp |= (FMT_TRUNCATE_EN | FMT_TRUNCATE_DEPTH);
1345 WREG32(FMT_BIT_DEPTH_CONTROL + radeon_crtc->crtc_offset, tmp);
1348 static bool dce4_is_in_vblank(struct radeon_device *rdev, int crtc)
1350 if (RREG32(EVERGREEN_CRTC_STATUS + crtc_offsets[crtc]) & EVERGREEN_CRTC_V_BLANK)
1356 static bool dce4_is_counter_moving(struct radeon_device *rdev, int crtc)
1360 pos1 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1361 pos2 = RREG32(EVERGREEN_CRTC_STATUS_POSITION + crtc_offsets[crtc]);
1370 * dce4_wait_for_vblank - vblank wait asic callback.
1372 * @rdev: radeon_device pointer
1373 * @crtc: crtc to wait for vblank on
1375 * Wait for vblank on the requested crtc (evergreen+).
1377 void dce4_wait_for_vblank(struct radeon_device *rdev, int crtc)
1381 if (crtc >= rdev->num_crtc)
1384 if (!(RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[crtc]) & EVERGREEN_CRTC_MASTER_EN))
1387 /* depending on when we hit vblank, we may be close to active; if so,
1388 * wait for another frame.
1390 while (dce4_is_in_vblank(rdev, crtc)) {
1391 if (i++ % 100 == 0) {
1392 if (!dce4_is_counter_moving(rdev, crtc))
1397 while (!dce4_is_in_vblank(rdev, crtc)) {
1398 if (i++ % 100 == 0) {
1399 if (!dce4_is_counter_moving(rdev, crtc))
1406 * evergreen_page_flip - pageflip callback.
1408 * @rdev: radeon_device pointer
1409 * @crtc_id: crtc to cleanup pageflip on
1410 * @crtc_base: new address of the crtc (GPU MC address)
1411 * @async: asynchronous flip
1413 * Triggers the actual pageflip by updating the primary
1414 * surface base address (evergreen+).
1416 void evergreen_page_flip(struct radeon_device *rdev, int crtc_id, u64 crtc_base,
1419 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1420 struct drm_framebuffer *fb = radeon_crtc->base.primary->fb;
1422 /* flip at hsync for async, default is vsync */
1423 WREG32(EVERGREEN_GRPH_FLIP_CONTROL + radeon_crtc->crtc_offset,
1424 async ? EVERGREEN_GRPH_SURFACE_UPDATE_H_RETRACE_EN : 0);
1426 WREG32(EVERGREEN_GRPH_PITCH + radeon_crtc->crtc_offset,
1427 fb->pitches[0] / fb->format->cpp[0]);
1428 /* update the scanout addresses */
1429 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + radeon_crtc->crtc_offset,
1430 upper_32_bits(crtc_base));
1431 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset,
1433 /* post the write */
1434 RREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + radeon_crtc->crtc_offset);
1438 * evergreen_page_flip_pending - check if page flip is still pending
1440 * @rdev: radeon_device pointer
1441 * @crtc_id: crtc to check
1443 * Returns the current update pending status.
1445 bool evergreen_page_flip_pending(struct radeon_device *rdev, int crtc_id)
1447 struct radeon_crtc *radeon_crtc = rdev->mode_info.crtcs[crtc_id];
1449 /* Return current update_pending status: */
1450 return !!(RREG32(EVERGREEN_GRPH_UPDATE + radeon_crtc->crtc_offset) &
1451 EVERGREEN_GRPH_SURFACE_UPDATE_PENDING);
1454 /* get temperature in millidegrees */
1455 int evergreen_get_temp(struct radeon_device *rdev)
1458 int actual_temp = 0;
1460 if (rdev->family == CHIP_JUNIPER) {
1461 toffset = (RREG32(CG_THERMAL_CTRL) & TOFFSET_MASK) >>
1463 temp = (RREG32(CG_TS0_STATUS) & TS0_ADC_DOUT_MASK) >>
1466 if (toffset & 0x100)
1467 actual_temp = temp / 2 - (0x200 - toffset);
1469 actual_temp = temp / 2 + toffset;
1471 actual_temp = actual_temp * 1000;
1474 temp = (RREG32(CG_MULT_THERMAL_STATUS) & ASIC_T_MASK) >>
1479 else if (temp & 0x200)
1481 else if (temp & 0x100) {
1482 actual_temp = temp & 0x1ff;
1483 actual_temp |= ~0x1ff;
1485 actual_temp = temp & 0xff;
1487 actual_temp = (actual_temp * 1000) / 2;
1493 int sumo_get_temp(struct radeon_device *rdev)
1495 u32 temp = RREG32(CG_THERMAL_STATUS) & 0xff;
1496 int actual_temp = temp - 49;
1498 return actual_temp * 1000;
1502 * sumo_pm_init_profile - Initialize power profiles callback.
1504 * @rdev: radeon_device pointer
1506 * Initialize the power states used in profile mode
1507 * (sumo, trinity, SI).
1508 * Used for profile mode only.
1510 void sumo_pm_init_profile(struct radeon_device *rdev)
1515 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1516 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1517 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1518 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 0;
1521 if (rdev->flags & RADEON_IS_MOBILITY)
1522 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1524 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1526 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1527 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1528 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1529 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1531 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1532 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1533 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1534 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1536 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1537 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1538 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1539 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 0;
1541 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1542 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1543 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1544 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 0;
1547 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1548 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1549 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1550 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1551 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx =
1552 rdev->pm.power_state[idx].num_clock_modes - 1;
1554 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1555 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1556 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1557 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx =
1558 rdev->pm.power_state[idx].num_clock_modes - 1;
1562 * btc_pm_init_profile - Initialize power profiles callback.
1564 * @rdev: radeon_device pointer
1566 * Initialize the power states used in profile mode
1568 * Used for profile mode only.
1570 void btc_pm_init_profile(struct radeon_device *rdev)
1575 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_ps_idx = rdev->pm.default_power_state_index;
1576 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_ps_idx = rdev->pm.default_power_state_index;
1577 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_off_cm_idx = 0;
1578 rdev->pm.profiles[PM_PROFILE_DEFAULT_IDX].dpms_on_cm_idx = 2;
1579 /* starting with BTC, there is one state that is used for both
1580 * MH and SH. Difference is that we always use the high clock index for
1583 if (rdev->flags & RADEON_IS_MOBILITY)
1584 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_BATTERY, 0);
1586 idx = radeon_pm_get_type_index(rdev, POWER_STATE_TYPE_PERFORMANCE, 0);
1588 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_ps_idx = idx;
1589 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_ps_idx = idx;
1590 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_off_cm_idx = 0;
1591 rdev->pm.profiles[PM_PROFILE_LOW_SH_IDX].dpms_on_cm_idx = 0;
1593 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_ps_idx = idx;
1594 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_ps_idx = idx;
1595 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_off_cm_idx = 0;
1596 rdev->pm.profiles[PM_PROFILE_MID_SH_IDX].dpms_on_cm_idx = 1;
1598 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_ps_idx = idx;
1599 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_ps_idx = idx;
1600 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_off_cm_idx = 0;
1601 rdev->pm.profiles[PM_PROFILE_HIGH_SH_IDX].dpms_on_cm_idx = 2;
1603 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_ps_idx = idx;
1604 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_ps_idx = idx;
1605 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_off_cm_idx = 0;
1606 rdev->pm.profiles[PM_PROFILE_LOW_MH_IDX].dpms_on_cm_idx = 0;
1608 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_ps_idx = idx;
1609 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_ps_idx = idx;
1610 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_off_cm_idx = 0;
1611 rdev->pm.profiles[PM_PROFILE_MID_MH_IDX].dpms_on_cm_idx = 1;
1613 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_ps_idx = idx;
1614 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_ps_idx = idx;
1615 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_off_cm_idx = 0;
1616 rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx = 2;
1620 * evergreen_pm_misc - set additional pm hw parameters callback.
1622 * @rdev: radeon_device pointer
1624 * Set non-clock parameters associated with a power state
1625 * (voltage, etc.) (evergreen+).
1627 void evergreen_pm_misc(struct radeon_device *rdev)
1629 int req_ps_idx = rdev->pm.requested_power_state_index;
1630 int req_cm_idx = rdev->pm.requested_clock_mode_index;
1631 struct radeon_power_state *ps = &rdev->pm.power_state[req_ps_idx];
1632 struct radeon_voltage *voltage = &ps->clock_info[req_cm_idx].voltage;
1634 if (voltage->type == VOLTAGE_SW) {
1635 /* 0xff0x are flags rather then an actual voltage */
1636 if ((voltage->voltage & 0xff00) == 0xff00)
1638 if (voltage->voltage && (voltage->voltage != rdev->pm.current_vddc)) {
1639 radeon_atom_set_voltage(rdev, voltage->voltage, SET_VOLTAGE_TYPE_ASIC_VDDC);
1640 rdev->pm.current_vddc = voltage->voltage;
1641 DRM_DEBUG("Setting: vddc: %d\n", voltage->voltage);
1644 /* starting with BTC, there is one state that is used for both
1645 * MH and SH. Difference is that we always use the high clock index for
1648 if ((rdev->pm.pm_method == PM_METHOD_PROFILE) &&
1649 (rdev->family >= CHIP_BARTS) &&
1650 rdev->pm.active_crtc_count &&
1651 ((rdev->pm.profile_index == PM_PROFILE_MID_MH_IDX) ||
1652 (rdev->pm.profile_index == PM_PROFILE_LOW_MH_IDX)))
1653 voltage = &rdev->pm.power_state[req_ps_idx].
1654 clock_info[rdev->pm.profiles[PM_PROFILE_HIGH_MH_IDX].dpms_on_cm_idx].voltage;
1656 /* 0xff0x are flags rather then an actual voltage */
1657 if ((voltage->vddci & 0xff00) == 0xff00)
1659 if (voltage->vddci && (voltage->vddci != rdev->pm.current_vddci)) {
1660 radeon_atom_set_voltage(rdev, voltage->vddci, SET_VOLTAGE_TYPE_ASIC_VDDCI);
1661 rdev->pm.current_vddci = voltage->vddci;
1662 DRM_DEBUG("Setting: vddci: %d\n", voltage->vddci);
1668 * evergreen_pm_prepare - pre-power state change callback.
1670 * @rdev: radeon_device pointer
1672 * Prepare for a power state change (evergreen+).
1674 void evergreen_pm_prepare(struct radeon_device *rdev)
1676 struct drm_device *ddev = rdev_to_drm(rdev);
1677 struct drm_crtc *crtc;
1678 struct radeon_crtc *radeon_crtc;
1681 /* disable any active CRTCs */
1682 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1683 radeon_crtc = to_radeon_crtc(crtc);
1684 if (radeon_crtc->enabled) {
1685 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1686 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1687 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1693 * evergreen_pm_finish - post-power state change callback.
1695 * @rdev: radeon_device pointer
1697 * Clean up after a power state change (evergreen+).
1699 void evergreen_pm_finish(struct radeon_device *rdev)
1701 struct drm_device *ddev = rdev_to_drm(rdev);
1702 struct drm_crtc *crtc;
1703 struct radeon_crtc *radeon_crtc;
1706 /* enable any active CRTCs */
1707 list_for_each_entry(crtc, &ddev->mode_config.crtc_list, head) {
1708 radeon_crtc = to_radeon_crtc(crtc);
1709 if (radeon_crtc->enabled) {
1710 tmp = RREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset);
1711 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
1712 WREG32(EVERGREEN_CRTC_CONTROL + radeon_crtc->crtc_offset, tmp);
1718 * evergreen_hpd_sense - hpd sense callback.
1720 * @rdev: radeon_device pointer
1721 * @hpd: hpd (hotplug detect) pin
1723 * Checks if a digital monitor is connected (evergreen+).
1724 * Returns true if connected, false if not connected.
1726 bool evergreen_hpd_sense(struct radeon_device *rdev, enum radeon_hpd_id hpd)
1728 if (hpd == RADEON_HPD_NONE)
1731 return !!(RREG32(DC_HPDx_INT_STATUS_REG(hpd)) & DC_HPDx_SENSE);
1735 * evergreen_hpd_set_polarity - hpd set polarity callback.
1737 * @rdev: radeon_device pointer
1738 * @hpd: hpd (hotplug detect) pin
1740 * Set the polarity of the hpd pin (evergreen+).
1742 void evergreen_hpd_set_polarity(struct radeon_device *rdev,
1743 enum radeon_hpd_id hpd)
1745 bool connected = evergreen_hpd_sense(rdev, hpd);
1747 if (hpd == RADEON_HPD_NONE)
1751 WREG32_AND(DC_HPDx_INT_CONTROL(hpd), ~DC_HPDx_INT_POLARITY);
1753 WREG32_OR(DC_HPDx_INT_CONTROL(hpd), DC_HPDx_INT_POLARITY);
1757 * evergreen_hpd_init - hpd setup callback.
1759 * @rdev: radeon_device pointer
1761 * Setup the hpd pins used by the card (evergreen+).
1762 * Enable the pin, set the polarity, and enable the hpd interrupts.
1764 void evergreen_hpd_init(struct radeon_device *rdev)
1766 struct drm_device *dev = rdev_to_drm(rdev);
1767 struct drm_connector *connector;
1768 unsigned enabled = 0;
1769 u32 tmp = DC_HPDx_CONNECTION_TIMER(0x9c4) |
1770 DC_HPDx_RX_INT_TIMER(0xfa) | DC_HPDx_EN;
1772 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1773 enum radeon_hpd_id hpd =
1774 to_radeon_connector(connector)->hpd.hpd;
1776 if (connector->connector_type == DRM_MODE_CONNECTOR_eDP ||
1777 connector->connector_type == DRM_MODE_CONNECTOR_LVDS) {
1778 /* don't try to enable hpd on eDP or LVDS avoid breaking the
1779 * aux dp channel on imac and help (but not completely fix)
1780 * https://bugzilla.redhat.com/show_bug.cgi?id=726143
1781 * also avoid interrupt storms during dpms.
1786 if (hpd == RADEON_HPD_NONE)
1789 WREG32(DC_HPDx_CONTROL(hpd), tmp);
1790 enabled |= 1 << hpd;
1792 radeon_hpd_set_polarity(rdev, hpd);
1794 radeon_irq_kms_enable_hpd(rdev, enabled);
1798 * evergreen_hpd_fini - hpd tear down callback.
1800 * @rdev: radeon_device pointer
1802 * Tear down the hpd pins used by the card (evergreen+).
1803 * Disable the hpd interrupts.
1805 void evergreen_hpd_fini(struct radeon_device *rdev)
1807 struct drm_device *dev = rdev_to_drm(rdev);
1808 struct drm_connector *connector;
1809 unsigned disabled = 0;
1811 list_for_each_entry(connector, &dev->mode_config.connector_list, head) {
1812 enum radeon_hpd_id hpd =
1813 to_radeon_connector(connector)->hpd.hpd;
1815 if (hpd == RADEON_HPD_NONE)
1818 WREG32(DC_HPDx_CONTROL(hpd), 0);
1819 disabled |= 1 << hpd;
1821 radeon_irq_kms_disable_hpd(rdev, disabled);
1824 /* watermark setup */
1826 static u32 evergreen_line_buffer_adjust(struct radeon_device *rdev,
1827 struct radeon_crtc *radeon_crtc,
1828 struct drm_display_mode *mode,
1829 struct drm_display_mode *other_mode)
1831 u32 tmp, buffer_alloc, i;
1832 u32 pipe_offset = radeon_crtc->crtc_id * 0x20;
1835 * There are 3 line buffers, each one shared by 2 display controllers.
1836 * DC_LB_MEMORY_SPLIT controls how that line buffer is shared between
1837 * the display controllers. The paritioning is done via one of four
1838 * preset allocations specified in bits 2:0:
1839 * first display controller
1840 * 0 - first half of lb (3840 * 2)
1841 * 1 - first 3/4 of lb (5760 * 2)
1842 * 2 - whole lb (7680 * 2), other crtc must be disabled
1843 * 3 - first 1/4 of lb (1920 * 2)
1844 * second display controller
1845 * 4 - second half of lb (3840 * 2)
1846 * 5 - second 3/4 of lb (5760 * 2)
1847 * 6 - whole lb (7680 * 2), other crtc must be disabled
1848 * 7 - last 1/4 of lb (1920 * 2)
1850 /* this can get tricky if we have two large displays on a paired group
1851 * of crtcs. Ideally for multiple large displays we'd assign them to
1852 * non-linked crtcs for maximum line buffer allocation.
1854 if (radeon_crtc->base.enabled && mode) {
1859 tmp = 2; /* whole */
1867 /* second controller of the pair uses second half of the lb */
1868 if (radeon_crtc->crtc_id % 2)
1870 WREG32(DC_LB_MEMORY_SPLIT + radeon_crtc->crtc_offset, tmp);
1872 if (ASIC_IS_DCE41(rdev) || ASIC_IS_DCE5(rdev)) {
1873 WREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset,
1874 DMIF_BUFFERS_ALLOCATED(buffer_alloc));
1875 for (i = 0; i < rdev->usec_timeout; i++) {
1876 if (RREG32(PIPE0_DMIF_BUFFER_CONTROL + pipe_offset) &
1877 DMIF_BUFFERS_ALLOCATED_COMPLETED)
1883 if (radeon_crtc->base.enabled && mode) {
1888 if (ASIC_IS_DCE5(rdev))
1894 if (ASIC_IS_DCE5(rdev))
1900 if (ASIC_IS_DCE5(rdev))
1906 if (ASIC_IS_DCE5(rdev))
1913 /* controller not enabled, so no lb used */
1917 u32 evergreen_get_number_of_dram_channels(struct radeon_device *rdev)
1919 u32 tmp = RREG32(MC_SHARED_CHMAP);
1921 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
1934 struct evergreen_wm_params {
1935 u32 dram_channels; /* number of dram channels */
1936 u32 yclk; /* bandwidth per dram data pin in kHz */
1937 u32 sclk; /* engine clock in kHz */
1938 u32 disp_clk; /* display clock in kHz */
1939 u32 src_width; /* viewport width */
1940 u32 active_time; /* active display time in ns */
1941 u32 blank_time; /* blank time in ns */
1942 bool interlaced; /* mode is interlaced */
1943 fixed20_12 vsc; /* vertical scale ratio */
1944 u32 num_heads; /* number of active crtcs */
1945 u32 bytes_per_pixel; /* bytes per pixel display + overlay */
1946 u32 lb_size; /* line buffer allocated to pipe */
1947 u32 vtaps; /* vertical scaler taps */
1950 static u32 evergreen_dram_bandwidth(struct evergreen_wm_params *wm)
1952 /* Calculate DRAM Bandwidth and the part allocated to display. */
1953 fixed20_12 dram_efficiency; /* 0.7 */
1954 fixed20_12 yclk, dram_channels, bandwidth;
1957 a.full = dfixed_const(1000);
1958 yclk.full = dfixed_const(wm->yclk);
1959 yclk.full = dfixed_div(yclk, a);
1960 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1961 a.full = dfixed_const(10);
1962 dram_efficiency.full = dfixed_const(7);
1963 dram_efficiency.full = dfixed_div(dram_efficiency, a);
1964 bandwidth.full = dfixed_mul(dram_channels, yclk);
1965 bandwidth.full = dfixed_mul(bandwidth, dram_efficiency);
1967 return dfixed_trunc(bandwidth);
1970 static u32 evergreen_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
1972 /* Calculate DRAM Bandwidth and the part allocated to display. */
1973 fixed20_12 disp_dram_allocation; /* 0.3 to 0.7 */
1974 fixed20_12 yclk, dram_channels, bandwidth;
1977 a.full = dfixed_const(1000);
1978 yclk.full = dfixed_const(wm->yclk);
1979 yclk.full = dfixed_div(yclk, a);
1980 dram_channels.full = dfixed_const(wm->dram_channels * 4);
1981 a.full = dfixed_const(10);
1982 disp_dram_allocation.full = dfixed_const(3); /* XXX worse case value 0.3 */
1983 disp_dram_allocation.full = dfixed_div(disp_dram_allocation, a);
1984 bandwidth.full = dfixed_mul(dram_channels, yclk);
1985 bandwidth.full = dfixed_mul(bandwidth, disp_dram_allocation);
1987 return dfixed_trunc(bandwidth);
1990 static u32 evergreen_data_return_bandwidth(struct evergreen_wm_params *wm)
1992 /* Calculate the display Data return Bandwidth */
1993 fixed20_12 return_efficiency; /* 0.8 */
1994 fixed20_12 sclk, bandwidth;
1997 a.full = dfixed_const(1000);
1998 sclk.full = dfixed_const(wm->sclk);
1999 sclk.full = dfixed_div(sclk, a);
2000 a.full = dfixed_const(10);
2001 return_efficiency.full = dfixed_const(8);
2002 return_efficiency.full = dfixed_div(return_efficiency, a);
2003 a.full = dfixed_const(32);
2004 bandwidth.full = dfixed_mul(a, sclk);
2005 bandwidth.full = dfixed_mul(bandwidth, return_efficiency);
2007 return dfixed_trunc(bandwidth);
2010 static u32 evergreen_dmif_request_bandwidth(struct evergreen_wm_params *wm)
2012 /* Calculate the DMIF Request Bandwidth */
2013 fixed20_12 disp_clk_request_efficiency; /* 0.8 */
2014 fixed20_12 disp_clk, bandwidth;
2017 a.full = dfixed_const(1000);
2018 disp_clk.full = dfixed_const(wm->disp_clk);
2019 disp_clk.full = dfixed_div(disp_clk, a);
2020 a.full = dfixed_const(10);
2021 disp_clk_request_efficiency.full = dfixed_const(8);
2022 disp_clk_request_efficiency.full = dfixed_div(disp_clk_request_efficiency, a);
2023 a.full = dfixed_const(32);
2024 bandwidth.full = dfixed_mul(a, disp_clk);
2025 bandwidth.full = dfixed_mul(bandwidth, disp_clk_request_efficiency);
2027 return dfixed_trunc(bandwidth);
2030 static u32 evergreen_available_bandwidth(struct evergreen_wm_params *wm)
2032 /* Calculate the Available bandwidth. Display can use this temporarily but not in average. */
2033 u32 dram_bandwidth = evergreen_dram_bandwidth(wm);
2034 u32 data_return_bandwidth = evergreen_data_return_bandwidth(wm);
2035 u32 dmif_req_bandwidth = evergreen_dmif_request_bandwidth(wm);
2037 return min(dram_bandwidth, min(data_return_bandwidth, dmif_req_bandwidth));
2040 static u32 evergreen_average_bandwidth(struct evergreen_wm_params *wm)
2042 /* Calculate the display mode Average Bandwidth
2043 * DisplayMode should contain the source and destination dimensions,
2047 fixed20_12 line_time;
2048 fixed20_12 src_width;
2049 fixed20_12 bandwidth;
2052 a.full = dfixed_const(1000);
2053 line_time.full = dfixed_const(wm->active_time + wm->blank_time);
2054 line_time.full = dfixed_div(line_time, a);
2055 bpp.full = dfixed_const(wm->bytes_per_pixel);
2056 src_width.full = dfixed_const(wm->src_width);
2057 bandwidth.full = dfixed_mul(src_width, bpp);
2058 bandwidth.full = dfixed_mul(bandwidth, wm->vsc);
2059 bandwidth.full = dfixed_div(bandwidth, line_time);
2061 return dfixed_trunc(bandwidth);
2064 static u32 evergreen_latency_watermark(struct evergreen_wm_params *wm)
2066 /* First calcualte the latency in ns */
2067 u32 mc_latency = 2000; /* 2000 ns. */
2068 u32 available_bandwidth = evergreen_available_bandwidth(wm);
2069 u32 worst_chunk_return_time = (512 * 8 * 1000) / available_bandwidth;
2070 u32 cursor_line_pair_return_time = (128 * 4 * 1000) / available_bandwidth;
2071 u32 dc_latency = 40000000 / wm->disp_clk; /* dc pipe latency */
2072 u32 other_heads_data_return_time = ((wm->num_heads + 1) * worst_chunk_return_time) +
2073 (wm->num_heads * cursor_line_pair_return_time);
2074 u32 latency = mc_latency + other_heads_data_return_time + dc_latency;
2075 u32 max_src_lines_per_dst_line, lb_fill_bw, line_fill_time;
2078 if (wm->num_heads == 0)
2081 a.full = dfixed_const(2);
2082 b.full = dfixed_const(1);
2083 if ((wm->vsc.full > a.full) ||
2084 ((wm->vsc.full > b.full) && (wm->vtaps >= 3)) ||
2086 ((wm->vsc.full >= a.full) && wm->interlaced))
2087 max_src_lines_per_dst_line = 4;
2089 max_src_lines_per_dst_line = 2;
2091 a.full = dfixed_const(available_bandwidth);
2092 b.full = dfixed_const(wm->num_heads);
2093 a.full = dfixed_div(a, b);
2095 lb_fill_bw = min(dfixed_trunc(a), wm->disp_clk * wm->bytes_per_pixel / 1000);
2097 a.full = dfixed_const(max_src_lines_per_dst_line * wm->src_width * wm->bytes_per_pixel);
2098 b.full = dfixed_const(1000);
2099 c.full = dfixed_const(lb_fill_bw);
2100 b.full = dfixed_div(c, b);
2101 a.full = dfixed_div(a, b);
2102 line_fill_time = dfixed_trunc(a);
2104 if (line_fill_time < wm->active_time)
2107 return latency + (line_fill_time - wm->active_time);
2111 static bool evergreen_average_bandwidth_vs_dram_bandwidth_for_display(struct evergreen_wm_params *wm)
2113 if (evergreen_average_bandwidth(wm) <=
2114 (evergreen_dram_bandwidth_for_display(wm) / wm->num_heads))
2120 static bool evergreen_average_bandwidth_vs_available_bandwidth(struct evergreen_wm_params *wm)
2122 if (evergreen_average_bandwidth(wm) <=
2123 (evergreen_available_bandwidth(wm) / wm->num_heads))
2129 static bool evergreen_check_latency_hiding(struct evergreen_wm_params *wm)
2131 u32 lb_partitions = wm->lb_size / wm->src_width;
2132 u32 line_time = wm->active_time + wm->blank_time;
2133 u32 latency_tolerant_lines;
2137 a.full = dfixed_const(1);
2138 if (wm->vsc.full > a.full)
2139 latency_tolerant_lines = 1;
2141 if (lb_partitions <= (wm->vtaps + 1))
2142 latency_tolerant_lines = 1;
2144 latency_tolerant_lines = 2;
2147 latency_hiding = (latency_tolerant_lines * line_time + wm->blank_time);
2149 if (evergreen_latency_watermark(wm) <= latency_hiding)
2155 static void evergreen_program_watermarks(struct radeon_device *rdev,
2156 struct radeon_crtc *radeon_crtc,
2157 u32 lb_size, u32 num_heads)
2159 struct drm_display_mode *mode = &radeon_crtc->base.mode;
2160 struct evergreen_wm_params wm_low, wm_high;
2164 u32 latency_watermark_a = 0, latency_watermark_b = 0;
2165 u32 priority_a_mark = 0, priority_b_mark = 0;
2166 u32 priority_a_cnt = PRIORITY_OFF;
2167 u32 priority_b_cnt = PRIORITY_OFF;
2168 u32 pipe_offset = radeon_crtc->crtc_id * 16;
2169 u32 tmp, arb_control3;
2172 if (radeon_crtc->base.enabled && num_heads && mode) {
2173 active_time = (u32) div_u64((u64)mode->crtc_hdisplay * 1000000,
2175 line_time = (u32) div_u64((u64)mode->crtc_htotal * 1000000,
2177 line_time = min(line_time, (u32)65535);
2180 dram_channels = evergreen_get_number_of_dram_channels(rdev);
2182 /* watermark for high clocks */
2183 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2185 radeon_dpm_get_mclk(rdev, false) * 10;
2187 radeon_dpm_get_sclk(rdev, false) * 10;
2189 wm_high.yclk = rdev->pm.current_mclk * 10;
2190 wm_high.sclk = rdev->pm.current_sclk * 10;
2193 wm_high.disp_clk = mode->clock;
2194 wm_high.src_width = mode->crtc_hdisplay;
2195 wm_high.active_time = active_time;
2196 wm_high.blank_time = line_time - wm_high.active_time;
2197 wm_high.interlaced = false;
2198 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2199 wm_high.interlaced = true;
2200 wm_high.vsc = radeon_crtc->vsc;
2202 if (radeon_crtc->rmx_type != RMX_OFF)
2204 wm_high.bytes_per_pixel = 4; /* XXX: get this from fb config */
2205 wm_high.lb_size = lb_size;
2206 wm_high.dram_channels = dram_channels;
2207 wm_high.num_heads = num_heads;
2209 /* watermark for low clocks */
2210 if ((rdev->pm.pm_method == PM_METHOD_DPM) && rdev->pm.dpm_enabled) {
2212 radeon_dpm_get_mclk(rdev, true) * 10;
2214 radeon_dpm_get_sclk(rdev, true) * 10;
2216 wm_low.yclk = rdev->pm.current_mclk * 10;
2217 wm_low.sclk = rdev->pm.current_sclk * 10;
2220 wm_low.disp_clk = mode->clock;
2221 wm_low.src_width = mode->crtc_hdisplay;
2222 wm_low.active_time = active_time;
2223 wm_low.blank_time = line_time - wm_low.active_time;
2224 wm_low.interlaced = false;
2225 if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2226 wm_low.interlaced = true;
2227 wm_low.vsc = radeon_crtc->vsc;
2229 if (radeon_crtc->rmx_type != RMX_OFF)
2231 wm_low.bytes_per_pixel = 4; /* XXX: get this from fb config */
2232 wm_low.lb_size = lb_size;
2233 wm_low.dram_channels = dram_channels;
2234 wm_low.num_heads = num_heads;
2236 /* set for high clocks */
2237 latency_watermark_a = min(evergreen_latency_watermark(&wm_high), (u32)65535);
2238 /* set for low clocks */
2239 latency_watermark_b = min(evergreen_latency_watermark(&wm_low), (u32)65535);
2241 /* possibly force display priority to high */
2242 /* should really do this at mode validation time... */
2243 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_high) ||
2244 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_high) ||
2245 !evergreen_check_latency_hiding(&wm_high) ||
2246 (rdev->disp_priority == 2)) {
2247 DRM_DEBUG_KMS("force priority a to high\n");
2248 priority_a_cnt |= PRIORITY_ALWAYS_ON;
2250 if (!evergreen_average_bandwidth_vs_dram_bandwidth_for_display(&wm_low) ||
2251 !evergreen_average_bandwidth_vs_available_bandwidth(&wm_low) ||
2252 !evergreen_check_latency_hiding(&wm_low) ||
2253 (rdev->disp_priority == 2)) {
2254 DRM_DEBUG_KMS("force priority b to high\n");
2255 priority_b_cnt |= PRIORITY_ALWAYS_ON;
2258 a.full = dfixed_const(1000);
2259 b.full = dfixed_const(mode->clock);
2260 b.full = dfixed_div(b, a);
2261 c.full = dfixed_const(latency_watermark_a);
2262 c.full = dfixed_mul(c, b);
2263 c.full = dfixed_mul(c, radeon_crtc->hsc);
2264 c.full = dfixed_div(c, a);
2265 a.full = dfixed_const(16);
2266 c.full = dfixed_div(c, a);
2267 priority_a_mark = dfixed_trunc(c);
2268 priority_a_cnt |= priority_a_mark & PRIORITY_MARK_MASK;
2270 a.full = dfixed_const(1000);
2271 b.full = dfixed_const(mode->clock);
2272 b.full = dfixed_div(b, a);
2273 c.full = dfixed_const(latency_watermark_b);
2274 c.full = dfixed_mul(c, b);
2275 c.full = dfixed_mul(c, radeon_crtc->hsc);
2276 c.full = dfixed_div(c, a);
2277 a.full = dfixed_const(16);
2278 c.full = dfixed_div(c, a);
2279 priority_b_mark = dfixed_trunc(c);
2280 priority_b_cnt |= priority_b_mark & PRIORITY_MARK_MASK;
2282 /* Save number of lines the linebuffer leads before the scanout */
2283 radeon_crtc->lb_vblank_lead_lines = DIV_ROUND_UP(lb_size, mode->crtc_hdisplay);
2287 arb_control3 = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2289 tmp &= ~LATENCY_WATERMARK_MASK(3);
2290 tmp |= LATENCY_WATERMARK_MASK(1);
2291 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2292 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2293 (LATENCY_LOW_WATERMARK(latency_watermark_a) |
2294 LATENCY_HIGH_WATERMARK(line_time)));
2296 tmp = RREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset);
2297 tmp &= ~LATENCY_WATERMARK_MASK(3);
2298 tmp |= LATENCY_WATERMARK_MASK(2);
2299 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, tmp);
2300 WREG32(PIPE0_LATENCY_CONTROL + pipe_offset,
2301 (LATENCY_LOW_WATERMARK(latency_watermark_b) |
2302 LATENCY_HIGH_WATERMARK(line_time)));
2303 /* restore original selection */
2304 WREG32(PIPE0_ARBITRATION_CONTROL3 + pipe_offset, arb_control3);
2306 /* write the priority marks */
2307 WREG32(PRIORITY_A_CNT + radeon_crtc->crtc_offset, priority_a_cnt);
2308 WREG32(PRIORITY_B_CNT + radeon_crtc->crtc_offset, priority_b_cnt);
2310 /* save values for DPM */
2311 radeon_crtc->line_time = line_time;
2312 radeon_crtc->wm_high = latency_watermark_a;
2313 radeon_crtc->wm_low = latency_watermark_b;
2317 * evergreen_bandwidth_update - update display watermarks callback.
2319 * @rdev: radeon_device pointer
2321 * Update the display watermarks based on the requested mode(s)
2324 void evergreen_bandwidth_update(struct radeon_device *rdev)
2326 struct drm_display_mode *mode0 = NULL;
2327 struct drm_display_mode *mode1 = NULL;
2328 u32 num_heads = 0, lb_size;
2331 if (!rdev->mode_info.mode_config_initialized)
2334 radeon_update_display_priority(rdev);
2336 for (i = 0; i < rdev->num_crtc; i++) {
2337 if (rdev->mode_info.crtcs[i]->base.enabled)
2340 for (i = 0; i < rdev->num_crtc; i += 2) {
2341 mode0 = &rdev->mode_info.crtcs[i]->base.mode;
2342 mode1 = &rdev->mode_info.crtcs[i+1]->base.mode;
2343 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i], mode0, mode1);
2344 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i], lb_size, num_heads);
2345 lb_size = evergreen_line_buffer_adjust(rdev, rdev->mode_info.crtcs[i+1], mode1, mode0);
2346 evergreen_program_watermarks(rdev, rdev->mode_info.crtcs[i+1], lb_size, num_heads);
2351 * evergreen_mc_wait_for_idle - wait for MC idle callback.
2353 * @rdev: radeon_device pointer
2355 * Wait for the MC (memory controller) to be idle.
2357 * Returns 0 if the MC is idle, -1 if not.
2359 int evergreen_mc_wait_for_idle(struct radeon_device *rdev)
2364 for (i = 0; i < rdev->usec_timeout; i++) {
2365 /* read MC_STATUS */
2366 tmp = RREG32(SRBM_STATUS) & 0x1F00;
2377 void evergreen_pcie_gart_tlb_flush(struct radeon_device *rdev)
2382 WREG32(HDP_MEM_COHERENCY_FLUSH_CNTL, 0x1);
2384 WREG32(VM_CONTEXT0_REQUEST_RESPONSE, REQUEST_TYPE(1));
2385 for (i = 0; i < rdev->usec_timeout; i++) {
2386 /* read MC_STATUS */
2387 tmp = RREG32(VM_CONTEXT0_REQUEST_RESPONSE);
2388 tmp = (tmp & RESPONSE_TYPE_MASK) >> RESPONSE_TYPE_SHIFT;
2390 pr_warn("[drm] r600 flush TLB failed\n");
2400 static int evergreen_pcie_gart_enable(struct radeon_device *rdev)
2405 if (rdev->gart.robj == NULL) {
2406 dev_err(rdev->dev, "No VRAM object for PCIE GART.\n");
2409 r = radeon_gart_table_vram_pin(rdev);
2412 /* Setup L2 cache */
2413 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2414 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2415 EFFECTIVE_L2_QUEUE_SIZE(7));
2416 WREG32(VM_L2_CNTL2, 0);
2417 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2418 /* Setup TLB control */
2419 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2420 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2421 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2422 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2423 if (rdev->flags & RADEON_IS_IGP) {
2424 WREG32(FUS_MC_VM_MD_L1_TLB0_CNTL, tmp);
2425 WREG32(FUS_MC_VM_MD_L1_TLB1_CNTL, tmp);
2426 WREG32(FUS_MC_VM_MD_L1_TLB2_CNTL, tmp);
2428 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2429 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2430 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2431 if ((rdev->family == CHIP_JUNIPER) ||
2432 (rdev->family == CHIP_CYPRESS) ||
2433 (rdev->family == CHIP_HEMLOCK) ||
2434 (rdev->family == CHIP_BARTS))
2435 WREG32(MC_VM_MD_L1_TLB3_CNTL, tmp);
2437 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2438 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2439 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2440 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2441 WREG32(VM_CONTEXT0_PAGE_TABLE_START_ADDR, rdev->mc.gtt_start >> 12);
2442 WREG32(VM_CONTEXT0_PAGE_TABLE_END_ADDR, rdev->mc.gtt_end >> 12);
2443 WREG32(VM_CONTEXT0_PAGE_TABLE_BASE_ADDR, rdev->gart.table_addr >> 12);
2444 WREG32(VM_CONTEXT0_CNTL, ENABLE_CONTEXT | PAGE_TABLE_DEPTH(0) |
2445 RANGE_PROTECTION_FAULT_ENABLE_DEFAULT);
2446 WREG32(VM_CONTEXT0_PROTECTION_FAULT_DEFAULT_ADDR,
2447 (u32)(rdev->dummy_page.addr >> 12));
2448 WREG32(VM_CONTEXT1_CNTL, 0);
2450 evergreen_pcie_gart_tlb_flush(rdev);
2451 DRM_INFO("PCIE GART of %uM enabled (table at 0x%016llX).\n",
2452 (unsigned)(rdev->mc.gtt_size >> 20),
2453 (unsigned long long)rdev->gart.table_addr);
2454 rdev->gart.ready = true;
2458 static void evergreen_pcie_gart_disable(struct radeon_device *rdev)
2462 /* Disable all tables */
2463 WREG32(VM_CONTEXT0_CNTL, 0);
2464 WREG32(VM_CONTEXT1_CNTL, 0);
2466 /* Setup L2 cache */
2467 WREG32(VM_L2_CNTL, ENABLE_L2_FRAGMENT_PROCESSING |
2468 EFFECTIVE_L2_QUEUE_SIZE(7));
2469 WREG32(VM_L2_CNTL2, 0);
2470 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2471 /* Setup TLB control */
2472 tmp = EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2473 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2474 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2475 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2476 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2477 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2478 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2479 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2480 radeon_gart_table_vram_unpin(rdev);
2483 static void evergreen_pcie_gart_fini(struct radeon_device *rdev)
2485 evergreen_pcie_gart_disable(rdev);
2486 radeon_gart_table_vram_free(rdev);
2487 radeon_gart_fini(rdev);
2491 static void evergreen_agp_enable(struct radeon_device *rdev)
2495 /* Setup L2 cache */
2496 WREG32(VM_L2_CNTL, ENABLE_L2_CACHE | ENABLE_L2_FRAGMENT_PROCESSING |
2497 ENABLE_L2_PTE_CACHE_LRU_UPDATE_BY_WRITE |
2498 EFFECTIVE_L2_QUEUE_SIZE(7));
2499 WREG32(VM_L2_CNTL2, 0);
2500 WREG32(VM_L2_CNTL3, BANK_SELECT(0) | CACHE_UPDATE_MODE(2));
2501 /* Setup TLB control */
2502 tmp = ENABLE_L1_TLB | ENABLE_L1_FRAGMENT_PROCESSING |
2503 SYSTEM_ACCESS_MODE_NOT_IN_SYS |
2504 SYSTEM_APERTURE_UNMAPPED_ACCESS_PASS_THRU |
2505 EFFECTIVE_L1_TLB_SIZE(5) | EFFECTIVE_L1_QUEUE_SIZE(5);
2506 WREG32(MC_VM_MD_L1_TLB0_CNTL, tmp);
2507 WREG32(MC_VM_MD_L1_TLB1_CNTL, tmp);
2508 WREG32(MC_VM_MD_L1_TLB2_CNTL, tmp);
2509 WREG32(MC_VM_MB_L1_TLB0_CNTL, tmp);
2510 WREG32(MC_VM_MB_L1_TLB1_CNTL, tmp);
2511 WREG32(MC_VM_MB_L1_TLB2_CNTL, tmp);
2512 WREG32(MC_VM_MB_L1_TLB3_CNTL, tmp);
2513 WREG32(VM_CONTEXT0_CNTL, 0);
2514 WREG32(VM_CONTEXT1_CNTL, 0);
2517 static const unsigned ni_dig_offsets[] = {
2518 NI_DIG0_REGISTER_OFFSET,
2519 NI_DIG1_REGISTER_OFFSET,
2520 NI_DIG2_REGISTER_OFFSET,
2521 NI_DIG3_REGISTER_OFFSET,
2522 NI_DIG4_REGISTER_OFFSET,
2523 NI_DIG5_REGISTER_OFFSET
2526 static const unsigned ni_tx_offsets[] = {
2527 NI_DCIO_UNIPHY0_UNIPHY_TX_CONTROL1,
2528 NI_DCIO_UNIPHY1_UNIPHY_TX_CONTROL1,
2529 NI_DCIO_UNIPHY2_UNIPHY_TX_CONTROL1,
2530 NI_DCIO_UNIPHY3_UNIPHY_TX_CONTROL1,
2531 NI_DCIO_UNIPHY4_UNIPHY_TX_CONTROL1,
2532 NI_DCIO_UNIPHY5_UNIPHY_TX_CONTROL1
2535 static const unsigned evergreen_dp_offsets[] = {
2536 EVERGREEN_DP0_REGISTER_OFFSET,
2537 EVERGREEN_DP1_REGISTER_OFFSET,
2538 EVERGREEN_DP2_REGISTER_OFFSET,
2539 EVERGREEN_DP3_REGISTER_OFFSET,
2540 EVERGREEN_DP4_REGISTER_OFFSET,
2541 EVERGREEN_DP5_REGISTER_OFFSET
2544 static const unsigned evergreen_disp_int_status[] = {
2545 DISP_INTERRUPT_STATUS,
2546 DISP_INTERRUPT_STATUS_CONTINUE,
2547 DISP_INTERRUPT_STATUS_CONTINUE2,
2548 DISP_INTERRUPT_STATUS_CONTINUE3,
2549 DISP_INTERRUPT_STATUS_CONTINUE4,
2550 DISP_INTERRUPT_STATUS_CONTINUE5
2554 * Assumption is that EVERGREEN_CRTC_MASTER_EN enable for requested crtc
2555 * We go from crtc to connector and it is not relible since it
2556 * should be an opposite direction .If crtc is enable then
2557 * find the dig_fe which selects this crtc and insure that it enable.
2558 * if such dig_fe is found then find dig_be which selects found dig_be and
2559 * insure that it enable and in DP_SST mode.
2560 * if UNIPHY_PLL_CONTROL1.enable then we should disconnect timing
2561 * from dp symbols clocks .
2563 static bool evergreen_is_dp_sst_stream_enabled(struct radeon_device *rdev,
2564 unsigned crtc_id, unsigned *ret_dig_fe)
2570 unsigned uniphy_pll;
2571 unsigned digs_fe_selected;
2572 unsigned dig_be_mode;
2573 unsigned dig_fe_mask;
2574 bool is_enabled = false;
2575 bool found_crtc = false;
2577 /* loop through all running dig_fe to find selected crtc */
2578 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2579 dig_fe = RREG32(NI_DIG_FE_CNTL + ni_dig_offsets[i]);
2580 if (dig_fe & NI_DIG_FE_CNTL_SYMCLK_FE_ON &&
2581 crtc_id == NI_DIG_FE_CNTL_SOURCE_SELECT(dig_fe)) {
2582 /* found running pipe */
2584 dig_fe_mask = 1 << i;
2591 /* loop through all running dig_be to find selected dig_fe */
2592 for (i = 0; i < ARRAY_SIZE(ni_dig_offsets); i++) {
2593 dig_be = RREG32(NI_DIG_BE_CNTL + ni_dig_offsets[i]);
2594 /* if dig_fe_selected by dig_be? */
2595 digs_fe_selected = NI_DIG_BE_CNTL_FE_SOURCE_SELECT(dig_be);
2596 dig_be_mode = NI_DIG_FE_CNTL_MODE(dig_be);
2597 if (dig_fe_mask & digs_fe_selected &&
2598 /* if dig_be in sst mode? */
2599 dig_be_mode == NI_DIG_BE_DPSST) {
2600 dig_en_be = RREG32(NI_DIG_BE_EN_CNTL +
2602 uniphy_pll = RREG32(NI_DCIO_UNIPHY0_PLL_CONTROL1 +
2604 /* dig_be enable and tx is running */
2605 if (dig_en_be & NI_DIG_BE_EN_CNTL_ENABLE &&
2606 dig_en_be & NI_DIG_BE_EN_CNTL_SYMBCLK_ON &&
2607 uniphy_pll & NI_DCIO_UNIPHY0_PLL_CONTROL1_ENABLE) {
2609 *ret_dig_fe = dig_fe;
2620 * Blank dig when in dp sst mode
2621 * Dig ignores crtc timing
2623 static void evergreen_blank_dp_output(struct radeon_device *rdev,
2626 unsigned stream_ctrl;
2628 unsigned counter = 0;
2630 if (dig_fe >= ARRAY_SIZE(evergreen_dp_offsets)) {
2631 DRM_ERROR("invalid dig_fe %d\n", dig_fe);
2635 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2636 evergreen_dp_offsets[dig_fe]);
2637 if (!(stream_ctrl & EVERGREEN_DP_VID_STREAM_CNTL_ENABLE)) {
2638 DRM_ERROR("dig %d , should be enable\n", dig_fe);
2642 stream_ctrl &= ~EVERGREEN_DP_VID_STREAM_CNTL_ENABLE;
2643 WREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2644 evergreen_dp_offsets[dig_fe], stream_ctrl);
2646 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2647 evergreen_dp_offsets[dig_fe]);
2648 while (counter < 32 && stream_ctrl & EVERGREEN_DP_VID_STREAM_STATUS) {
2651 stream_ctrl = RREG32(EVERGREEN_DP_VID_STREAM_CNTL +
2652 evergreen_dp_offsets[dig_fe]);
2655 DRM_ERROR("counter exceeds %d\n", counter);
2657 fifo_ctrl = RREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe]);
2658 fifo_ctrl |= EVERGREEN_DP_STEER_FIFO_RESET;
2659 WREG32(EVERGREEN_DP_STEER_FIFO + evergreen_dp_offsets[dig_fe], fifo_ctrl);
2663 void evergreen_mc_stop(struct radeon_device *rdev, struct evergreen_mc_save *save)
2665 u32 crtc_enabled, tmp, frame_count, blackout;
2669 if (!ASIC_IS_NODCE(rdev)) {
2670 save->vga_render_control = RREG32(VGA_RENDER_CONTROL);
2671 save->vga_hdp_control = RREG32(VGA_HDP_CONTROL);
2673 /* disable VGA render */
2674 WREG32(VGA_RENDER_CONTROL, 0);
2676 /* blank the display controllers */
2677 for (i = 0; i < rdev->num_crtc; i++) {
2678 crtc_enabled = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN;
2680 save->crtc_enabled[i] = true;
2681 if (ASIC_IS_DCE6(rdev)) {
2682 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2683 if (!(tmp & EVERGREEN_CRTC_BLANK_DATA_EN)) {
2684 radeon_wait_for_vblank(rdev, i);
2685 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2686 tmp |= EVERGREEN_CRTC_BLANK_DATA_EN;
2687 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2688 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2691 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2692 if (!(tmp & EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE)) {
2693 radeon_wait_for_vblank(rdev, i);
2694 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2695 tmp |= EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2696 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2697 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2700 /* wait for the next frame */
2701 frame_count = radeon_get_vblank_counter(rdev, i);
2702 for (j = 0; j < rdev->usec_timeout; j++) {
2703 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2707 /*we should disable dig if it drives dp sst*/
2708 /*but we are in radeon_device_init and the topology is unknown*/
2709 /*and it is available after radeon_modeset_init*/
2710 /*the following method radeon_atom_encoder_dpms_dig*/
2711 /*does the job if we initialize it properly*/
2712 /*for now we do it this manually*/
2714 if (ASIC_IS_DCE5(rdev) &&
2715 evergreen_is_dp_sst_stream_enabled(rdev, i, &dig_fe))
2716 evergreen_blank_dp_output(rdev, dig_fe);
2717 /*we could remove 6 lines below*/
2718 /* XXX this is a hack to avoid strange behavior with EFI on certain systems */
2719 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2720 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2721 tmp &= ~EVERGREEN_CRTC_MASTER_EN;
2722 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2723 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2724 save->crtc_enabled[i] = false;
2727 save->crtc_enabled[i] = false;
2731 radeon_mc_wait_for_idle(rdev);
2733 blackout = RREG32(MC_SHARED_BLACKOUT_CNTL);
2734 if ((blackout & BLACKOUT_MODE_MASK) != 1) {
2735 /* Block CPU access */
2736 WREG32(BIF_FB_EN, 0);
2737 /* blackout the MC */
2738 blackout &= ~BLACKOUT_MODE_MASK;
2739 WREG32(MC_SHARED_BLACKOUT_CNTL, blackout | 1);
2741 /* wait for the MC to settle */
2744 /* lock double buffered regs */
2745 for (i = 0; i < rdev->num_crtc; i++) {
2746 if (save->crtc_enabled[i]) {
2747 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2748 if (!(tmp & EVERGREEN_GRPH_UPDATE_LOCK)) {
2749 tmp |= EVERGREEN_GRPH_UPDATE_LOCK;
2750 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2752 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2755 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2761 void evergreen_mc_resume(struct radeon_device *rdev, struct evergreen_mc_save *save)
2763 u32 tmp, frame_count;
2766 /* update crtc base addresses */
2767 for (i = 0; i < rdev->num_crtc; i++) {
2768 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2769 upper_32_bits(rdev->mc.vram_start));
2770 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS_HIGH + crtc_offsets[i],
2771 upper_32_bits(rdev->mc.vram_start));
2772 WREG32(EVERGREEN_GRPH_PRIMARY_SURFACE_ADDRESS + crtc_offsets[i],
2773 (u32)rdev->mc.vram_start);
2774 WREG32(EVERGREEN_GRPH_SECONDARY_SURFACE_ADDRESS + crtc_offsets[i],
2775 (u32)rdev->mc.vram_start);
2778 if (!ASIC_IS_NODCE(rdev)) {
2779 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS_HIGH, upper_32_bits(rdev->mc.vram_start));
2780 WREG32(EVERGREEN_VGA_MEMORY_BASE_ADDRESS, (u32)rdev->mc.vram_start);
2783 /* unlock regs and wait for update */
2784 for (i = 0; i < rdev->num_crtc; i++) {
2785 if (save->crtc_enabled[i]) {
2786 tmp = RREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i]);
2787 if ((tmp & 0x7) != 0) {
2789 WREG32(EVERGREEN_MASTER_UPDATE_MODE + crtc_offsets[i], tmp);
2791 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2792 if (tmp & EVERGREEN_GRPH_UPDATE_LOCK) {
2793 tmp &= ~EVERGREEN_GRPH_UPDATE_LOCK;
2794 WREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i], tmp);
2796 tmp = RREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i]);
2799 WREG32(EVERGREEN_MASTER_UPDATE_LOCK + crtc_offsets[i], tmp);
2801 for (j = 0; j < rdev->usec_timeout; j++) {
2802 tmp = RREG32(EVERGREEN_GRPH_UPDATE + crtc_offsets[i]);
2803 if ((tmp & EVERGREEN_GRPH_SURFACE_UPDATE_PENDING) == 0)
2810 /* unblackout the MC */
2811 tmp = RREG32(MC_SHARED_BLACKOUT_CNTL);
2812 tmp &= ~BLACKOUT_MODE_MASK;
2813 WREG32(MC_SHARED_BLACKOUT_CNTL, tmp);
2814 /* allow CPU access */
2815 WREG32(BIF_FB_EN, FB_READ_EN | FB_WRITE_EN);
2817 for (i = 0; i < rdev->num_crtc; i++) {
2818 if (save->crtc_enabled[i]) {
2819 if (ASIC_IS_DCE6(rdev)) {
2820 tmp = RREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i]);
2821 tmp &= ~EVERGREEN_CRTC_BLANK_DATA_EN;
2822 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2823 WREG32(EVERGREEN_CRTC_BLANK_CONTROL + crtc_offsets[i], tmp);
2824 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2826 tmp = RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]);
2827 tmp &= ~EVERGREEN_CRTC_DISP_READ_REQUEST_DISABLE;
2828 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 1);
2829 WREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i], tmp);
2830 WREG32(EVERGREEN_CRTC_UPDATE_LOCK + crtc_offsets[i], 0);
2832 /* wait for the next frame */
2833 frame_count = radeon_get_vblank_counter(rdev, i);
2834 for (j = 0; j < rdev->usec_timeout; j++) {
2835 if (radeon_get_vblank_counter(rdev, i) != frame_count)
2841 if (!ASIC_IS_NODCE(rdev)) {
2842 /* Unlock vga access */
2843 WREG32(VGA_HDP_CONTROL, save->vga_hdp_control);
2845 WREG32(VGA_RENDER_CONTROL, save->vga_render_control);
2849 void evergreen_mc_program(struct radeon_device *rdev)
2851 struct evergreen_mc_save save;
2855 /* Initialize HDP */
2856 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
2857 WREG32((0x2c14 + j), 0x00000000);
2858 WREG32((0x2c18 + j), 0x00000000);
2859 WREG32((0x2c1c + j), 0x00000000);
2860 WREG32((0x2c20 + j), 0x00000000);
2861 WREG32((0x2c24 + j), 0x00000000);
2863 WREG32(HDP_REG_COHERENCY_FLUSH_CNTL, 0);
2865 evergreen_mc_stop(rdev, &save);
2866 if (evergreen_mc_wait_for_idle(rdev)) {
2867 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2869 /* Lockout access through VGA aperture*/
2870 WREG32(VGA_HDP_CONTROL, VGA_MEMORY_DISABLE);
2871 /* Update configuration */
2872 if (rdev->flags & RADEON_IS_AGP) {
2873 if (rdev->mc.vram_start < rdev->mc.gtt_start) {
2874 /* VRAM before AGP */
2875 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2876 rdev->mc.vram_start >> 12);
2877 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2878 rdev->mc.gtt_end >> 12);
2880 /* VRAM after AGP */
2881 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2882 rdev->mc.gtt_start >> 12);
2883 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2884 rdev->mc.vram_end >> 12);
2887 WREG32(MC_VM_SYSTEM_APERTURE_LOW_ADDR,
2888 rdev->mc.vram_start >> 12);
2889 WREG32(MC_VM_SYSTEM_APERTURE_HIGH_ADDR,
2890 rdev->mc.vram_end >> 12);
2892 WREG32(MC_VM_SYSTEM_APERTURE_DEFAULT_ADDR, rdev->vram_scratch.gpu_addr >> 12);
2893 /* llano/ontario only */
2894 if ((rdev->family == CHIP_PALM) ||
2895 (rdev->family == CHIP_SUMO) ||
2896 (rdev->family == CHIP_SUMO2)) {
2897 tmp = RREG32(MC_FUS_VM_FB_OFFSET) & 0x000FFFFF;
2898 tmp |= ((rdev->mc.vram_end >> 20) & 0xF) << 24;
2899 tmp |= ((rdev->mc.vram_start >> 20) & 0xF) << 20;
2900 WREG32(MC_FUS_VM_FB_OFFSET, tmp);
2902 tmp = ((rdev->mc.vram_end >> 24) & 0xFFFF) << 16;
2903 tmp |= ((rdev->mc.vram_start >> 24) & 0xFFFF);
2904 WREG32(MC_VM_FB_LOCATION, tmp);
2905 WREG32(HDP_NONSURFACE_BASE, (rdev->mc.vram_start >> 8));
2906 WREG32(HDP_NONSURFACE_INFO, (2 << 7) | (1 << 30));
2907 WREG32(HDP_NONSURFACE_SIZE, 0x3FFFFFFF);
2908 if (rdev->flags & RADEON_IS_AGP) {
2909 WREG32(MC_VM_AGP_TOP, rdev->mc.gtt_end >> 16);
2910 WREG32(MC_VM_AGP_BOT, rdev->mc.gtt_start >> 16);
2911 WREG32(MC_VM_AGP_BASE, rdev->mc.agp_base >> 22);
2913 WREG32(MC_VM_AGP_BASE, 0);
2914 WREG32(MC_VM_AGP_TOP, 0x0FFFFFFF);
2915 WREG32(MC_VM_AGP_BOT, 0x0FFFFFFF);
2917 if (evergreen_mc_wait_for_idle(rdev)) {
2918 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
2920 evergreen_mc_resume(rdev, &save);
2921 /* we need to own VRAM, so turn off the VGA renderer here
2922 * to stop it overwriting our objects */
2923 rv515_vga_render_disable(rdev);
2929 void evergreen_ring_ib_execute(struct radeon_device *rdev, struct radeon_ib *ib)
2931 struct radeon_ring *ring = &rdev->ring[ib->ring];
2934 /* set to DX10/11 mode */
2935 radeon_ring_write(ring, PACKET3(PACKET3_MODE_CONTROL, 0));
2936 radeon_ring_write(ring, 1);
2938 if (ring->rptr_save_reg) {
2939 next_rptr = ring->wptr + 3 + 4;
2940 radeon_ring_write(ring, PACKET3(PACKET3_SET_CONFIG_REG, 1));
2941 radeon_ring_write(ring, ((ring->rptr_save_reg -
2942 PACKET3_SET_CONFIG_REG_START) >> 2));
2943 radeon_ring_write(ring, next_rptr);
2944 } else if (rdev->wb.enabled) {
2945 next_rptr = ring->wptr + 5 + 4;
2946 radeon_ring_write(ring, PACKET3(PACKET3_MEM_WRITE, 3));
2947 radeon_ring_write(ring, ring->next_rptr_gpu_addr & 0xfffffffc);
2948 radeon_ring_write(ring, (upper_32_bits(ring->next_rptr_gpu_addr) & 0xff) | (1 << 18));
2949 radeon_ring_write(ring, next_rptr);
2950 radeon_ring_write(ring, 0);
2953 radeon_ring_write(ring, PACKET3(PACKET3_INDIRECT_BUFFER, 2));
2954 radeon_ring_write(ring,
2958 (ib->gpu_addr & 0xFFFFFFFC));
2959 radeon_ring_write(ring, upper_32_bits(ib->gpu_addr) & 0xFF);
2960 radeon_ring_write(ring, ib->length_dw);
2964 static int evergreen_cp_load_microcode(struct radeon_device *rdev)
2966 const __be32 *fw_data;
2969 if (!rdev->me_fw || !rdev->pfp_fw)
2977 RB_NO_UPDATE | RB_BLKSZ(15) | RB_BUFSZ(3));
2979 fw_data = (const __be32 *)rdev->pfp_fw->data;
2980 WREG32(CP_PFP_UCODE_ADDR, 0);
2981 for (i = 0; i < EVERGREEN_PFP_UCODE_SIZE; i++)
2982 WREG32(CP_PFP_UCODE_DATA, be32_to_cpup(fw_data++));
2983 WREG32(CP_PFP_UCODE_ADDR, 0);
2985 fw_data = (const __be32 *)rdev->me_fw->data;
2986 WREG32(CP_ME_RAM_WADDR, 0);
2987 for (i = 0; i < EVERGREEN_PM4_UCODE_SIZE; i++)
2988 WREG32(CP_ME_RAM_DATA, be32_to_cpup(fw_data++));
2990 WREG32(CP_PFP_UCODE_ADDR, 0);
2991 WREG32(CP_ME_RAM_WADDR, 0);
2992 WREG32(CP_ME_RAM_RADDR, 0);
2996 static int evergreen_cp_start(struct radeon_device *rdev)
2998 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3002 r = radeon_ring_lock(rdev, ring, 7);
3004 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3007 radeon_ring_write(ring, PACKET3(PACKET3_ME_INITIALIZE, 5));
3008 radeon_ring_write(ring, 0x1);
3009 radeon_ring_write(ring, 0x0);
3010 radeon_ring_write(ring, rdev->config.evergreen.max_hw_contexts - 1);
3011 radeon_ring_write(ring, PACKET3_ME_INITIALIZE_DEVICE_ID(1));
3012 radeon_ring_write(ring, 0);
3013 radeon_ring_write(ring, 0);
3014 radeon_ring_unlock_commit(rdev, ring, false);
3017 WREG32(CP_ME_CNTL, cp_me);
3019 r = radeon_ring_lock(rdev, ring, evergreen_default_size + 19);
3021 DRM_ERROR("radeon: cp failed to lock ring (%d).\n", r);
3025 /* setup clear context state */
3026 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3027 radeon_ring_write(ring, PACKET3_PREAMBLE_BEGIN_CLEAR_STATE);
3029 for (i = 0; i < evergreen_default_size; i++)
3030 radeon_ring_write(ring, evergreen_default_state[i]);
3032 radeon_ring_write(ring, PACKET3(PACKET3_PREAMBLE_CNTL, 0));
3033 radeon_ring_write(ring, PACKET3_PREAMBLE_END_CLEAR_STATE);
3035 /* set clear context state */
3036 radeon_ring_write(ring, PACKET3(PACKET3_CLEAR_STATE, 0));
3037 radeon_ring_write(ring, 0);
3039 /* SQ_VTX_BASE_VTX_LOC */
3040 radeon_ring_write(ring, 0xc0026f00);
3041 radeon_ring_write(ring, 0x00000000);
3042 radeon_ring_write(ring, 0x00000000);
3043 radeon_ring_write(ring, 0x00000000);
3046 radeon_ring_write(ring, 0xc0036f00);
3047 radeon_ring_write(ring, 0x00000bc4);
3048 radeon_ring_write(ring, 0xffffffff);
3049 radeon_ring_write(ring, 0xffffffff);
3050 radeon_ring_write(ring, 0xffffffff);
3052 radeon_ring_write(ring, 0xc0026900);
3053 radeon_ring_write(ring, 0x00000316);
3054 radeon_ring_write(ring, 0x0000000e); /* VGT_VERTEX_REUSE_BLOCK_CNTL */
3055 radeon_ring_write(ring, 0x00000010); /* */
3057 radeon_ring_unlock_commit(rdev, ring, false);
3062 static int evergreen_cp_resume(struct radeon_device *rdev)
3064 struct radeon_ring *ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
3069 /* Reset cp; if cp is reset, then PA, SH, VGT also need to be reset */
3070 WREG32(GRBM_SOFT_RESET, (SOFT_RESET_CP |
3076 RREG32(GRBM_SOFT_RESET);
3078 WREG32(GRBM_SOFT_RESET, 0);
3079 RREG32(GRBM_SOFT_RESET);
3081 /* Set ring buffer size */
3082 rb_bufsz = order_base_2(ring->ring_size / 8);
3083 tmp = (order_base_2(RADEON_GPU_PAGE_SIZE/8) << 8) | rb_bufsz;
3085 tmp |= BUF_SWAP_32BIT;
3087 WREG32(CP_RB_CNTL, tmp);
3088 WREG32(CP_SEM_WAIT_TIMER, 0x0);
3089 WREG32(CP_SEM_INCOMPLETE_TIMER_CNTL, 0x0);
3091 /* Set the write pointer delay */
3092 WREG32(CP_RB_WPTR_DELAY, 0);
3094 /* Initialize the ring buffer's read and write pointers */
3095 WREG32(CP_RB_CNTL, tmp | RB_RPTR_WR_ENA);
3096 WREG32(CP_RB_RPTR_WR, 0);
3098 WREG32(CP_RB_WPTR, ring->wptr);
3100 /* set the wb address whether it's enabled or not */
3101 WREG32(CP_RB_RPTR_ADDR,
3102 ((rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFFFFFFFC));
3103 WREG32(CP_RB_RPTR_ADDR_HI, upper_32_bits(rdev->wb.gpu_addr + RADEON_WB_CP_RPTR_OFFSET) & 0xFF);
3104 WREG32(SCRATCH_ADDR, ((rdev->wb.gpu_addr + RADEON_WB_SCRATCH_OFFSET) >> 8) & 0xFFFFFFFF);
3106 if (rdev->wb.enabled)
3107 WREG32(SCRATCH_UMSK, 0xff);
3109 tmp |= RB_NO_UPDATE;
3110 WREG32(SCRATCH_UMSK, 0);
3114 WREG32(CP_RB_CNTL, tmp);
3116 WREG32(CP_RB_BASE, ring->gpu_addr >> 8);
3117 WREG32(CP_DEBUG, (1 << 27) | (1 << 28));
3119 evergreen_cp_start(rdev);
3121 r = radeon_ring_test(rdev, RADEON_RING_TYPE_GFX_INDEX, ring);
3123 ring->ready = false;
3132 static void evergreen_gpu_init(struct radeon_device *rdev)
3139 u32 sq_lds_resource_mgmt;
3140 u32 sq_gpr_resource_mgmt_1;
3141 u32 sq_gpr_resource_mgmt_2;
3142 u32 sq_gpr_resource_mgmt_3;
3143 u32 sq_thread_resource_mgmt;
3144 u32 sq_thread_resource_mgmt_2;
3145 u32 sq_stack_resource_mgmt_1;
3146 u32 sq_stack_resource_mgmt_2;
3147 u32 sq_stack_resource_mgmt_3;
3148 u32 vgt_cache_invalidation;
3149 u32 hdp_host_path_cntl, tmp;
3150 u32 disabled_rb_mask;
3151 int i, j, ps_thread_count;
3153 switch (rdev->family) {
3156 rdev->config.evergreen.num_ses = 2;
3157 rdev->config.evergreen.max_pipes = 4;
3158 rdev->config.evergreen.max_tile_pipes = 8;
3159 rdev->config.evergreen.max_simds = 10;
3160 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3161 rdev->config.evergreen.max_gprs = 256;
3162 rdev->config.evergreen.max_threads = 248;
3163 rdev->config.evergreen.max_gs_threads = 32;
3164 rdev->config.evergreen.max_stack_entries = 512;
3165 rdev->config.evergreen.sx_num_of_sets = 4;
3166 rdev->config.evergreen.sx_max_export_size = 256;
3167 rdev->config.evergreen.sx_max_export_pos_size = 64;
3168 rdev->config.evergreen.sx_max_export_smx_size = 192;
3169 rdev->config.evergreen.max_hw_contexts = 8;
3170 rdev->config.evergreen.sq_num_cf_insts = 2;
3172 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3173 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3174 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3175 gb_addr_config = CYPRESS_GB_ADDR_CONFIG_GOLDEN;
3178 rdev->config.evergreen.num_ses = 1;
3179 rdev->config.evergreen.max_pipes = 4;
3180 rdev->config.evergreen.max_tile_pipes = 4;
3181 rdev->config.evergreen.max_simds = 10;
3182 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3183 rdev->config.evergreen.max_gprs = 256;
3184 rdev->config.evergreen.max_threads = 248;
3185 rdev->config.evergreen.max_gs_threads = 32;
3186 rdev->config.evergreen.max_stack_entries = 512;
3187 rdev->config.evergreen.sx_num_of_sets = 4;
3188 rdev->config.evergreen.sx_max_export_size = 256;
3189 rdev->config.evergreen.sx_max_export_pos_size = 64;
3190 rdev->config.evergreen.sx_max_export_smx_size = 192;
3191 rdev->config.evergreen.max_hw_contexts = 8;
3192 rdev->config.evergreen.sq_num_cf_insts = 2;
3194 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3195 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3196 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3197 gb_addr_config = JUNIPER_GB_ADDR_CONFIG_GOLDEN;
3200 rdev->config.evergreen.num_ses = 1;
3201 rdev->config.evergreen.max_pipes = 4;
3202 rdev->config.evergreen.max_tile_pipes = 4;
3203 rdev->config.evergreen.max_simds = 5;
3204 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3205 rdev->config.evergreen.max_gprs = 256;
3206 rdev->config.evergreen.max_threads = 248;
3207 rdev->config.evergreen.max_gs_threads = 32;
3208 rdev->config.evergreen.max_stack_entries = 256;
3209 rdev->config.evergreen.sx_num_of_sets = 4;
3210 rdev->config.evergreen.sx_max_export_size = 256;
3211 rdev->config.evergreen.sx_max_export_pos_size = 64;
3212 rdev->config.evergreen.sx_max_export_smx_size = 192;
3213 rdev->config.evergreen.max_hw_contexts = 8;
3214 rdev->config.evergreen.sq_num_cf_insts = 2;
3216 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3217 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3218 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3219 gb_addr_config = REDWOOD_GB_ADDR_CONFIG_GOLDEN;
3223 rdev->config.evergreen.num_ses = 1;
3224 rdev->config.evergreen.max_pipes = 2;
3225 rdev->config.evergreen.max_tile_pipes = 2;
3226 rdev->config.evergreen.max_simds = 2;
3227 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3228 rdev->config.evergreen.max_gprs = 256;
3229 rdev->config.evergreen.max_threads = 192;
3230 rdev->config.evergreen.max_gs_threads = 16;
3231 rdev->config.evergreen.max_stack_entries = 256;
3232 rdev->config.evergreen.sx_num_of_sets = 4;
3233 rdev->config.evergreen.sx_max_export_size = 128;
3234 rdev->config.evergreen.sx_max_export_pos_size = 32;
3235 rdev->config.evergreen.sx_max_export_smx_size = 96;
3236 rdev->config.evergreen.max_hw_contexts = 4;
3237 rdev->config.evergreen.sq_num_cf_insts = 1;
3239 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3240 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3241 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3242 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3245 rdev->config.evergreen.num_ses = 1;
3246 rdev->config.evergreen.max_pipes = 2;
3247 rdev->config.evergreen.max_tile_pipes = 2;
3248 rdev->config.evergreen.max_simds = 2;
3249 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3250 rdev->config.evergreen.max_gprs = 256;
3251 rdev->config.evergreen.max_threads = 192;
3252 rdev->config.evergreen.max_gs_threads = 16;
3253 rdev->config.evergreen.max_stack_entries = 256;
3254 rdev->config.evergreen.sx_num_of_sets = 4;
3255 rdev->config.evergreen.sx_max_export_size = 128;
3256 rdev->config.evergreen.sx_max_export_pos_size = 32;
3257 rdev->config.evergreen.sx_max_export_smx_size = 96;
3258 rdev->config.evergreen.max_hw_contexts = 4;
3259 rdev->config.evergreen.sq_num_cf_insts = 1;
3261 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3262 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3263 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3264 gb_addr_config = CEDAR_GB_ADDR_CONFIG_GOLDEN;
3267 rdev->config.evergreen.num_ses = 1;
3268 rdev->config.evergreen.max_pipes = 4;
3269 rdev->config.evergreen.max_tile_pipes = 4;
3270 if (rdev->pdev->device == 0x9648)
3271 rdev->config.evergreen.max_simds = 3;
3272 else if ((rdev->pdev->device == 0x9647) ||
3273 (rdev->pdev->device == 0x964a))
3274 rdev->config.evergreen.max_simds = 4;
3276 rdev->config.evergreen.max_simds = 5;
3277 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3278 rdev->config.evergreen.max_gprs = 256;
3279 rdev->config.evergreen.max_threads = 248;
3280 rdev->config.evergreen.max_gs_threads = 32;
3281 rdev->config.evergreen.max_stack_entries = 256;
3282 rdev->config.evergreen.sx_num_of_sets = 4;
3283 rdev->config.evergreen.sx_max_export_size = 256;
3284 rdev->config.evergreen.sx_max_export_pos_size = 64;
3285 rdev->config.evergreen.sx_max_export_smx_size = 192;
3286 rdev->config.evergreen.max_hw_contexts = 8;
3287 rdev->config.evergreen.sq_num_cf_insts = 2;
3289 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3290 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3291 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3292 gb_addr_config = SUMO_GB_ADDR_CONFIG_GOLDEN;
3295 rdev->config.evergreen.num_ses = 1;
3296 rdev->config.evergreen.max_pipes = 4;
3297 rdev->config.evergreen.max_tile_pipes = 4;
3298 rdev->config.evergreen.max_simds = 2;
3299 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3300 rdev->config.evergreen.max_gprs = 256;
3301 rdev->config.evergreen.max_threads = 248;
3302 rdev->config.evergreen.max_gs_threads = 32;
3303 rdev->config.evergreen.max_stack_entries = 512;
3304 rdev->config.evergreen.sx_num_of_sets = 4;
3305 rdev->config.evergreen.sx_max_export_size = 256;
3306 rdev->config.evergreen.sx_max_export_pos_size = 64;
3307 rdev->config.evergreen.sx_max_export_smx_size = 192;
3308 rdev->config.evergreen.max_hw_contexts = 4;
3309 rdev->config.evergreen.sq_num_cf_insts = 2;
3311 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3312 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3313 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3314 gb_addr_config = SUMO2_GB_ADDR_CONFIG_GOLDEN;
3317 rdev->config.evergreen.num_ses = 2;
3318 rdev->config.evergreen.max_pipes = 4;
3319 rdev->config.evergreen.max_tile_pipes = 8;
3320 rdev->config.evergreen.max_simds = 7;
3321 rdev->config.evergreen.max_backends = 4 * rdev->config.evergreen.num_ses;
3322 rdev->config.evergreen.max_gprs = 256;
3323 rdev->config.evergreen.max_threads = 248;
3324 rdev->config.evergreen.max_gs_threads = 32;
3325 rdev->config.evergreen.max_stack_entries = 512;
3326 rdev->config.evergreen.sx_num_of_sets = 4;
3327 rdev->config.evergreen.sx_max_export_size = 256;
3328 rdev->config.evergreen.sx_max_export_pos_size = 64;
3329 rdev->config.evergreen.sx_max_export_smx_size = 192;
3330 rdev->config.evergreen.max_hw_contexts = 8;
3331 rdev->config.evergreen.sq_num_cf_insts = 2;
3333 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3334 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3335 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3336 gb_addr_config = BARTS_GB_ADDR_CONFIG_GOLDEN;
3339 rdev->config.evergreen.num_ses = 1;
3340 rdev->config.evergreen.max_pipes = 4;
3341 rdev->config.evergreen.max_tile_pipes = 4;
3342 rdev->config.evergreen.max_simds = 6;
3343 rdev->config.evergreen.max_backends = 2 * rdev->config.evergreen.num_ses;
3344 rdev->config.evergreen.max_gprs = 256;
3345 rdev->config.evergreen.max_threads = 248;
3346 rdev->config.evergreen.max_gs_threads = 32;
3347 rdev->config.evergreen.max_stack_entries = 256;
3348 rdev->config.evergreen.sx_num_of_sets = 4;
3349 rdev->config.evergreen.sx_max_export_size = 256;
3350 rdev->config.evergreen.sx_max_export_pos_size = 64;
3351 rdev->config.evergreen.sx_max_export_smx_size = 192;
3352 rdev->config.evergreen.max_hw_contexts = 8;
3353 rdev->config.evergreen.sq_num_cf_insts = 2;
3355 rdev->config.evergreen.sc_prim_fifo_size = 0x100;
3356 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3357 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3358 gb_addr_config = TURKS_GB_ADDR_CONFIG_GOLDEN;
3361 rdev->config.evergreen.num_ses = 1;
3362 rdev->config.evergreen.max_pipes = 2;
3363 rdev->config.evergreen.max_tile_pipes = 2;
3364 rdev->config.evergreen.max_simds = 2;
3365 rdev->config.evergreen.max_backends = 1 * rdev->config.evergreen.num_ses;
3366 rdev->config.evergreen.max_gprs = 256;
3367 rdev->config.evergreen.max_threads = 192;
3368 rdev->config.evergreen.max_gs_threads = 16;
3369 rdev->config.evergreen.max_stack_entries = 256;
3370 rdev->config.evergreen.sx_num_of_sets = 4;
3371 rdev->config.evergreen.sx_max_export_size = 128;
3372 rdev->config.evergreen.sx_max_export_pos_size = 32;
3373 rdev->config.evergreen.sx_max_export_smx_size = 96;
3374 rdev->config.evergreen.max_hw_contexts = 4;
3375 rdev->config.evergreen.sq_num_cf_insts = 1;
3377 rdev->config.evergreen.sc_prim_fifo_size = 0x40;
3378 rdev->config.evergreen.sc_hiz_tile_fifo_size = 0x30;
3379 rdev->config.evergreen.sc_earlyz_tile_fifo_size = 0x130;
3380 gb_addr_config = CAICOS_GB_ADDR_CONFIG_GOLDEN;
3384 /* Initialize HDP */
3385 for (i = 0, j = 0; i < 32; i++, j += 0x18) {
3386 WREG32((0x2c14 + j), 0x00000000);
3387 WREG32((0x2c18 + j), 0x00000000);
3388 WREG32((0x2c1c + j), 0x00000000);
3389 WREG32((0x2c20 + j), 0x00000000);
3390 WREG32((0x2c24 + j), 0x00000000);
3393 WREG32(GRBM_CNTL, GRBM_READ_TIMEOUT(0xff));
3394 WREG32(SRBM_INT_CNTL, 0x1);
3395 WREG32(SRBM_INT_ACK, 0x1);
3397 evergreen_fix_pci_max_read_req_size(rdev);
3399 RREG32(MC_SHARED_CHMAP);
3400 if ((rdev->family == CHIP_PALM) ||
3401 (rdev->family == CHIP_SUMO) ||
3402 (rdev->family == CHIP_SUMO2))
3403 mc_arb_ramcfg = RREG32(FUS_MC_ARB_RAMCFG);
3405 mc_arb_ramcfg = RREG32(MC_ARB_RAMCFG);
3407 /* setup tiling info dword. gb_addr_config is not adequate since it does
3408 * not have bank info, so create a custom tiling dword.
3409 * bits 3:0 num_pipes
3410 * bits 7:4 num_banks
3411 * bits 11:8 group_size
3412 * bits 15:12 row_size
3414 rdev->config.evergreen.tile_config = 0;
3415 switch (rdev->config.evergreen.max_tile_pipes) {
3418 rdev->config.evergreen.tile_config |= (0 << 0);
3421 rdev->config.evergreen.tile_config |= (1 << 0);
3424 rdev->config.evergreen.tile_config |= (2 << 0);
3427 rdev->config.evergreen.tile_config |= (3 << 0);
3430 /* num banks is 8 on all fusion asics. 0 = 4, 1 = 8, 2 = 16 */
3431 if (rdev->flags & RADEON_IS_IGP)
3432 rdev->config.evergreen.tile_config |= 1 << 4;
3434 switch ((mc_arb_ramcfg & NOOFBANK_MASK) >> NOOFBANK_SHIFT) {
3435 case 0: /* four banks */
3436 rdev->config.evergreen.tile_config |= 0 << 4;
3438 case 1: /* eight banks */
3439 rdev->config.evergreen.tile_config |= 1 << 4;
3441 case 2: /* sixteen banks */
3443 rdev->config.evergreen.tile_config |= 2 << 4;
3447 rdev->config.evergreen.tile_config |= 0 << 8;
3448 rdev->config.evergreen.tile_config |=
3449 ((gb_addr_config & 0x30000000) >> 28) << 12;
3451 if ((rdev->family >= CHIP_CEDAR) && (rdev->family <= CHIP_HEMLOCK)) {
3455 efuse_straps_4 = RREG32_RCU(0x204);
3456 efuse_straps_3 = RREG32_RCU(0x203);
3457 tmp = (((efuse_straps_4 & 0xf) << 4) |
3458 ((efuse_straps_3 & 0xf0000000) >> 28));
3461 for (i = (rdev->config.evergreen.num_ses - 1); i >= 0; i--) {
3462 u32 rb_disable_bitmap;
3464 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3465 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3466 rb_disable_bitmap = (RREG32(CC_RB_BACKEND_DISABLE) & 0x00ff0000) >> 16;
3468 tmp |= rb_disable_bitmap;
3471 /* enabled rb are just the one not disabled :) */
3472 disabled_rb_mask = tmp;
3474 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3476 /* if all the backends are disabled, fix it up here */
3477 if ((disabled_rb_mask & tmp) == tmp) {
3478 for (i = 0; i < rdev->config.evergreen.max_backends; i++)
3479 disabled_rb_mask &= ~(1 << i);
3482 for (i = 0; i < rdev->config.evergreen.num_ses; i++) {
3483 u32 simd_disable_bitmap;
3485 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3486 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_INDEX(i));
3487 simd_disable_bitmap = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
3488 simd_disable_bitmap |= 0xffffffff << rdev->config.evergreen.max_simds;
3490 tmp |= simd_disable_bitmap;
3492 rdev->config.evergreen.active_simds = hweight32(~tmp);
3494 WREG32(GRBM_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3495 WREG32(RLC_GFX_INDEX, INSTANCE_BROADCAST_WRITES | SE_BROADCAST_WRITES);
3497 WREG32(GB_ADDR_CONFIG, gb_addr_config);
3498 WREG32(DMIF_ADDR_CONFIG, gb_addr_config);
3499 WREG32(HDP_ADDR_CONFIG, gb_addr_config);
3500 WREG32(DMA_TILING_CONFIG, gb_addr_config);
3501 WREG32(UVD_UDEC_ADDR_CONFIG, gb_addr_config);
3502 WREG32(UVD_UDEC_DB_ADDR_CONFIG, gb_addr_config);
3503 WREG32(UVD_UDEC_DBW_ADDR_CONFIG, gb_addr_config);
3505 if ((rdev->config.evergreen.max_backends == 1) &&
3506 (rdev->flags & RADEON_IS_IGP)) {
3507 if ((disabled_rb_mask & 3) == 1) {
3508 /* RB0 disabled, RB1 enabled */
3511 /* RB1 disabled, RB0 enabled */
3515 tmp = gb_addr_config & NUM_PIPES_MASK;
3516 tmp = r6xx_remap_render_backend(rdev, tmp, rdev->config.evergreen.max_backends,
3517 EVERGREEN_MAX_BACKENDS, disabled_rb_mask);
3519 rdev->config.evergreen.backend_map = tmp;
3520 WREG32(GB_BACKEND_MAP, tmp);
3522 WREG32(CGTS_SYS_TCC_DISABLE, 0);
3523 WREG32(CGTS_TCC_DISABLE, 0);
3524 WREG32(CGTS_USER_SYS_TCC_DISABLE, 0);
3525 WREG32(CGTS_USER_TCC_DISABLE, 0);
3527 /* set HW defaults for 3D engine */
3528 WREG32(CP_QUEUE_THRESHOLDS, (ROQ_IB1_START(0x16) |
3529 ROQ_IB2_START(0x2b)));
3531 WREG32(CP_MEQ_THRESHOLDS, STQ_SPLIT(0x30));
3533 WREG32(TA_CNTL_AUX, (DISABLE_CUBE_ANISO |
3538 sx_debug_1 = RREG32(SX_DEBUG_1);
3539 sx_debug_1 |= ENABLE_NEW_SMX_ADDRESS;
3540 WREG32(SX_DEBUG_1, sx_debug_1);
3543 smx_dc_ctl0 = RREG32(SMX_DC_CTL0);
3544 smx_dc_ctl0 &= ~NUMBER_OF_SETS(0x1ff);
3545 smx_dc_ctl0 |= NUMBER_OF_SETS(rdev->config.evergreen.sx_num_of_sets);
3546 WREG32(SMX_DC_CTL0, smx_dc_ctl0);
3548 if (rdev->family <= CHIP_SUMO2)
3549 WREG32(SMX_SAR_CTL0, 0x00010000);
3551 WREG32(SX_EXPORT_BUFFER_SIZES, (COLOR_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_size / 4) - 1) |
3552 POSITION_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_pos_size / 4) - 1) |
3553 SMX_BUFFER_SIZE((rdev->config.evergreen.sx_max_export_smx_size / 4) - 1)));
3555 WREG32(PA_SC_FIFO_SIZE, (SC_PRIM_FIFO_SIZE(rdev->config.evergreen.sc_prim_fifo_size) |
3556 SC_HIZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_hiz_tile_fifo_size) |
3557 SC_EARLYZ_TILE_FIFO_SIZE(rdev->config.evergreen.sc_earlyz_tile_fifo_size)));
3559 WREG32(VGT_NUM_INSTANCES, 1);
3560 WREG32(SPI_CONFIG_CNTL, 0);
3561 WREG32(SPI_CONFIG_CNTL_1, VTX_DONE_DELAY(4));
3562 WREG32(CP_PERFMON_CNTL, 0);
3564 WREG32(SQ_MS_FIFO_SIZES, (CACHE_FIFO_SIZE(16 * rdev->config.evergreen.sq_num_cf_insts) |
3565 FETCH_FIFO_HIWATER(0x4) |
3566 DONE_FIFO_HIWATER(0xe0) |
3567 ALU_UPDATE_FIFO_HIWATER(0x8)));
3569 sq_config = RREG32(SQ_CONFIG);
3570 sq_config &= ~(PS_PRIO(3) |
3574 sq_config |= (VC_ENABLE |
3581 switch (rdev->family) {
3587 /* no vertex cache */
3588 sq_config &= ~VC_ENABLE;
3594 sq_lds_resource_mgmt = RREG32(SQ_LDS_RESOURCE_MGMT);
3596 sq_gpr_resource_mgmt_1 = NUM_PS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 12 / 32);
3597 sq_gpr_resource_mgmt_1 |= NUM_VS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 6 / 32);
3598 sq_gpr_resource_mgmt_1 |= NUM_CLAUSE_TEMP_GPRS(4);
3599 sq_gpr_resource_mgmt_2 = NUM_GS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3600 sq_gpr_resource_mgmt_2 |= NUM_ES_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 4 / 32);
3601 sq_gpr_resource_mgmt_3 = NUM_HS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3602 sq_gpr_resource_mgmt_3 |= NUM_LS_GPRS((rdev->config.evergreen.max_gprs - (4 * 2)) * 3 / 32);
3604 switch (rdev->family) {
3609 ps_thread_count = 96;
3612 ps_thread_count = 128;
3616 sq_thread_resource_mgmt = NUM_PS_THREADS(ps_thread_count);
3617 sq_thread_resource_mgmt |= NUM_VS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3618 sq_thread_resource_mgmt |= NUM_GS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3619 sq_thread_resource_mgmt |= NUM_ES_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3620 sq_thread_resource_mgmt_2 = NUM_HS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3621 sq_thread_resource_mgmt_2 |= NUM_LS_THREADS((((rdev->config.evergreen.max_threads - ps_thread_count) / 6) / 8) * 8);
3623 sq_stack_resource_mgmt_1 = NUM_PS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3624 sq_stack_resource_mgmt_1 |= NUM_VS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3625 sq_stack_resource_mgmt_2 = NUM_GS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3626 sq_stack_resource_mgmt_2 |= NUM_ES_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3627 sq_stack_resource_mgmt_3 = NUM_HS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3628 sq_stack_resource_mgmt_3 |= NUM_LS_STACK_ENTRIES((rdev->config.evergreen.max_stack_entries * 1) / 6);
3630 WREG32(SQ_CONFIG, sq_config);
3631 WREG32(SQ_GPR_RESOURCE_MGMT_1, sq_gpr_resource_mgmt_1);
3632 WREG32(SQ_GPR_RESOURCE_MGMT_2, sq_gpr_resource_mgmt_2);
3633 WREG32(SQ_GPR_RESOURCE_MGMT_3, sq_gpr_resource_mgmt_3);
3634 WREG32(SQ_THREAD_RESOURCE_MGMT, sq_thread_resource_mgmt);
3635 WREG32(SQ_THREAD_RESOURCE_MGMT_2, sq_thread_resource_mgmt_2);
3636 WREG32(SQ_STACK_RESOURCE_MGMT_1, sq_stack_resource_mgmt_1);
3637 WREG32(SQ_STACK_RESOURCE_MGMT_2, sq_stack_resource_mgmt_2);
3638 WREG32(SQ_STACK_RESOURCE_MGMT_3, sq_stack_resource_mgmt_3);
3639 WREG32(SQ_DYN_GPR_CNTL_PS_FLUSH_REQ, 0);
3640 WREG32(SQ_LDS_RESOURCE_MGMT, sq_lds_resource_mgmt);
3642 WREG32(PA_SC_FORCE_EOV_MAX_CNTS, (FORCE_EOV_MAX_CLK_CNT(4095) |
3643 FORCE_EOV_MAX_REZ_CNT(255)));
3645 switch (rdev->family) {
3651 vgt_cache_invalidation = CACHE_INVALIDATION(TC_ONLY);
3654 vgt_cache_invalidation = CACHE_INVALIDATION(VC_AND_TC);
3657 vgt_cache_invalidation |= AUTO_INVLD_EN(ES_AND_GS_AUTO);
3658 WREG32(VGT_CACHE_INVALIDATION, vgt_cache_invalidation);
3660 WREG32(VGT_GS_VERTEX_REUSE, 16);
3661 WREG32(PA_SU_LINE_STIPPLE_VALUE, 0);
3662 WREG32(PA_SC_LINE_STIPPLE_STATE, 0);
3664 WREG32(VGT_VERTEX_REUSE_BLOCK_CNTL, 14);
3665 WREG32(VGT_OUT_DEALLOC_CNTL, 16);
3667 WREG32(CB_PERF_CTR0_SEL_0, 0);
3668 WREG32(CB_PERF_CTR0_SEL_1, 0);
3669 WREG32(CB_PERF_CTR1_SEL_0, 0);
3670 WREG32(CB_PERF_CTR1_SEL_1, 0);
3671 WREG32(CB_PERF_CTR2_SEL_0, 0);
3672 WREG32(CB_PERF_CTR2_SEL_1, 0);
3673 WREG32(CB_PERF_CTR3_SEL_0, 0);
3674 WREG32(CB_PERF_CTR3_SEL_1, 0);
3676 /* clear render buffer base addresses */
3677 WREG32(CB_COLOR0_BASE, 0);
3678 WREG32(CB_COLOR1_BASE, 0);
3679 WREG32(CB_COLOR2_BASE, 0);
3680 WREG32(CB_COLOR3_BASE, 0);
3681 WREG32(CB_COLOR4_BASE, 0);
3682 WREG32(CB_COLOR5_BASE, 0);
3683 WREG32(CB_COLOR6_BASE, 0);
3684 WREG32(CB_COLOR7_BASE, 0);
3685 WREG32(CB_COLOR8_BASE, 0);
3686 WREG32(CB_COLOR9_BASE, 0);
3687 WREG32(CB_COLOR10_BASE, 0);
3688 WREG32(CB_COLOR11_BASE, 0);
3690 /* set the shader const cache sizes to 0 */
3691 for (i = SQ_ALU_CONST_BUFFER_SIZE_PS_0; i < 0x28200; i += 4)
3693 for (i = SQ_ALU_CONST_BUFFER_SIZE_HS_0; i < 0x29000; i += 4)
3696 tmp = RREG32(HDP_MISC_CNTL);
3697 tmp |= HDP_FLUSH_INVALIDATE_CACHE;
3698 WREG32(HDP_MISC_CNTL, tmp);
3700 hdp_host_path_cntl = RREG32(HDP_HOST_PATH_CNTL);
3701 WREG32(HDP_HOST_PATH_CNTL, hdp_host_path_cntl);
3703 WREG32(PA_CL_ENHANCE, CLIP_VTX_REORDER_ENA | NUM_CLIP_SEQ(3));
3709 int evergreen_mc_init(struct radeon_device *rdev)
3712 int chansize, numchan;
3714 /* Get VRAM informations */
3715 rdev->mc.vram_is_ddr = true;
3716 if ((rdev->family == CHIP_PALM) ||
3717 (rdev->family == CHIP_SUMO) ||
3718 (rdev->family == CHIP_SUMO2))
3719 tmp = RREG32(FUS_MC_ARB_RAMCFG);
3721 tmp = RREG32(MC_ARB_RAMCFG);
3722 if (tmp & CHANSIZE_OVERRIDE) {
3724 } else if (tmp & CHANSIZE_MASK) {
3729 tmp = RREG32(MC_SHARED_CHMAP);
3730 switch ((tmp & NOOFCHAN_MASK) >> NOOFCHAN_SHIFT) {
3745 rdev->mc.vram_width = numchan * chansize;
3746 /* Could aper size report 0 ? */
3747 rdev->mc.aper_base = pci_resource_start(rdev->pdev, 0);
3748 rdev->mc.aper_size = pci_resource_len(rdev->pdev, 0);
3749 /* Setup GPU memory space */
3750 if ((rdev->family == CHIP_PALM) ||
3751 (rdev->family == CHIP_SUMO) ||
3752 (rdev->family == CHIP_SUMO2)) {
3753 /* size in bytes on fusion */
3754 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE);
3755 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE);
3757 /* size in MB on evergreen/cayman/tn */
3758 rdev->mc.mc_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3759 rdev->mc.real_vram_size = RREG32(CONFIG_MEMSIZE) * 1024ULL * 1024ULL;
3761 rdev->mc.visible_vram_size = rdev->mc.aper_size;
3762 r700_vram_gtt_location(rdev, &rdev->mc);
3763 radeon_update_bandwidth_info(rdev);
3768 void evergreen_print_gpu_status_regs(struct radeon_device *rdev)
3770 dev_info(rdev->dev, " GRBM_STATUS = 0x%08X\n",
3771 RREG32(GRBM_STATUS));
3772 dev_info(rdev->dev, " GRBM_STATUS_SE0 = 0x%08X\n",
3773 RREG32(GRBM_STATUS_SE0));
3774 dev_info(rdev->dev, " GRBM_STATUS_SE1 = 0x%08X\n",
3775 RREG32(GRBM_STATUS_SE1));
3776 dev_info(rdev->dev, " SRBM_STATUS = 0x%08X\n",
3777 RREG32(SRBM_STATUS));
3778 dev_info(rdev->dev, " SRBM_STATUS2 = 0x%08X\n",
3779 RREG32(SRBM_STATUS2));
3780 dev_info(rdev->dev, " R_008674_CP_STALLED_STAT1 = 0x%08X\n",
3781 RREG32(CP_STALLED_STAT1));
3782 dev_info(rdev->dev, " R_008678_CP_STALLED_STAT2 = 0x%08X\n",
3783 RREG32(CP_STALLED_STAT2));
3784 dev_info(rdev->dev, " R_00867C_CP_BUSY_STAT = 0x%08X\n",
3785 RREG32(CP_BUSY_STAT));
3786 dev_info(rdev->dev, " R_008680_CP_STAT = 0x%08X\n",
3788 dev_info(rdev->dev, " R_00D034_DMA_STATUS_REG = 0x%08X\n",
3789 RREG32(DMA_STATUS_REG));
3790 if (rdev->family >= CHIP_CAYMAN) {
3791 dev_info(rdev->dev, " R_00D834_DMA_STATUS_REG = 0x%08X\n",
3792 RREG32(DMA_STATUS_REG + 0x800));
3796 bool evergreen_is_display_hung(struct radeon_device *rdev)
3802 for (i = 0; i < rdev->num_crtc; i++) {
3803 if (RREG32(EVERGREEN_CRTC_CONTROL + crtc_offsets[i]) & EVERGREEN_CRTC_MASTER_EN) {
3804 crtc_status[i] = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3805 crtc_hung |= (1 << i);
3809 for (j = 0; j < 10; j++) {
3810 for (i = 0; i < rdev->num_crtc; i++) {
3811 if (crtc_hung & (1 << i)) {
3812 tmp = RREG32(EVERGREEN_CRTC_STATUS_HV_COUNT + crtc_offsets[i]);
3813 if (tmp != crtc_status[i])
3814 crtc_hung &= ~(1 << i);
3825 u32 evergreen_gpu_check_soft_reset(struct radeon_device *rdev)
3831 tmp = RREG32(GRBM_STATUS);
3832 if (tmp & (PA_BUSY | SC_BUSY |
3834 TA_BUSY | VGT_BUSY |
3836 SPI_BUSY | VGT_BUSY_NO_DMA))
3837 reset_mask |= RADEON_RESET_GFX;
3839 if (tmp & (CF_RQ_PENDING | PF_RQ_PENDING |
3840 CP_BUSY | CP_COHERENCY_BUSY))
3841 reset_mask |= RADEON_RESET_CP;
3843 if (tmp & GRBM_EE_BUSY)
3844 reset_mask |= RADEON_RESET_GRBM | RADEON_RESET_GFX | RADEON_RESET_CP;
3846 /* DMA_STATUS_REG */
3847 tmp = RREG32(DMA_STATUS_REG);
3848 if (!(tmp & DMA_IDLE))
3849 reset_mask |= RADEON_RESET_DMA;
3852 tmp = RREG32(SRBM_STATUS2);
3854 reset_mask |= RADEON_RESET_DMA;
3857 tmp = RREG32(SRBM_STATUS);
3858 if (tmp & (RLC_RQ_PENDING | RLC_BUSY))
3859 reset_mask |= RADEON_RESET_RLC;
3862 reset_mask |= RADEON_RESET_IH;
3865 reset_mask |= RADEON_RESET_SEM;
3867 if (tmp & GRBM_RQ_PENDING)
3868 reset_mask |= RADEON_RESET_GRBM;
3871 reset_mask |= RADEON_RESET_VMC;
3873 if (tmp & (MCB_BUSY | MCB_NON_DISPLAY_BUSY |
3874 MCC_BUSY | MCD_BUSY))
3875 reset_mask |= RADEON_RESET_MC;
3877 if (evergreen_is_display_hung(rdev))
3878 reset_mask |= RADEON_RESET_DISPLAY;
3881 tmp = RREG32(VM_L2_STATUS);
3883 reset_mask |= RADEON_RESET_VMC;
3885 /* Skip MC reset as it's mostly likely not hung, just busy */
3886 if (reset_mask & RADEON_RESET_MC) {
3887 DRM_DEBUG("MC busy: 0x%08X, clearing.\n", reset_mask);
3888 reset_mask &= ~RADEON_RESET_MC;
3894 static void evergreen_gpu_soft_reset(struct radeon_device *rdev, u32 reset_mask)
3896 struct evergreen_mc_save save;
3897 u32 grbm_soft_reset = 0, srbm_soft_reset = 0;
3900 if (reset_mask == 0)
3903 dev_info(rdev->dev, "GPU softreset: 0x%08X\n", reset_mask);
3905 evergreen_print_gpu_status_regs(rdev);
3907 /* Disable CP parsing/prefetching */
3908 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
3910 if (reset_mask & RADEON_RESET_DMA) {
3912 tmp = RREG32(DMA_RB_CNTL);
3913 tmp &= ~DMA_RB_ENABLE;
3914 WREG32(DMA_RB_CNTL, tmp);
3919 evergreen_mc_stop(rdev, &save);
3920 if (evergreen_mc_wait_for_idle(rdev)) {
3921 dev_warn(rdev->dev, "Wait for MC idle timedout !\n");
3924 if (reset_mask & (RADEON_RESET_GFX | RADEON_RESET_COMPUTE)) {
3925 grbm_soft_reset |= SOFT_RESET_DB |
3938 if (reset_mask & RADEON_RESET_CP) {
3939 grbm_soft_reset |= SOFT_RESET_CP |
3942 srbm_soft_reset |= SOFT_RESET_GRBM;
3945 if (reset_mask & RADEON_RESET_DMA)
3946 srbm_soft_reset |= SOFT_RESET_DMA;
3948 if (reset_mask & RADEON_RESET_DISPLAY)
3949 srbm_soft_reset |= SOFT_RESET_DC;
3951 if (reset_mask & RADEON_RESET_RLC)
3952 srbm_soft_reset |= SOFT_RESET_RLC;
3954 if (reset_mask & RADEON_RESET_SEM)
3955 srbm_soft_reset |= SOFT_RESET_SEM;
3957 if (reset_mask & RADEON_RESET_IH)
3958 srbm_soft_reset |= SOFT_RESET_IH;
3960 if (reset_mask & RADEON_RESET_GRBM)
3961 srbm_soft_reset |= SOFT_RESET_GRBM;
3963 if (reset_mask & RADEON_RESET_VMC)
3964 srbm_soft_reset |= SOFT_RESET_VMC;
3966 if (!(rdev->flags & RADEON_IS_IGP)) {
3967 if (reset_mask & RADEON_RESET_MC)
3968 srbm_soft_reset |= SOFT_RESET_MC;
3971 if (grbm_soft_reset) {
3972 tmp = RREG32(GRBM_SOFT_RESET);
3973 tmp |= grbm_soft_reset;
3974 dev_info(rdev->dev, "GRBM_SOFT_RESET=0x%08X\n", tmp);
3975 WREG32(GRBM_SOFT_RESET, tmp);
3976 tmp = RREG32(GRBM_SOFT_RESET);
3980 tmp &= ~grbm_soft_reset;
3981 WREG32(GRBM_SOFT_RESET, tmp);
3982 tmp = RREG32(GRBM_SOFT_RESET);
3985 if (srbm_soft_reset) {
3986 tmp = RREG32(SRBM_SOFT_RESET);
3987 tmp |= srbm_soft_reset;
3988 dev_info(rdev->dev, "SRBM_SOFT_RESET=0x%08X\n", tmp);
3989 WREG32(SRBM_SOFT_RESET, tmp);
3990 tmp = RREG32(SRBM_SOFT_RESET);
3994 tmp &= ~srbm_soft_reset;
3995 WREG32(SRBM_SOFT_RESET, tmp);
3996 tmp = RREG32(SRBM_SOFT_RESET);
3999 /* Wait a little for things to settle down */
4002 evergreen_mc_resume(rdev, &save);
4005 evergreen_print_gpu_status_regs(rdev);
4008 void evergreen_gpu_pci_config_reset(struct radeon_device *rdev)
4010 struct evergreen_mc_save save;
4013 dev_info(rdev->dev, "GPU pci config reset\n");
4017 /* Disable CP parsing/prefetching */
4018 WREG32(CP_ME_CNTL, CP_ME_HALT | CP_PFP_HALT);
4021 tmp = RREG32(DMA_RB_CNTL);
4022 tmp &= ~DMA_RB_ENABLE;
4023 WREG32(DMA_RB_CNTL, tmp);
4024 /* XXX other engines? */
4027 r600_rlc_stop(rdev);
4031 /* set mclk/sclk to bypass */
4032 rv770_set_clk_bypass_mode(rdev);
4034 pci_clear_master(rdev->pdev);
4035 /* disable mem access */
4036 evergreen_mc_stop(rdev, &save);
4037 if (evergreen_mc_wait_for_idle(rdev)) {
4038 dev_warn(rdev->dev, "Wait for MC idle timed out !\n");
4041 radeon_pci_config_reset(rdev);
4042 /* wait for asic to come out of reset */
4043 for (i = 0; i < rdev->usec_timeout; i++) {
4044 if (RREG32(CONFIG_MEMSIZE) != 0xffffffff)
4050 int evergreen_asic_reset(struct radeon_device *rdev, bool hard)
4055 evergreen_gpu_pci_config_reset(rdev);
4059 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4062 r600_set_bios_scratch_engine_hung(rdev, true);
4064 /* try soft reset */
4065 evergreen_gpu_soft_reset(rdev, reset_mask);
4067 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4069 /* try pci config reset */
4070 if (reset_mask && radeon_hard_reset)
4071 evergreen_gpu_pci_config_reset(rdev);
4073 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4076 r600_set_bios_scratch_engine_hung(rdev, false);
4082 * evergreen_gfx_is_lockup - Check if the GFX engine is locked up
4084 * @rdev: radeon_device pointer
4085 * @ring: radeon_ring structure holding ring information
4087 * Check if the GFX engine is locked up.
4088 * Returns true if the engine appears to be locked up, false if not.
4090 bool evergreen_gfx_is_lockup(struct radeon_device *rdev, struct radeon_ring *ring)
4092 u32 reset_mask = evergreen_gpu_check_soft_reset(rdev);
4094 if (!(reset_mask & (RADEON_RESET_GFX |
4095 RADEON_RESET_COMPUTE |
4096 RADEON_RESET_CP))) {
4097 radeon_ring_lockup_update(rdev, ring);
4100 return radeon_ring_test_lockup(rdev, ring);
4106 #define RLC_SAVE_RESTORE_LIST_END_MARKER 0x00000000
4107 #define RLC_CLEAR_STATE_END_MARKER 0x00000001
4109 void sumo_rlc_fini(struct radeon_device *rdev)
4113 /* save restore block */
4114 if (rdev->rlc.save_restore_obj) {
4115 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4116 if (unlikely(r != 0))
4117 dev_warn(rdev->dev, "(%d) reserve RLC sr bo failed\n", r);
4118 radeon_bo_unpin(rdev->rlc.save_restore_obj);
4119 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4121 radeon_bo_unref(&rdev->rlc.save_restore_obj);
4122 rdev->rlc.save_restore_obj = NULL;
4125 /* clear state block */
4126 if (rdev->rlc.clear_state_obj) {
4127 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4128 if (unlikely(r != 0))
4129 dev_warn(rdev->dev, "(%d) reserve RLC c bo failed\n", r);
4130 radeon_bo_unpin(rdev->rlc.clear_state_obj);
4131 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4133 radeon_bo_unref(&rdev->rlc.clear_state_obj);
4134 rdev->rlc.clear_state_obj = NULL;
4137 /* clear state block */
4138 if (rdev->rlc.cp_table_obj) {
4139 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4140 if (unlikely(r != 0))
4141 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4142 radeon_bo_unpin(rdev->rlc.cp_table_obj);
4143 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4145 radeon_bo_unref(&rdev->rlc.cp_table_obj);
4146 rdev->rlc.cp_table_obj = NULL;
4150 #define CP_ME_TABLE_SIZE 96
4152 int sumo_rlc_init(struct radeon_device *rdev)
4155 volatile u32 *dst_ptr;
4156 u32 dws, data, i, j, k, reg_num;
4157 u32 reg_list_num, reg_list_hdr_blk_index, reg_list_blk_index = 0;
4158 u64 reg_list_mc_addr;
4159 const struct cs_section_def *cs_data;
4162 src_ptr = rdev->rlc.reg_list;
4163 dws = rdev->rlc.reg_list_size;
4164 if (rdev->family >= CHIP_BONAIRE) {
4165 dws += (5 * 16) + 48 + 48 + 64;
4167 cs_data = rdev->rlc.cs_data;
4170 /* save restore block */
4171 if (rdev->rlc.save_restore_obj == NULL) {
4172 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4173 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4174 NULL, &rdev->rlc.save_restore_obj);
4176 dev_warn(rdev->dev, "(%d) create RLC sr bo failed\n", r);
4181 r = radeon_bo_reserve(rdev->rlc.save_restore_obj, false);
4182 if (unlikely(r != 0)) {
4183 sumo_rlc_fini(rdev);
4186 r = radeon_bo_pin(rdev->rlc.save_restore_obj, RADEON_GEM_DOMAIN_VRAM,
4187 &rdev->rlc.save_restore_gpu_addr);
4189 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4190 dev_warn(rdev->dev, "(%d) pin RLC sr bo failed\n", r);
4191 sumo_rlc_fini(rdev);
4195 r = radeon_bo_kmap(rdev->rlc.save_restore_obj, (void **)&rdev->rlc.sr_ptr);
4197 dev_warn(rdev->dev, "(%d) map RLC sr bo failed\n", r);
4198 sumo_rlc_fini(rdev);
4201 /* write the sr buffer */
4202 dst_ptr = rdev->rlc.sr_ptr;
4203 if (rdev->family >= CHIP_TAHITI) {
4205 for (i = 0; i < rdev->rlc.reg_list_size; i++)
4206 dst_ptr[i] = cpu_to_le32(src_ptr[i]);
4210 * dw0: (reg2 << 16) | reg1
4211 * dw1: reg1 save space
4212 * dw2: reg2 save space
4214 for (i = 0; i < dws; i++) {
4215 data = src_ptr[i] >> 2;
4218 data |= (src_ptr[i] >> 2) << 16;
4219 j = (((i - 1) * 3) / 2);
4220 dst_ptr[j] = cpu_to_le32(data);
4223 dst_ptr[j] = cpu_to_le32(RLC_SAVE_RESTORE_LIST_END_MARKER);
4225 radeon_bo_kunmap(rdev->rlc.save_restore_obj);
4226 radeon_bo_unreserve(rdev->rlc.save_restore_obj);
4230 /* clear state block */
4231 if (rdev->family >= CHIP_BONAIRE) {
4232 rdev->rlc.clear_state_size = dws = cik_get_csb_size(rdev);
4233 } else if (rdev->family >= CHIP_TAHITI) {
4234 rdev->rlc.clear_state_size = si_get_csb_size(rdev);
4235 dws = rdev->rlc.clear_state_size + (256 / 4);
4239 for (i = 0; cs_data[i].section != NULL; i++) {
4240 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4242 dws += cs_data[i].section[j].reg_count;
4245 reg_list_blk_index = (3 * reg_list_num + 2);
4246 dws += reg_list_blk_index;
4247 rdev->rlc.clear_state_size = dws;
4250 if (rdev->rlc.clear_state_obj == NULL) {
4251 r = radeon_bo_create(rdev, dws * 4, PAGE_SIZE, true,
4252 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4253 NULL, &rdev->rlc.clear_state_obj);
4255 dev_warn(rdev->dev, "(%d) create RLC c bo failed\n", r);
4256 sumo_rlc_fini(rdev);
4260 r = radeon_bo_reserve(rdev->rlc.clear_state_obj, false);
4261 if (unlikely(r != 0)) {
4262 sumo_rlc_fini(rdev);
4265 r = radeon_bo_pin(rdev->rlc.clear_state_obj, RADEON_GEM_DOMAIN_VRAM,
4266 &rdev->rlc.clear_state_gpu_addr);
4268 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4269 dev_warn(rdev->dev, "(%d) pin RLC c bo failed\n", r);
4270 sumo_rlc_fini(rdev);
4274 r = radeon_bo_kmap(rdev->rlc.clear_state_obj, (void **)&rdev->rlc.cs_ptr);
4276 dev_warn(rdev->dev, "(%d) map RLC c bo failed\n", r);
4277 sumo_rlc_fini(rdev);
4280 /* set up the cs buffer */
4281 dst_ptr = rdev->rlc.cs_ptr;
4282 if (rdev->family >= CHIP_BONAIRE) {
4283 cik_get_csb_buffer(rdev, dst_ptr);
4284 } else if (rdev->family >= CHIP_TAHITI) {
4285 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + 256;
4286 dst_ptr[0] = cpu_to_le32(upper_32_bits(reg_list_mc_addr));
4287 dst_ptr[1] = cpu_to_le32(lower_32_bits(reg_list_mc_addr));
4288 dst_ptr[2] = cpu_to_le32(rdev->rlc.clear_state_size);
4289 si_get_csb_buffer(rdev, &dst_ptr[(256/4)]);
4291 reg_list_hdr_blk_index = 0;
4292 reg_list_mc_addr = rdev->rlc.clear_state_gpu_addr + (reg_list_blk_index * 4);
4293 data = upper_32_bits(reg_list_mc_addr);
4294 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4295 reg_list_hdr_blk_index++;
4296 for (i = 0; cs_data[i].section != NULL; i++) {
4297 for (j = 0; cs_data[i].section[j].extent != NULL; j++) {
4298 reg_num = cs_data[i].section[j].reg_count;
4299 data = reg_list_mc_addr & 0xffffffff;
4300 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4301 reg_list_hdr_blk_index++;
4303 data = (cs_data[i].section[j].reg_index * 4) & 0xffffffff;
4304 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4305 reg_list_hdr_blk_index++;
4307 data = 0x08000000 | (reg_num * 4);
4308 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(data);
4309 reg_list_hdr_blk_index++;
4311 for (k = 0; k < reg_num; k++) {
4312 data = cs_data[i].section[j].extent[k];
4313 dst_ptr[reg_list_blk_index + k] = cpu_to_le32(data);
4315 reg_list_mc_addr += reg_num * 4;
4316 reg_list_blk_index += reg_num;
4319 dst_ptr[reg_list_hdr_blk_index] = cpu_to_le32(RLC_CLEAR_STATE_END_MARKER);
4321 radeon_bo_kunmap(rdev->rlc.clear_state_obj);
4322 radeon_bo_unreserve(rdev->rlc.clear_state_obj);
4325 if (rdev->rlc.cp_table_size) {
4326 if (rdev->rlc.cp_table_obj == NULL) {
4327 r = radeon_bo_create(rdev, rdev->rlc.cp_table_size,
4329 RADEON_GEM_DOMAIN_VRAM, 0, NULL,
4330 NULL, &rdev->rlc.cp_table_obj);
4332 dev_warn(rdev->dev, "(%d) create RLC cp table bo failed\n", r);
4333 sumo_rlc_fini(rdev);
4338 r = radeon_bo_reserve(rdev->rlc.cp_table_obj, false);
4339 if (unlikely(r != 0)) {
4340 dev_warn(rdev->dev, "(%d) reserve RLC cp table bo failed\n", r);
4341 sumo_rlc_fini(rdev);
4344 r = radeon_bo_pin(rdev->rlc.cp_table_obj, RADEON_GEM_DOMAIN_VRAM,
4345 &rdev->rlc.cp_table_gpu_addr);
4347 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4348 dev_warn(rdev->dev, "(%d) pin RLC cp_table bo failed\n", r);
4349 sumo_rlc_fini(rdev);
4352 r = radeon_bo_kmap(rdev->rlc.cp_table_obj, (void **)&rdev->rlc.cp_table_ptr);
4354 dev_warn(rdev->dev, "(%d) map RLC cp table bo failed\n", r);
4355 sumo_rlc_fini(rdev);
4359 cik_init_cp_pg_table(rdev);
4361 radeon_bo_kunmap(rdev->rlc.cp_table_obj);
4362 radeon_bo_unreserve(rdev->rlc.cp_table_obj);
4369 static void evergreen_rlc_start(struct radeon_device *rdev)
4371 u32 mask = RLC_ENABLE;
4373 if (rdev->flags & RADEON_IS_IGP) {
4374 mask |= GFX_POWER_GATING_ENABLE | GFX_POWER_GATING_SRC;
4377 WREG32(RLC_CNTL, mask);
4380 int evergreen_rlc_resume(struct radeon_device *rdev)
4383 const __be32 *fw_data;
4388 r600_rlc_stop(rdev);
4390 WREG32(RLC_HB_CNTL, 0);
4392 if (rdev->flags & RADEON_IS_IGP) {
4393 if (rdev->family == CHIP_ARUBA) {
4394 u32 always_on_bitmap =
4395 3 | (3 << (16 * rdev->config.cayman.max_shader_engines));
4396 /* find out the number of active simds */
4397 u32 tmp = (RREG32(CC_GC_SHADER_PIPE_CONFIG) & 0xffff0000) >> 16;
4398 tmp |= 0xffffffff << rdev->config.cayman.max_simds_per_se;
4399 tmp = hweight32(~tmp);
4400 if (tmp == rdev->config.cayman.max_simds_per_se) {
4401 WREG32(TN_RLC_LB_ALWAYS_ACTIVE_SIMD_MASK, always_on_bitmap);
4402 WREG32(TN_RLC_LB_PARAMS, 0x00601004);
4403 WREG32(TN_RLC_LB_INIT_SIMD_MASK, 0xffffffff);
4404 WREG32(TN_RLC_LB_CNTR_INIT, 0x00000000);
4405 WREG32(TN_RLC_LB_CNTR_MAX, 0x00002000);
4408 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4409 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4411 WREG32(TN_RLC_SAVE_AND_RESTORE_BASE, rdev->rlc.save_restore_gpu_addr >> 8);
4412 WREG32(TN_RLC_CLEAR_STATE_RESTORE_BASE, rdev->rlc.clear_state_gpu_addr >> 8);
4414 WREG32(RLC_HB_BASE, 0);
4415 WREG32(RLC_HB_RPTR, 0);
4416 WREG32(RLC_HB_WPTR, 0);
4417 WREG32(RLC_HB_WPTR_LSB_ADDR, 0);
4418 WREG32(RLC_HB_WPTR_MSB_ADDR, 0);
4420 WREG32(RLC_MC_CNTL, 0);
4421 WREG32(RLC_UCODE_CNTL, 0);
4423 fw_data = (const __be32 *)rdev->rlc_fw->data;
4424 if (rdev->family >= CHIP_ARUBA) {
4425 for (i = 0; i < ARUBA_RLC_UCODE_SIZE; i++) {
4426 WREG32(RLC_UCODE_ADDR, i);
4427 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4429 } else if (rdev->family >= CHIP_CAYMAN) {
4430 for (i = 0; i < CAYMAN_RLC_UCODE_SIZE; i++) {
4431 WREG32(RLC_UCODE_ADDR, i);
4432 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4435 for (i = 0; i < EVERGREEN_RLC_UCODE_SIZE; i++) {
4436 WREG32(RLC_UCODE_ADDR, i);
4437 WREG32(RLC_UCODE_DATA, be32_to_cpup(fw_data++));
4440 WREG32(RLC_UCODE_ADDR, 0);
4442 evergreen_rlc_start(rdev);
4449 u32 evergreen_get_vblank_counter(struct radeon_device *rdev, int crtc)
4451 if (crtc >= rdev->num_crtc)
4454 return RREG32(CRTC_STATUS_FRAME_COUNT + crtc_offsets[crtc]);
4457 void evergreen_disable_interrupt_state(struct radeon_device *rdev)
4462 if (rdev->family >= CHIP_CAYMAN) {
4463 cayman_cp_int_cntl_setup(rdev, 0,
4464 CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4465 cayman_cp_int_cntl_setup(rdev, 1, 0);
4466 cayman_cp_int_cntl_setup(rdev, 2, 0);
4467 tmp = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4468 WREG32(CAYMAN_DMA1_CNTL, tmp);
4470 WREG32(CP_INT_CNTL, CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE);
4471 tmp = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4472 WREG32(DMA_CNTL, tmp);
4473 WREG32(GRBM_INT_CNTL, 0);
4474 WREG32(SRBM_INT_CNTL, 0);
4475 for (i = 0; i < rdev->num_crtc; i++)
4476 WREG32(INT_MASK + crtc_offsets[i], 0);
4477 for (i = 0; i < rdev->num_crtc; i++)
4478 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], 0);
4480 /* only one DAC on DCE5 */
4481 if (!ASIC_IS_DCE5(rdev))
4482 WREG32(DACA_AUTODETECT_INT_CONTROL, 0);
4483 WREG32(DACB_AUTODETECT_INT_CONTROL, 0);
4485 for (i = 0; i < 6; i++)
4486 WREG32_AND(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_POLARITY);
4489 /* Note that the order we write back regs here is important */
4490 int evergreen_irq_set(struct radeon_device *rdev)
4493 u32 cp_int_cntl = CNTX_BUSY_INT_ENABLE | CNTX_EMPTY_INT_ENABLE;
4494 u32 cp_int_cntl1 = 0, cp_int_cntl2 = 0;
4495 u32 grbm_int_cntl = 0;
4496 u32 dma_cntl, dma_cntl1 = 0;
4497 u32 thermal_int = 0;
4499 if (!rdev->irq.installed) {
4500 WARN(1, "Can't enable IRQ/MSI because no handler is installed\n");
4503 /* don't enable anything if the ih is disabled */
4504 if (!rdev->ih.enabled) {
4505 r600_disable_interrupts(rdev);
4506 /* force the active interrupt state to all disabled */
4507 evergreen_disable_interrupt_state(rdev);
4511 if (rdev->family == CHIP_ARUBA)
4512 thermal_int = RREG32(TN_CG_THERMAL_INT_CTRL) &
4513 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4515 thermal_int = RREG32(CG_THERMAL_INT) &
4516 ~(THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW);
4518 dma_cntl = RREG32(DMA_CNTL) & ~TRAP_ENABLE;
4520 if (rdev->family >= CHIP_CAYMAN) {
4521 /* enable CP interrupts on all rings */
4522 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4523 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4524 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4526 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP1_INDEX])) {
4527 DRM_DEBUG("evergreen_irq_set: sw int cp1\n");
4528 cp_int_cntl1 |= TIME_STAMP_INT_ENABLE;
4530 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_CP2_INDEX])) {
4531 DRM_DEBUG("evergreen_irq_set: sw int cp2\n");
4532 cp_int_cntl2 |= TIME_STAMP_INT_ENABLE;
4535 if (atomic_read(&rdev->irq.ring_int[RADEON_RING_TYPE_GFX_INDEX])) {
4536 DRM_DEBUG("evergreen_irq_set: sw int gfx\n");
4537 cp_int_cntl |= RB_INT_ENABLE;
4538 cp_int_cntl |= TIME_STAMP_INT_ENABLE;
4542 if (atomic_read(&rdev->irq.ring_int[R600_RING_TYPE_DMA_INDEX])) {
4543 DRM_DEBUG("r600_irq_set: sw int dma\n");
4544 dma_cntl |= TRAP_ENABLE;
4547 if (rdev->family >= CHIP_CAYMAN) {
4548 dma_cntl1 = RREG32(CAYMAN_DMA1_CNTL) & ~TRAP_ENABLE;
4549 if (atomic_read(&rdev->irq.ring_int[CAYMAN_RING_TYPE_DMA1_INDEX])) {
4550 DRM_DEBUG("r600_irq_set: sw int dma1\n");
4551 dma_cntl1 |= TRAP_ENABLE;
4555 if (rdev->irq.dpm_thermal) {
4556 DRM_DEBUG("dpm thermal\n");
4557 thermal_int |= THERM_INT_MASK_HIGH | THERM_INT_MASK_LOW;
4560 if (rdev->family >= CHIP_CAYMAN) {
4561 cayman_cp_int_cntl_setup(rdev, 0, cp_int_cntl);
4562 cayman_cp_int_cntl_setup(rdev, 1, cp_int_cntl1);
4563 cayman_cp_int_cntl_setup(rdev, 2, cp_int_cntl2);
4565 WREG32(CP_INT_CNTL, cp_int_cntl);
4567 WREG32(DMA_CNTL, dma_cntl);
4569 if (rdev->family >= CHIP_CAYMAN)
4570 WREG32(CAYMAN_DMA1_CNTL, dma_cntl1);
4572 WREG32(GRBM_INT_CNTL, grbm_int_cntl);
4574 for (i = 0; i < rdev->num_crtc; i++) {
4575 radeon_irq_kms_set_irq_n_enabled(
4576 rdev, INT_MASK + crtc_offsets[i],
4578 rdev->irq.crtc_vblank_int[i] ||
4579 atomic_read(&rdev->irq.pflip[i]), "vblank", i);
4582 for (i = 0; i < rdev->num_crtc; i++)
4583 WREG32(GRPH_INT_CONTROL + crtc_offsets[i], GRPH_PFLIP_INT_MASK);
4585 for (i = 0; i < 6; i++) {
4586 radeon_irq_kms_set_irq_n_enabled(
4587 rdev, DC_HPDx_INT_CONTROL(i),
4588 DC_HPDx_INT_EN | DC_HPDx_RX_INT_EN,
4589 rdev->irq.hpd[i], "HPD", i);
4592 if (rdev->family == CHIP_ARUBA)
4593 WREG32(TN_CG_THERMAL_INT_CTRL, thermal_int);
4595 WREG32(CG_THERMAL_INT, thermal_int);
4597 for (i = 0; i < 6; i++) {
4598 radeon_irq_kms_set_irq_n_enabled(
4599 rdev, AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4600 AFMT_AZ_FORMAT_WTRIG_MASK,
4601 rdev->irq.afmt[i], "HDMI", i);
4605 RREG32(SRBM_STATUS);
4610 /* Note that the order we write back regs here is important */
4611 static void evergreen_irq_ack(struct radeon_device *rdev)
4614 u32 *grph_int = rdev->irq.stat_regs.evergreen.grph_int;
4615 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4616 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4618 for (i = 0; i < 6; i++) {
4619 disp_int[i] = RREG32(evergreen_disp_int_status[i]);
4620 afmt_status[i] = RREG32(AFMT_STATUS + crtc_offsets[i]);
4621 if (i < rdev->num_crtc)
4622 grph_int[i] = RREG32(GRPH_INT_STATUS + crtc_offsets[i]);
4625 /* We write back each interrupt register in pairs of two */
4626 for (i = 0; i < rdev->num_crtc; i += 2) {
4627 for (j = i; j < (i + 2); j++) {
4628 if (grph_int[j] & GRPH_PFLIP_INT_OCCURRED)
4629 WREG32(GRPH_INT_STATUS + crtc_offsets[j],
4630 GRPH_PFLIP_INT_CLEAR);
4633 for (j = i; j < (i + 2); j++) {
4634 if (disp_int[j] & LB_D1_VBLANK_INTERRUPT)
4635 WREG32(VBLANK_STATUS + crtc_offsets[j],
4637 if (disp_int[j] & LB_D1_VLINE_INTERRUPT)
4638 WREG32(VLINE_STATUS + crtc_offsets[j],
4643 for (i = 0; i < 6; i++) {
4644 if (disp_int[i] & DC_HPD1_INTERRUPT)
4645 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_INT_ACK);
4648 for (i = 0; i < 6; i++) {
4649 if (disp_int[i] & DC_HPD1_RX_INTERRUPT)
4650 WREG32_OR(DC_HPDx_INT_CONTROL(i), DC_HPDx_RX_INT_ACK);
4653 for (i = 0; i < 6; i++) {
4654 if (afmt_status[i] & AFMT_AZ_FORMAT_WTRIG)
4655 WREG32_OR(AFMT_AUDIO_PACKET_CONTROL + crtc_offsets[i],
4656 AFMT_AZ_FORMAT_WTRIG_ACK);
4660 static void evergreen_irq_disable(struct radeon_device *rdev)
4662 r600_disable_interrupts(rdev);
4663 /* Wait and acknowledge irq */
4665 evergreen_irq_ack(rdev);
4666 evergreen_disable_interrupt_state(rdev);
4669 void evergreen_irq_suspend(struct radeon_device *rdev)
4671 evergreen_irq_disable(rdev);
4672 r600_rlc_stop(rdev);
4675 static u32 evergreen_get_ih_wptr(struct radeon_device *rdev)
4679 if (rdev->wb.enabled)
4680 wptr = le32_to_cpu(rdev->wb.wb[R600_WB_IH_WPTR_OFFSET/4]);
4682 wptr = RREG32(IH_RB_WPTR);
4684 if (wptr & RB_OVERFLOW) {
4685 wptr &= ~RB_OVERFLOW;
4686 /* When a ring buffer overflow happen start parsing interrupt
4687 * from the last not overwritten vector (wptr + 16). Hopefully
4688 * this should allow us to catchup.
4690 dev_warn(rdev->dev, "IH ring buffer overflow (0x%08X, 0x%08X, 0x%08X)\n",
4691 wptr, rdev->ih.rptr, (wptr + 16) & rdev->ih.ptr_mask);
4692 rdev->ih.rptr = (wptr + 16) & rdev->ih.ptr_mask;
4693 tmp = RREG32(IH_RB_CNTL);
4694 tmp |= IH_WPTR_OVERFLOW_CLEAR;
4695 WREG32(IH_RB_CNTL, tmp);
4697 return (wptr & rdev->ih.ptr_mask);
4700 int evergreen_irq_process(struct radeon_device *rdev)
4702 u32 *disp_int = rdev->irq.stat_regs.evergreen.disp_int;
4703 u32 *afmt_status = rdev->irq.stat_regs.evergreen.afmt_status;
4704 u32 crtc_idx, hpd_idx, afmt_idx;
4708 u32 src_id, src_data;
4710 bool queue_hotplug = false;
4711 bool queue_hdmi = false;
4712 bool queue_dp = false;
4713 bool queue_thermal = false;
4715 const char *event_name;
4717 if (!rdev->ih.enabled || rdev->shutdown)
4720 wptr = evergreen_get_ih_wptr(rdev);
4723 /* is somebody else already processing irqs? */
4724 if (atomic_xchg(&rdev->ih.lock, 1))
4727 rptr = rdev->ih.rptr;
4728 DRM_DEBUG("evergreen_irq_process start: rptr %d, wptr %d\n", rptr, wptr);
4730 /* Order reading of wptr vs. reading of IH ring data */
4733 /* display interrupts */
4734 evergreen_irq_ack(rdev);
4736 while (rptr != wptr) {
4737 /* wptr/rptr are in bytes! */
4738 ring_index = rptr / 4;
4739 src_id = le32_to_cpu(rdev->ih.ring[ring_index]) & 0xff;
4740 src_data = le32_to_cpu(rdev->ih.ring[ring_index + 1]) & 0xfffffff;
4743 case 1: /* D1 vblank/vline */
4744 case 2: /* D2 vblank/vline */
4745 case 3: /* D3 vblank/vline */
4746 case 4: /* D4 vblank/vline */
4747 case 5: /* D5 vblank/vline */
4748 case 6: /* D6 vblank/vline */
4749 crtc_idx = src_id - 1;
4751 if (src_data == 0) { /* vblank */
4752 mask = LB_D1_VBLANK_INTERRUPT;
4753 event_name = "vblank";
4755 if (rdev->irq.crtc_vblank_int[crtc_idx]) {
4756 drm_handle_vblank(rdev_to_drm(rdev), crtc_idx);
4757 rdev->pm.vblank_sync = true;
4758 wake_up(&rdev->irq.vblank_queue);
4760 if (atomic_read(&rdev->irq.pflip[crtc_idx])) {
4761 radeon_crtc_handle_vblank(rdev,
4765 } else if (src_data == 1) { /* vline */
4766 mask = LB_D1_VLINE_INTERRUPT;
4767 event_name = "vline";
4769 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4774 if (!(disp_int[crtc_idx] & mask)) {
4775 DRM_DEBUG("IH: D%d %s - IH event w/o asserted irq bit?\n",
4776 crtc_idx + 1, event_name);
4779 disp_int[crtc_idx] &= ~mask;
4780 DRM_DEBUG("IH: D%d %s\n", crtc_idx + 1, event_name);
4783 case 8: /* D1 page flip */
4784 case 10: /* D2 page flip */
4785 case 12: /* D3 page flip */
4786 case 14: /* D4 page flip */
4787 case 16: /* D5 page flip */
4788 case 18: /* D6 page flip */
4789 DRM_DEBUG("IH: D%d flip\n", ((src_id - 8) >> 1) + 1);
4790 if (radeon_use_pflipirq > 0)
4791 radeon_crtc_handle_flip(rdev, (src_id - 8) >> 1);
4793 case 42: /* HPD hotplug */
4794 if (src_data <= 5) {
4796 mask = DC_HPD1_INTERRUPT;
4797 queue_hotplug = true;
4800 } else if (src_data <= 11) {
4801 hpd_idx = src_data - 6;
4802 mask = DC_HPD1_RX_INTERRUPT;
4804 event_name = "HPD_RX";
4807 DRM_DEBUG("Unhandled interrupt: %d %d\n",
4812 if (!(disp_int[hpd_idx] & mask))
4813 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4815 disp_int[hpd_idx] &= ~mask;
4816 DRM_DEBUG("IH: %s%d\n", event_name, hpd_idx + 1);
4820 afmt_idx = src_data;
4822 DRM_ERROR("Unhandled interrupt: %d %d\n",
4827 if (!(afmt_status[afmt_idx] & AFMT_AZ_FORMAT_WTRIG))
4828 DRM_DEBUG("IH: IH event w/o asserted irq bit?\n");
4830 afmt_status[afmt_idx] &= ~AFMT_AZ_FORMAT_WTRIG;
4832 DRM_DEBUG("IH: HDMI%d\n", afmt_idx + 1);
4835 DRM_ERROR("SRBM_READ_ERROR: 0x%x\n", RREG32(SRBM_READ_ERROR));
4836 WREG32(SRBM_INT_ACK, 0x1);
4839 DRM_DEBUG("IH: UVD int: 0x%08x\n", src_data);
4840 radeon_fence_process(rdev, R600_RING_TYPE_UVD_INDEX);
4844 addr = RREG32(VM_CONTEXT1_PROTECTION_FAULT_ADDR);
4845 status = RREG32(VM_CONTEXT1_PROTECTION_FAULT_STATUS);
4846 /* reset addr and status */
4847 WREG32_P(VM_CONTEXT1_CNTL2, 1, ~1);
4848 if (addr == 0x0 && status == 0x0)
4850 dev_err(rdev->dev, "GPU fault detected: %d 0x%08x\n", src_id, src_data);
4851 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_ADDR 0x%08X\n",
4853 dev_err(rdev->dev, " VM_CONTEXT1_PROTECTION_FAULT_STATUS 0x%08X\n",
4855 cayman_vm_decode_fault(rdev, status, addr);
4857 case 176: /* CP_INT in ring buffer */
4858 case 177: /* CP_INT in IB1 */
4859 case 178: /* CP_INT in IB2 */
4860 DRM_DEBUG("IH: CP int: 0x%08x\n", src_data);
4861 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4863 case 181: /* CP EOP event */
4864 DRM_DEBUG("IH: CP EOP\n");
4865 if (rdev->family >= CHIP_CAYMAN) {
4868 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4871 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP1_INDEX);
4874 radeon_fence_process(rdev, CAYMAN_RING_TYPE_CP2_INDEX);
4878 radeon_fence_process(rdev, RADEON_RING_TYPE_GFX_INDEX);
4880 case 224: /* DMA trap event */
4881 DRM_DEBUG("IH: DMA trap\n");
4882 radeon_fence_process(rdev, R600_RING_TYPE_DMA_INDEX);
4884 case 230: /* thermal low to high */
4885 DRM_DEBUG("IH: thermal low to high\n");
4886 rdev->pm.dpm.thermal.high_to_low = false;
4887 queue_thermal = true;
4889 case 231: /* thermal high to low */
4890 DRM_DEBUG("IH: thermal high to low\n");
4891 rdev->pm.dpm.thermal.high_to_low = true;
4892 queue_thermal = true;
4894 case 233: /* GUI IDLE */
4895 DRM_DEBUG("IH: GUI idle\n");
4897 case 244: /* DMA trap event */
4898 if (rdev->family >= CHIP_CAYMAN) {
4899 DRM_DEBUG("IH: DMA1 trap\n");
4900 radeon_fence_process(rdev, CAYMAN_RING_TYPE_DMA1_INDEX);
4904 DRM_DEBUG("Unhandled interrupt: %d %d\n", src_id, src_data);
4908 /* wptr/rptr are in bytes! */
4910 rptr &= rdev->ih.ptr_mask;
4911 WREG32(IH_RB_RPTR, rptr);
4914 schedule_work(&rdev->dp_work);
4916 schedule_delayed_work(&rdev->hotplug_work, 0);
4918 schedule_work(&rdev->audio_work);
4919 if (queue_thermal && rdev->pm.dpm_enabled)
4920 schedule_work(&rdev->pm.dpm.thermal.work);
4921 rdev->ih.rptr = rptr;
4922 atomic_set(&rdev->ih.lock, 0);
4924 /* make sure wptr hasn't changed while processing */
4925 wptr = evergreen_get_ih_wptr(rdev);
4932 static void evergreen_uvd_init(struct radeon_device *rdev)
4939 r = radeon_uvd_init(rdev);
4941 dev_err(rdev->dev, "failed UVD (%d) init.\n", r);
4943 * At this point rdev->uvd.vcpu_bo is NULL which trickles down
4944 * to early fails uvd_v2_2_resume() and thus nothing happens
4945 * there. So it is pointless to try to go through that code
4946 * hence why we disable uvd here.
4948 rdev->has_uvd = false;
4951 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_obj = NULL;
4952 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_UVD_INDEX], 4096);
4955 static void evergreen_uvd_start(struct radeon_device *rdev)
4962 r = uvd_v2_2_resume(rdev);
4964 dev_err(rdev->dev, "failed UVD resume (%d).\n", r);
4967 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_UVD_INDEX);
4969 dev_err(rdev->dev, "failed initializing UVD fences (%d).\n", r);
4975 rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size = 0;
4978 static void evergreen_uvd_resume(struct radeon_device *rdev)
4980 struct radeon_ring *ring;
4983 if (!rdev->has_uvd || !rdev->ring[R600_RING_TYPE_UVD_INDEX].ring_size)
4986 ring = &rdev->ring[R600_RING_TYPE_UVD_INDEX];
4987 r = radeon_ring_init(rdev, ring, ring->ring_size, 0, PACKET0(UVD_NO_OP, 0));
4989 dev_err(rdev->dev, "failed initializing UVD ring (%d).\n", r);
4992 r = uvd_v1_0_init(rdev);
4994 dev_err(rdev->dev, "failed initializing UVD (%d).\n", r);
4999 static int evergreen_startup(struct radeon_device *rdev)
5001 struct radeon_ring *ring;
5004 /* enable pcie gen2 link */
5005 evergreen_pcie_gen2_enable(rdev);
5007 evergreen_program_aspm(rdev);
5009 /* scratch needs to be initialized before MC */
5010 r = r600_vram_scratch_init(rdev);
5014 evergreen_mc_program(rdev);
5016 if (ASIC_IS_DCE5(rdev) && !rdev->pm.dpm_enabled) {
5017 r = ni_mc_load_microcode(rdev);
5019 DRM_ERROR("Failed to load MC firmware!\n");
5024 if (rdev->flags & RADEON_IS_AGP) {
5025 evergreen_agp_enable(rdev);
5027 r = evergreen_pcie_gart_enable(rdev);
5031 evergreen_gpu_init(rdev);
5033 /* allocate rlc buffers */
5034 if (rdev->flags & RADEON_IS_IGP) {
5035 rdev->rlc.reg_list = sumo_rlc_save_restore_register_list;
5036 rdev->rlc.reg_list_size =
5037 (u32)ARRAY_SIZE(sumo_rlc_save_restore_register_list);
5038 rdev->rlc.cs_data = evergreen_cs_data;
5039 r = sumo_rlc_init(rdev);
5041 DRM_ERROR("Failed to init rlc BOs!\n");
5046 /* allocate wb buffer */
5047 r = radeon_wb_init(rdev);
5051 r = radeon_fence_driver_start_ring(rdev, RADEON_RING_TYPE_GFX_INDEX);
5053 dev_err(rdev->dev, "failed initializing CP fences (%d).\n", r);
5057 r = radeon_fence_driver_start_ring(rdev, R600_RING_TYPE_DMA_INDEX);
5059 dev_err(rdev->dev, "failed initializing DMA fences (%d).\n", r);
5063 evergreen_uvd_start(rdev);
5066 if (!rdev->irq.installed) {
5067 r = radeon_irq_kms_init(rdev);
5072 r = r600_irq_init(rdev);
5074 DRM_ERROR("radeon: IH init failed (%d).\n", r);
5075 radeon_irq_kms_fini(rdev);
5078 evergreen_irq_set(rdev);
5080 ring = &rdev->ring[RADEON_RING_TYPE_GFX_INDEX];
5081 r = radeon_ring_init(rdev, ring, ring->ring_size, RADEON_WB_CP_RPTR_OFFSET,
5086 ring = &rdev->ring[R600_RING_TYPE_DMA_INDEX];
5087 r = radeon_ring_init(rdev, ring, ring->ring_size, R600_WB_DMA_RPTR_OFFSET,
5088 DMA_PACKET(DMA_PACKET_NOP, 0, 0));
5092 r = evergreen_cp_load_microcode(rdev);
5095 r = evergreen_cp_resume(rdev);
5098 r = r600_dma_resume(rdev);
5102 evergreen_uvd_resume(rdev);
5104 r = radeon_ib_pool_init(rdev);
5106 dev_err(rdev->dev, "IB initialization failed (%d).\n", r);
5110 r = radeon_audio_init(rdev);
5112 DRM_ERROR("radeon: audio init failed\n");
5119 int evergreen_resume(struct radeon_device *rdev)
5123 /* reset the asic, the gfx blocks are often in a bad state
5124 * after the driver is unloaded or after a resume
5126 if (radeon_asic_reset(rdev))
5127 dev_warn(rdev->dev, "GPU reset failed !\n");
5128 /* Do not reset GPU before posting, on rv770 hw unlike on r500 hw,
5129 * posting will perform necessary task to bring back GPU into good
5133 atom_asic_init(rdev->mode_info.atom_context);
5135 /* init golden registers */
5136 evergreen_init_golden_registers(rdev);
5138 if (rdev->pm.pm_method == PM_METHOD_DPM)
5139 radeon_pm_resume(rdev);
5141 rdev->accel_working = true;
5142 r = evergreen_startup(rdev);
5144 DRM_ERROR("evergreen startup failed on resume\n");
5145 rdev->accel_working = false;
5153 int evergreen_suspend(struct radeon_device *rdev)
5155 radeon_pm_suspend(rdev);
5156 radeon_audio_fini(rdev);
5157 if (rdev->has_uvd) {
5158 radeon_uvd_suspend(rdev);
5159 uvd_v1_0_fini(rdev);
5162 r600_dma_stop(rdev);
5163 evergreen_irq_suspend(rdev);
5164 radeon_wb_disable(rdev);
5165 evergreen_pcie_gart_disable(rdev);
5170 /* Plan is to move initialization in that function and use
5171 * helper function so that radeon_device_init pretty much
5172 * do nothing more than calling asic specific function. This
5173 * should also allow to remove a bunch of callback function
5176 int evergreen_init(struct radeon_device *rdev)
5181 if (!radeon_get_bios(rdev)) {
5182 if (ASIC_IS_AVIVO(rdev))
5185 /* Must be an ATOMBIOS */
5186 if (!rdev->is_atom_bios) {
5187 dev_err(rdev->dev, "Expecting atombios for evergreen GPU\n");
5190 r = radeon_atombios_init(rdev);
5193 /* reset the asic, the gfx blocks are often in a bad state
5194 * after the driver is unloaded or after a resume
5196 if (radeon_asic_reset(rdev))
5197 dev_warn(rdev->dev, "GPU reset failed !\n");
5198 /* Post card if necessary */
5199 if (!radeon_card_posted(rdev)) {
5201 dev_err(rdev->dev, "Card not posted and no BIOS - ignoring\n");
5204 DRM_INFO("GPU not posted. posting now...\n");
5205 atom_asic_init(rdev->mode_info.atom_context);
5207 /* init golden registers */
5208 evergreen_init_golden_registers(rdev);
5209 /* Initialize scratch registers */
5210 r600_scratch_init(rdev);
5211 /* Initialize surface registers */
5212 radeon_surface_init(rdev);
5213 /* Initialize clocks */
5214 radeon_get_clock_info(rdev_to_drm(rdev));
5216 radeon_fence_driver_init(rdev);
5217 /* initialize AGP */
5218 if (rdev->flags & RADEON_IS_AGP) {
5219 r = radeon_agp_init(rdev);
5221 radeon_agp_disable(rdev);
5223 /* initialize memory controller */
5224 r = evergreen_mc_init(rdev);
5227 /* Memory manager */
5228 r = radeon_bo_init(rdev);
5232 if (ASIC_IS_DCE5(rdev)) {
5233 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw || !rdev->mc_fw) {
5234 r = ni_init_microcode(rdev);
5236 DRM_ERROR("Failed to load firmware!\n");
5241 if (!rdev->me_fw || !rdev->pfp_fw || !rdev->rlc_fw) {
5242 r = r600_init_microcode(rdev);
5244 DRM_ERROR("Failed to load firmware!\n");
5250 /* Initialize power management */
5251 radeon_pm_init(rdev);
5253 rdev->ring[RADEON_RING_TYPE_GFX_INDEX].ring_obj = NULL;
5254 r600_ring_init(rdev, &rdev->ring[RADEON_RING_TYPE_GFX_INDEX], 1024 * 1024);
5256 rdev->ring[R600_RING_TYPE_DMA_INDEX].ring_obj = NULL;
5257 r600_ring_init(rdev, &rdev->ring[R600_RING_TYPE_DMA_INDEX], 64 * 1024);
5259 evergreen_uvd_init(rdev);
5261 rdev->ih.ring_obj = NULL;
5262 r600_ih_ring_init(rdev, 64 * 1024);
5264 r = r600_pcie_gart_init(rdev);
5268 rdev->accel_working = true;
5269 r = evergreen_startup(rdev);
5271 dev_err(rdev->dev, "disabling GPU acceleration\n");
5273 r600_dma_fini(rdev);
5274 r600_irq_fini(rdev);
5275 if (rdev->flags & RADEON_IS_IGP)
5276 sumo_rlc_fini(rdev);
5277 radeon_wb_fini(rdev);
5278 radeon_ib_pool_fini(rdev);
5279 radeon_irq_kms_fini(rdev);
5280 evergreen_pcie_gart_fini(rdev);
5281 rdev->accel_working = false;
5284 /* Don't start up if the MC ucode is missing on BTC parts.
5285 * The default clocks and voltages before the MC ucode
5286 * is loaded are not suffient for advanced operations.
5288 if (ASIC_IS_DCE5(rdev)) {
5289 if (!rdev->mc_fw && !(rdev->flags & RADEON_IS_IGP)) {
5290 DRM_ERROR("radeon: MC ucode required for NI+.\n");
5298 void evergreen_fini(struct radeon_device *rdev)
5300 radeon_pm_fini(rdev);
5301 radeon_audio_fini(rdev);
5303 r600_dma_fini(rdev);
5304 r600_irq_fini(rdev);
5305 if (rdev->flags & RADEON_IS_IGP)
5306 sumo_rlc_fini(rdev);
5307 radeon_wb_fini(rdev);
5308 radeon_ib_pool_fini(rdev);
5309 radeon_irq_kms_fini(rdev);
5310 uvd_v1_0_fini(rdev);
5311 radeon_uvd_fini(rdev);
5312 evergreen_pcie_gart_fini(rdev);
5313 r600_vram_scratch_fini(rdev);
5314 radeon_gem_fini(rdev);
5315 radeon_fence_driver_fini(rdev);
5316 radeon_agp_fini(rdev);
5317 radeon_bo_fini(rdev);
5318 radeon_atombios_fini(rdev);
5323 void evergreen_pcie_gen2_enable(struct radeon_device *rdev)
5325 u32 link_width_cntl, speed_cntl;
5327 if (radeon_pcie_gen2 == 0)
5330 if (rdev->flags & RADEON_IS_IGP)
5333 if (!(rdev->flags & RADEON_IS_PCIE))
5336 /* x2 cards have a special sequence */
5337 if (ASIC_IS_X2(rdev))
5340 if ((rdev->pdev->bus->max_bus_speed != PCIE_SPEED_5_0GT) &&
5341 (rdev->pdev->bus->max_bus_speed != PCIE_SPEED_8_0GT))
5344 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5345 if (speed_cntl & LC_CURRENT_DATA_RATE) {
5346 DRM_INFO("PCIE gen 2 link speeds already enabled\n");
5350 DRM_INFO("enabling PCIE gen 2 link speeds, disable with radeon.pcie_gen2=0\n");
5352 if ((speed_cntl & LC_OTHER_SIDE_EVER_SENT_GEN2) ||
5353 (speed_cntl & LC_OTHER_SIDE_SUPPORTS_GEN2)) {
5355 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5356 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5357 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5359 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5360 speed_cntl &= ~LC_TARGET_LINK_SPEED_OVERRIDE_EN;
5361 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5363 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5364 speed_cntl |= LC_CLR_FAILED_SPD_CHANGE_CNT;
5365 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5367 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5368 speed_cntl &= ~LC_CLR_FAILED_SPD_CHANGE_CNT;
5369 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5371 speed_cntl = RREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL);
5372 speed_cntl |= LC_GEN2_EN_STRAP;
5373 WREG32_PCIE_PORT(PCIE_LC_SPEED_CNTL, speed_cntl);
5376 link_width_cntl = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5377 /* XXX: only disable it if gen1 bridge vendor == 0x111d or 0x1106 */
5379 link_width_cntl |= LC_UPCONFIGURE_DIS;
5381 link_width_cntl &= ~LC_UPCONFIGURE_DIS;
5382 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, link_width_cntl);
5386 void evergreen_program_aspm(struct radeon_device *rdev)
5389 u32 pcie_lc_cntl, pcie_lc_cntl_old;
5390 bool disable_l0s, disable_l1 = false, disable_plloff_in_l1 = false;
5391 /* fusion_platform = true
5392 * if the system is a fusion system
5393 * (APU or DGPU in a fusion system).
5394 * todo: check if the system is a fusion platform.
5396 bool fusion_platform = false;
5398 if (radeon_aspm == 0)
5401 if (!(rdev->flags & RADEON_IS_PCIE))
5404 switch (rdev->family) {
5417 disable_l0s = false;
5421 if (rdev->flags & RADEON_IS_IGP)
5422 fusion_platform = true; /* XXX also dGPUs in a fusion system */
5424 data = orig = RREG32_PIF_PHY0(PB0_PIF_PAIRING);
5425 if (fusion_platform)
5430 WREG32_PIF_PHY0(PB0_PIF_PAIRING, data);
5432 data = orig = RREG32_PIF_PHY1(PB1_PIF_PAIRING);
5433 if (fusion_platform)
5438 WREG32_PIF_PHY1(PB1_PIF_PAIRING, data);
5440 pcie_lc_cntl = pcie_lc_cntl_old = RREG32_PCIE_PORT(PCIE_LC_CNTL);
5441 pcie_lc_cntl &= ~(LC_L0S_INACTIVITY_MASK | LC_L1_INACTIVITY_MASK);
5443 if (rdev->family >= CHIP_BARTS)
5444 pcie_lc_cntl |= LC_L0S_INACTIVITY(7);
5446 pcie_lc_cntl |= LC_L0S_INACTIVITY(3);
5450 if (rdev->family >= CHIP_BARTS)
5451 pcie_lc_cntl |= LC_L1_INACTIVITY(7);
5453 pcie_lc_cntl |= LC_L1_INACTIVITY(8);
5455 if (!disable_plloff_in_l1) {
5456 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5457 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5458 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5460 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5462 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5463 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5464 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5466 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5468 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5469 data &= ~(PLL_POWER_STATE_IN_OFF_0_MASK | PLL_POWER_STATE_IN_TXS2_0_MASK);
5470 data |= PLL_POWER_STATE_IN_OFF_0(7) | PLL_POWER_STATE_IN_TXS2_0(7);
5472 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5474 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5475 data &= ~(PLL_POWER_STATE_IN_OFF_1_MASK | PLL_POWER_STATE_IN_TXS2_1_MASK);
5476 data |= PLL_POWER_STATE_IN_OFF_1(7) | PLL_POWER_STATE_IN_TXS2_1(7);
5478 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5480 if (rdev->family >= CHIP_BARTS) {
5481 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0);
5482 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5483 data |= PLL_RAMP_UP_TIME_0(4);
5485 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_0, data);
5487 data = orig = RREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1);
5488 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5489 data |= PLL_RAMP_UP_TIME_1(4);
5491 WREG32_PIF_PHY0(PB0_PIF_PWRDOWN_1, data);
5493 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0);
5494 data &= ~PLL_RAMP_UP_TIME_0_MASK;
5495 data |= PLL_RAMP_UP_TIME_0(4);
5497 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_0, data);
5499 data = orig = RREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1);
5500 data &= ~PLL_RAMP_UP_TIME_1_MASK;
5501 data |= PLL_RAMP_UP_TIME_1(4);
5503 WREG32_PIF_PHY1(PB1_PIF_PWRDOWN_1, data);
5506 data = orig = RREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL);
5507 data &= ~LC_DYN_LANES_PWR_STATE_MASK;
5508 data |= LC_DYN_LANES_PWR_STATE(3);
5510 WREG32_PCIE_PORT(PCIE_LC_LINK_WIDTH_CNTL, data);
5512 if (rdev->family >= CHIP_BARTS) {
5513 data = orig = RREG32_PIF_PHY0(PB0_PIF_CNTL);
5514 data &= ~LS2_EXIT_TIME_MASK;
5515 data |= LS2_EXIT_TIME(1);
5517 WREG32_PIF_PHY0(PB0_PIF_CNTL, data);
5519 data = orig = RREG32_PIF_PHY1(PB1_PIF_CNTL);
5520 data &= ~LS2_EXIT_TIME_MASK;
5521 data |= LS2_EXIT_TIME(1);
5523 WREG32_PIF_PHY1(PB1_PIF_CNTL, data);
5528 /* evergreen parts only */
5529 if (rdev->family < CHIP_BARTS)
5530 pcie_lc_cntl |= LC_PMI_TO_L1_DIS;
5532 if (pcie_lc_cntl != pcie_lc_cntl_old)
5533 WREG32_PCIE_PORT(PCIE_LC_CNTL, pcie_lc_cntl);