drivers/gpu/drm/nouveau/nv50_display.c

   1 /*
   2  * Copyright 2011 Red Hat Inc.
   3  *
   4  * Permission is hereby granted, free of charge, to any person obtaining a
   5  * copy of this software and associated documentation files (the "Software"),
   6  * to deal in the Software without restriction, including without limitation
   7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8  * and/or sell copies of the Software, and to permit persons to whom the
   9  * Software is furnished to do so, subject to the following conditions:
  10  *
  11  * The above copyright notice and this permission notice shall be included in
  12  * all copies or substantial portions of the Software.
  13  *
  14  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  17  * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18  * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19  * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20  * OTHER DEALINGS IN THE SOFTWARE.
  21  *
  22  * Authors: Ben Skeggs
  23  */
  24
  25 #include <linux/dma-mapping.h>
  26
  27 #include <drm/drmP.h>
  28 #include <drm/drm_atomic.h>
  29 #include <drm/drm_atomic_helper.h>
  30 #include <drm/drm_crtc_helper.h>
  31 #include <drm/drm_dp_helper.h>
  32 #include <drm/drm_fb_helper.h>
  33 #include <drm/drm_plane_helper.h>
  34
  35 #include <nvif/class.h>
  36 #include <nvif/cl0002.h>
  37 #include <nvif/cl5070.h>
  38 #include <nvif/cl507a.h>
  39 #include <nvif/cl507b.h>
  40 #include <nvif/cl507c.h>
  41 #include <nvif/cl507d.h>
  42 #include <nvif/cl507e.h>
  43 #include <nvif/event.h>
  44
  45 #include "nouveau_drv.h"
  46 #include "nouveau_dma.h"
  47 #include "nouveau_gem.h"
  48 #include "nouveau_connector.h"
  49 #include "nouveau_encoder.h"
  50 #include "nouveau_crtc.h"
  51 #include "nouveau_fence.h"
  52 #include "nv50_display.h"
  53
  54 #define EVO_DMA_NR 9
  55
  56 #define EVO_MASTER  (0x00)
  57 #define EVO_FLIP(c) (0x01 + (c))
  58 #define EVO_OVLY(c) (0x05 + (c))
  59 #define EVO_OIMM(c) (0x09 + (c))
  60 #define EVO_CURS(c) (0x0d + (c))
  61
  62 /* offsets in shared sync bo of various structures */
  63 #define EVO_SYNC(c, o) ((c) * 0x0100 + (o))
  64 #define EVO_MAST_NTFY     EVO_SYNC(      0, 0x00)
  65 #define EVO_FLIP_SEM0(c)  EVO_SYNC((c) + 1, 0x00)
  66 #define EVO_FLIP_SEM1(c)  EVO_SYNC((c) + 1, 0x10)
  67 #define EVO_FLIP_NTFY0(c) EVO_SYNC((c) + 1, 0x20)
  68 #define EVO_FLIP_NTFY1(c) EVO_SYNC((c) + 1, 0x30)
  69
  70 /******************************************************************************
  71  * Atomic state
  72  *****************************************************************************/
  73 #define nv50_head_atom(p) container_of((p), struct nv50_head_atom, state)
  74
  75 struct nv50_head_atom {
  76         struct drm_crtc_state state;
  77
  78         struct {
  79                 u16 iW;
  80                 u16 iH;
  81                 u16 oW;
  82                 u16 oH;
  83         } view;
  84
  85         struct nv50_head_mode {
  86                 bool interlace;
  87                 u32 clock;
  88                 struct {
  89                         u16 active;
  90                         u16 synce;
  91                         u16 blanke;
  92                         u16 blanks;
  93                 } h;
  94                 struct {
  95                         u32 active;
  96                         u16 synce;
  97                         u16 blanke;
  98                         u16 blanks;
  99                         u16 blank2s;
 100                         u16 blank2e;
 101                         u16 blankus;
 102                 } v;
 103         } mode;
 104
 105         struct {
 106                 u32 handle;
 107                 u64 offset:40;
 108         } lut;
 109
 110         struct {
 111                 bool visible;
 112                 u32 handle;
 113                 u64 offset:40;
 114                 u8  format;
 115                 u8  kind:7;
 116                 u8  layout:1;
 117                 u8  block:4;
 118                 u32 pitch:20;
 119                 u16 x;
 120                 u16 y;
 121                 u16 w;
 122                 u16 h;
 123         } core;
 124
 125         struct {
 126                 bool visible;
 127                 u32 handle;
 128                 u64 offset:40;
 129                 u8  layout:1;
 130                 u8  format:1;
 131         } curs;
 132
 133         struct {
 134                 u8  depth;
 135                 u8  cpp;
 136                 u16 x;
 137                 u16 y;
 138                 u16 w;
 139                 u16 h;
 140         } base;
 141
 142         struct {
 143                 u8 cpp;
 144         } ovly;
 145
 146         struct {
 147                 bool enable:1;
 148                 u8 bits:2;
 149                 u8 mode:4;
 150         } dither;
 151
 152         struct {
 153                 struct {
 154                         u16 cos:12;
 155                         u16 sin:12;
 156                 } sat;
 157         } procamp;
 158
 159         union {
 160                 struct {
 161                         bool core:1;
 162                         bool curs:1;
 163                 };
 164                 u8 mask;
 165         } clr;
 166
 167         union {
 168                 struct {
 169                         bool core:1;
 170                         bool curs:1;
 171                         bool view:1;
 172                         bool mode:1;
 173                         bool base:1;
 174                         bool ovly:1;
 175                         bool dither:1;
 176                         bool procamp:1;
 177                 };
 178                 u16 mask;
 179         } set;
 180 };
 181
 182 #define nv50_wndw_atom(p) container_of((p), struct nv50_wndw_atom, state)
 183
 184 struct nv50_wndw_atom {
 185         struct drm_plane_state state;
 186         u8 interval;
 187
 188         struct drm_rect clip;
 189
 190         struct {
 191                 u32  handle;
 192                 u16  offset:12;
 193                 bool awaken:1;
 194         } ntfy;
 195
 196         struct {
 197                 u32 handle;
 198                 u16 offset:12;
 199                 u32 acquire;
 200                 u32 release;
 201         } sema;
 202
 203         struct {
 204                 u8 enable:2;
 205         } lut;
 206
 207         struct {
 208                 u8  mode:2;
 209                 u8  interval:4;
 210
 211                 u8  format;
 212                 u8  kind:7;
 213                 u8  layout:1;
 214                 u8  block:4;
 215                 u32 pitch:20;
 216                 u16 w;
 217                 u16 h;
 218
 219                 u32 handle;
 220                 u64 offset;
 221         } image;
 222
 223         struct {
 224                 u16 x;
 225                 u16 y;
 226         } point;
 227
 228         union {
 229                 struct {
 230                         bool ntfy:1;
 231                         bool sema:1;
 232                         bool image:1;
 233                 };
 234                 u8 mask;
 235         } clr;
 236
 237         union {
 238                 struct {
 239                         bool ntfy:1;
 240                         bool sema:1;
 241                         bool image:1;
 242                         bool lut:1;
 243                         bool point:1;
 244                 };
 245                 u8 mask;
 246         } set;
 247 };
 248
 249 /******************************************************************************
 250  * EVO channel
 251  *****************************************************************************/
 252
 253 struct nv50_chan {
 254         struct nvif_object user;
 255         struct nvif_device *device;
 256 };
 257
 258 static int
 259 nv50_chan_create(struct nvif_device *device, struct nvif_object *disp,
 260                  const s32 *oclass, u8 head, void *data, u32 size,
 261                  struct nv50_chan *chan)
 262 {
 263         struct nvif_sclass *sclass;
 264         int ret, i, n;
 265
 266         chan->device = device;
 267
 268         ret = n = nvif_object_sclass_get(disp, &sclass);
 269         if (ret < 0)
 270                 return ret;
 271
 272         while (oclass[0]) {
 273                 for (i = 0; i < n; i++) {
 274                         if (sclass[i].oclass == oclass[0]) {
 275                                 ret = nvif_object_init(disp, 0, oclass[0],
 276                                                        data, size, &chan->user);
 277                                 if (ret == 0)
 278                                         nvif_object_map(&chan->user);
 279                                 nvif_object_sclass_put(&sclass);
 280                                 return ret;
 281                         }
 282                 }
 283                 oclass++;
 284         }
 285
 286         nvif_object_sclass_put(&sclass);
 287         return -ENOSYS;
 288 }
 289
 290 static void
 291 nv50_chan_destroy(struct nv50_chan *chan)
 292 {
 293         nvif_object_fini(&chan->user);
 294 }
 295
 296 /******************************************************************************
 297  * PIO EVO channel
 298  *****************************************************************************/
 299
 300 struct nv50_pioc {
 301         struct nv50_chan base;
 302 };
 303
 304 static void
 305 nv50_pioc_destroy(struct nv50_pioc *pioc)
 306 {
 307         nv50_chan_destroy(&pioc->base);
 308 }
 309
 310 static int
 311 nv50_pioc_create(struct nvif_device *device, struct nvif_object *disp,
 312                  const s32 *oclass, u8 head, void *data, u32 size,
 313                  struct nv50_pioc *pioc)
 314 {
 315         return nv50_chan_create(device, disp, oclass, head, data, size,
 316                                 &pioc->base);
 317 }
 318
 319 /******************************************************************************
 320  * Overlay Immediate
 321  *****************************************************************************/
 322
 323 struct nv50_oimm {
 324         struct nv50_pioc base;
 325 };
 326
 327 static int
 328 nv50_oimm_create(struct nvif_device *device, struct nvif_object *disp,
 329                  int head, struct nv50_oimm *oimm)
 330 {
 331         struct nv50_disp_cursor_v0 args = {
 332                 .head = head,
 333         };
 334         static const s32 oclass[] = {
 335                 GK104_DISP_OVERLAY,
 336                 GF110_DISP_OVERLAY,
 337                 GT214_DISP_OVERLAY,
 338                 G82_DISP_OVERLAY,
 339                 NV50_DISP_OVERLAY,
 340                 0
 341         };
 342
 343         return nv50_pioc_create(device, disp, oclass, head, &args, sizeof(args),
 344                                 &oimm->base);
 345 }
 346
 347 /******************************************************************************
 348  * DMA EVO channel
 349  *****************************************************************************/
 350
 351 struct nv50_dmac_ctxdma {
 352         struct list_head head;
 353         struct nvif_object object;
 354 };
 355
 356 struct nv50_dmac {
 357         struct nv50_chan base;
 358         dma_addr_t handle;
 359         u32 *ptr;
 360
 361         struct nvif_object sync;
 362         struct nvif_object vram;
 363         struct list_head ctxdma;
 364
 365         /* Protects against concurrent pushbuf access to this channel, lock is
 366          * grabbed by evo_wait (if the pushbuf reservation is successful) and
 367          * dropped again by evo_kick. */
 368         struct mutex lock;
 369 };
 370
 371 static void
 372 nv50_dmac_ctxdma_del(struct nv50_dmac_ctxdma *ctxdma)
 373 {
 374         nvif_object_fini(&ctxdma->object);
 375         list_del(&ctxdma->head);
 376         kfree(ctxdma);
 377 }
 378
 379 static struct nv50_dmac_ctxdma *
 380 nv50_dmac_ctxdma_new(struct nv50_dmac *dmac, u32 handle,
 381                      struct nouveau_framebuffer *fb)
 382 {
 383         struct nouveau_drm *drm = nouveau_drm(fb->base.dev);
 384         struct nv50_dmac_ctxdma *ctxdma;
 385         const u8  kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8;
 386         struct {
 387                 struct nv_dma_v0 base;
 388                 union {
 389                         struct nv50_dma_v0 nv50;
 390                         struct gf100_dma_v0 gf100;
 391                         struct gf119_dma_v0 gf119;
 392                 };
 393         } args = {};
 394         u32 argc = sizeof(args.base);
 395         int ret;
 396
 397         list_for_each_entry(ctxdma, &dmac->ctxdma, head) {
 398                 if (ctxdma->object.handle == handle)
 399                         return ctxdma;
 400         }
 401
 402         if (!(ctxdma = kzalloc(sizeof(*ctxdma), GFP_KERNEL)))
 403                 return ERR_PTR(-ENOMEM);
 404         list_add(&ctxdma->head, &dmac->ctxdma);
 405
 406         args.base.target = NV_DMA_V0_TARGET_VRAM;
 407         args.base.access = NV_DMA_V0_ACCESS_RDWR;
 408         args.base.start  = 0;
 409         args.base.limit  = drm->device.info.ram_user - 1;
 410
 411         if (drm->device.info.chipset < 0x80) {
 412                 args.nv50.part = NV50_DMA_V0_PART_256;
 413                 argc += sizeof(args.nv50);
 414         } else
 415         if (drm->device.info.chipset < 0xc0) {
 416                 args.nv50.part = NV50_DMA_V0_PART_256;
 417                 args.nv50.kind = kind;
 418                 argc += sizeof(args.nv50);
 419         } else
 420         if (drm->device.info.chipset < 0xd0) {
 421                 args.gf100.kind = kind;
 422                 argc += sizeof(args.gf100);
 423         } else {
 424                 args.gf119.page = GF119_DMA_V0_PAGE_LP;
 425                 args.gf119.kind = kind;
 426                 argc += sizeof(args.gf119);
 427         }
 428
 429         ret = nvif_object_init(&dmac->base.user, handle, NV_DMA_IN_MEMORY,
 430                                &args, argc, &ctxdma->object);
 431         if (ret) {
 432                 nv50_dmac_ctxdma_del(ctxdma);
 433                 return ERR_PTR(ret);
 434         }
 435
 436         return ctxdma;
 437 }
 438
 439 static void
 440 nv50_dmac_destroy(struct nv50_dmac *dmac, struct nvif_object *disp)
 441 {
 442         struct nvif_device *device = dmac->base.device;
 443         struct nv50_dmac_ctxdma *ctxdma, *ctxtmp;
 444
 445         list_for_each_entry_safe(ctxdma, ctxtmp, &dmac->ctxdma, head) {
 446                 nv50_dmac_ctxdma_del(ctxdma);
 447         }
 448
 449         nvif_object_fini(&dmac->vram);
 450         nvif_object_fini(&dmac->sync);
 451
 452         nv50_chan_destroy(&dmac->base);
 453
 454         if (dmac->ptr) {
 455                 struct device *dev = nvxx_device(device)->dev;
 456                 dma_free_coherent(dev, PAGE_SIZE, dmac->ptr, dmac->handle);
 457         }
 458 }
 459
 460 static int
 461 nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp,
 462                  const s32 *oclass, u8 head, void *data, u32 size, u64 syncbuf,
 463                  struct nv50_dmac *dmac)
 464 {
 465         struct nv50_disp_core_channel_dma_v0 *args = data;
 466         struct nvif_object pushbuf;
 467         int ret;
 468
 469         mutex_init(&dmac->lock);
 470
 471         dmac->ptr = dma_alloc_coherent(nvxx_device(device)->dev, PAGE_SIZE,
 472                                        &dmac->handle, GFP_KERNEL);
 473         if (!dmac->ptr)
 474                 return -ENOMEM;
 475
 476         ret = nvif_object_init(&device->object, 0, NV_DMA_FROM_MEMORY,
 477                                &(struct nv_dma_v0) {
 478                                         .target = NV_DMA_V0_TARGET_PCI_US,
 479                                         .access = NV_DMA_V0_ACCESS_RD,
 480                                         .start = dmac->handle + 0x0000,
 481                                         .limit = dmac->handle + 0x0fff,
 482                                }, sizeof(struct nv_dma_v0), &pushbuf);
 483         if (ret)
 484                 return ret;
 485
 486         args->pushbuf = nvif_handle(&pushbuf);
 487
 488         ret = nv50_chan_create(device, disp, oclass, head, data, size,
 489                                &dmac->base);
 490         nvif_object_fini(&pushbuf);
 491         if (ret)
 492                 return ret;
 493
 494         ret = nvif_object_init(&dmac->base.user, 0xf0000000, NV_DMA_IN_MEMORY,
 495                                &(struct nv_dma_v0) {
 496                                         .target = NV_DMA_V0_TARGET_VRAM,
 497                                         .access = NV_DMA_V0_ACCESS_RDWR,
 498                                         .start = syncbuf + 0x0000,
 499                                         .limit = syncbuf + 0x0fff,
 500                                }, sizeof(struct nv_dma_v0),
 501                                &dmac->sync);
 502         if (ret)
 503                 return ret;
 504
 505         ret = nvif_object_init(&dmac->base.user, 0xf0000001, NV_DMA_IN_MEMORY,
 506                                &(struct nv_dma_v0) {
 507                                         .target = NV_DMA_V0_TARGET_VRAM,
 508                                         .access = NV_DMA_V0_ACCESS_RDWR,
 509                                         .start = 0,
 510                                         .limit = device->info.ram_user - 1,
 511                                }, sizeof(struct nv_dma_v0),
 512                                &dmac->vram);
 513         if (ret)
 514                 return ret;
 515
 516         INIT_LIST_HEAD(&dmac->ctxdma);
 517         return ret;
 518 }
 519
 520 /******************************************************************************
 521  * Core
 522  *****************************************************************************/
 523
 524 struct nv50_mast {
 525         struct nv50_dmac base;
 526 };
 527
 528 static int
 529 nv50_core_create(struct nvif_device *device, struct nvif_object *disp,
 530                  u64 syncbuf, struct nv50_mast *core)
 531 {
 532         struct nv50_disp_core_channel_dma_v0 args = {
 533                 .pushbuf = 0xb0007d00,
 534         };
 535         static const s32 oclass[] = {
 536                 GP104_DISP_CORE_CHANNEL_DMA,
 537                 GP100_DISP_CORE_CHANNEL_DMA,
 538                 GM200_DISP_CORE_CHANNEL_DMA,
 539                 GM107_DISP_CORE_CHANNEL_DMA,
 540                 GK110_DISP_CORE_CHANNEL_DMA,
 541                 GK104_DISP_CORE_CHANNEL_DMA,
 542                 GF110_DISP_CORE_CHANNEL_DMA,
 543                 GT214_DISP_CORE_CHANNEL_DMA,
 544                 GT206_DISP_CORE_CHANNEL_DMA,
 545                 GT200_DISP_CORE_CHANNEL_DMA,
 546                 G82_DISP_CORE_CHANNEL_DMA,
 547                 NV50_DISP_CORE_CHANNEL_DMA,
 548                 0
 549         };
 550
 551         return nv50_dmac_create(device, disp, oclass, 0, &args, sizeof(args),
 552                                 syncbuf, &core->base);
 553 }
 554
 555 /******************************************************************************
 556  * Base
 557  *****************************************************************************/
 558
 559 struct nv50_sync {
 560         struct nv50_dmac base;
 561         u32 addr;
 562         u32 data;
 563 };
 564
 565 static int
 566 nv50_base_create(struct nvif_device *device, struct nvif_object *disp,
 567                  int head, u64 syncbuf, struct nv50_sync *base)
 568 {
 569         struct nv50_disp_base_channel_dma_v0 args = {
 570                 .pushbuf = 0xb0007c00 | head,
 571                 .head = head,
 572         };
 573         static const s32 oclass[] = {
 574                 GK110_DISP_BASE_CHANNEL_DMA,
 575                 GK104_DISP_BASE_CHANNEL_DMA,
 576                 GF110_DISP_BASE_CHANNEL_DMA,
 577                 GT214_DISP_BASE_CHANNEL_DMA,
 578                 GT200_DISP_BASE_CHANNEL_DMA,
 579                 G82_DISP_BASE_CHANNEL_DMA,
 580                 NV50_DISP_BASE_CHANNEL_DMA,
 581                 0
 582         };
 583
 584         return nv50_dmac_create(device, disp, oclass, head, &args, sizeof(args),
 585                                 syncbuf, &base->base);
 586 }
 587
 588 /******************************************************************************
 589  * Overlay
 590  *****************************************************************************/
 591
 592 struct nv50_ovly {
 593         struct nv50_dmac base;
 594 };
 595
 596 static int
 597 nv50_ovly_create(struct nvif_device *device, struct nvif_object *disp,
 598                  int head, u64 syncbuf, struct nv50_ovly *ovly)
 599 {
 600         struct nv50_disp_overlay_channel_dma_v0 args = {
 601                 .pushbuf = 0xb0007e00 | head,
 602                 .head = head,
 603         };
 604         static const s32 oclass[] = {
 605                 GK104_DISP_OVERLAY_CONTROL_DMA,
 606                 GF110_DISP_OVERLAY_CONTROL_DMA,
 607                 GT214_DISP_OVERLAY_CHANNEL_DMA,
 608                 GT200_DISP_OVERLAY_CHANNEL_DMA,
 609                 G82_DISP_OVERLAY_CHANNEL_DMA,
 610                 NV50_DISP_OVERLAY_CHANNEL_DMA,
 611                 0
 612         };
 613
 614         return nv50_dmac_create(device, disp, oclass, head, &args, sizeof(args),
 615                                 syncbuf, &ovly->base);
 616 }
 617
 618 struct nv50_head {
 619         struct nouveau_crtc base;
 620         struct nouveau_bo *image;
 621         struct nv50_ovly ovly;
 622         struct nv50_oimm oimm;
 623
 624         struct nv50_head_atom arm;
 625         struct nv50_head_atom asy;
 626
 627         struct nv50_base *_base;
 628         struct nv50_curs *_curs;
 629 };
 630
 631 #define nv50_head(c) ((struct nv50_head *)nouveau_crtc(c))
 632 #define nv50_ovly(c) (&nv50_head(c)->ovly)
 633 #define nv50_oimm(c) (&nv50_head(c)->oimm)
 634 #define nv50_chan(c) (&(c)->base.base)
 635 #define nv50_vers(c) nv50_chan(c)->user.oclass
 636
 637 struct nv50_disp {
 638         struct nvif_object *disp;
 639         struct nv50_mast mast;
 640
 641         struct nouveau_bo *sync;
 642 };
 643
 644 static struct nv50_disp *
 645 nv50_disp(struct drm_device *dev)
 646 {
 647         return nouveau_display(dev)->priv;
 648 }
 649
 650 #define nv50_mast(d) (&nv50_disp(d)->mast)
 651
 652 static struct drm_crtc *
 653 nv50_display_crtc_get(struct drm_encoder *encoder)
 654 {
 655         return nouveau_encoder(encoder)->crtc;
 656 }
 657
 658 /******************************************************************************
 659  * EVO channel helpers
 660  *****************************************************************************/
 661 static u32 *
 662 evo_wait(void *evoc, int nr)
 663 {
 664         struct nv50_dmac *dmac = evoc;
 665         struct nvif_device *device = dmac->base.device;
 666         u32 put = nvif_rd32(&dmac->base.user, 0x0000) / 4;
 667
 668         mutex_lock(&dmac->lock);
 669         if (put + nr >= (PAGE_SIZE / 4) - 8) {
 670                 dmac->ptr[put] = 0x20000000;
 671
 672                 nvif_wr32(&dmac->base.user, 0x0000, 0x00000000);
 673                 if (nvif_msec(device, 2000,
 674                         if (!nvif_rd32(&dmac->base.user, 0x0004))
 675                                 break;
 676                 ) < 0) {
 677                         mutex_unlock(&dmac->lock);
 678                         printk(KERN_ERR "nouveau: evo channel stalled\n");
 679                         return NULL;
 680                 }
 681
 682                 put = 0;
 683         }
 684
 685         return dmac->ptr + put;
 686 }
 687
 688 static void
 689 evo_kick(u32 *push, void *evoc)
 690 {
 691         struct nv50_dmac *dmac = evoc;
 692         nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2);
 693         mutex_unlock(&dmac->lock);
 694 }
 695
 696 #define evo_mthd(p,m,s) do {                                                   \
 697         const u32 _m = (m), _s = (s);                                          \
 698         if (drm_debug & DRM_UT_KMS)                                            \
 699                 printk(KERN_ERR "%04x %d %s\n", _m, _s, __func__);             \
 700         *((p)++) = ((_s << 18) | _m);                                          \
 701 } while(0)
 702
 703 #define evo_data(p,d) do {                                                     \
 704         const u32 _d = (d);                                                    \
 705         if (drm_debug & DRM_UT_KMS)                                            \
 706                 printk(KERN_ERR "\t%08x\n", _d);                               \
 707         *((p)++) = _d;                                                         \
 708 } while(0)
 709
 710 static bool
 711 evo_sync_wait(void *data)
 712 {
 713         if (nouveau_bo_rd32(data, EVO_MAST_NTFY) != 0x00000000)
 714                 return true;
 715         usleep_range(1, 2);
 716         return false;
 717 }
 718
 719 static int
 720 evo_sync(struct drm_device *dev)
 721 {
 722         struct nvif_device *device = &nouveau_drm(dev)->device;
 723         struct nv50_disp *disp = nv50_disp(dev);
 724         struct nv50_mast *mast = nv50_mast(dev);
 725         u32 *push = evo_wait(mast, 8);
 726         if (push) {
 727                 nouveau_bo_wr32(disp->sync, EVO_MAST_NTFY, 0x00000000);
 728                 evo_mthd(push, 0x0084, 1);
 729                 evo_data(push, 0x80000000 | EVO_MAST_NTFY);
 730                 evo_mthd(push, 0x0080, 2);
 731                 evo_data(push, 0x00000000);
 732                 evo_data(push, 0x00000000);
 733                 evo_kick(push, mast);
 734                 if (nvif_msec(device, 2000,
 735                         if (evo_sync_wait(disp->sync))
 736                                 break;
 737                 ) >= 0)
 738                         return 0;
 739         }
 740
 741         return -EBUSY;
 742 }
 743
 744 /******************************************************************************
 745  * Plane
 746  *****************************************************************************/
 747 #define nv50_wndw(p) container_of((p), struct nv50_wndw, plane)
 748
 749 struct nv50_wndw {
 750         const struct nv50_wndw_func *func;
 751         struct nv50_dmac *dmac;
 752
 753         struct drm_plane plane;
 754
 755         struct nvif_notify notify;
 756         u16 ntfy;
 757         u16 sema;
 758         u32 data;
 759
 760         struct nv50_wndw_atom arm;
 761         struct nv50_wndw_atom asy;
 762 };
 763
 764 struct nv50_wndw_func {
 765         void *(*dtor)(struct nv50_wndw *);
 766         int (*acquire)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
 767                        struct nv50_head_atom *asyh);
 768         void (*release)(struct nv50_wndw *, struct nv50_wndw_atom *asyw,
 769                         struct nv50_head_atom *asyh);
 770         void (*prepare)(struct nv50_wndw *, struct nv50_head_atom *asyh,
 771                         struct nv50_wndw_atom *asyw);
 772
 773         void (*sema_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
 774         void (*sema_clr)(struct nv50_wndw *);
 775         void (*ntfy_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
 776         void (*ntfy_clr)(struct nv50_wndw *);
 777         int (*ntfy_wait_begun)(struct nv50_wndw *, struct nv50_wndw_atom *);
 778         void (*image_set)(struct nv50_wndw *, struct nv50_wndw_atom *);
 779         void (*image_clr)(struct nv50_wndw *);
 780         void (*lut)(struct nv50_wndw *, struct nv50_wndw_atom *);
 781         void (*point)(struct nv50_wndw *, struct nv50_wndw_atom *);
 782
 783         u32 (*update)(struct nv50_wndw *, u32 interlock);
 784 };
 785
 786 static int
 787 nv50_wndw_wait_armed(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
 788 {
 789         if (asyw->set.ntfy)
 790                 return wndw->func->ntfy_wait_begun(wndw, asyw);
 791         return 0;
 792 }
 793
 794 static u32
 795 nv50_wndw_flush_clr(struct nv50_wndw *wndw, u32 interlock, bool flush,
 796                     struct nv50_wndw_atom *asyw)
 797 {
 798         if (asyw->clr.sema && (!asyw->set.sema || flush))
 799                 wndw->func->sema_clr(wndw);
 800         if (asyw->clr.ntfy && (!asyw->set.ntfy || flush))
 801                 wndw->func->ntfy_clr(wndw);
 802         if (asyw->clr.image && (!asyw->set.image || flush))
 803                 wndw->func->image_clr(wndw);
 804
 805         return flush ? wndw->func->update(wndw, interlock) : 0;
 806 }
 807
 808 static u32
 809 nv50_wndw_flush_set(struct nv50_wndw *wndw, u32 interlock,
 810                     struct nv50_wndw_atom *asyw)
 811 {
 812         if (interlock) {
 813                 asyw->image.mode = 0;
 814                 asyw->image.interval = 1;
 815         }
 816
 817         if (asyw->set.sema ) wndw->func->sema_set (wndw, asyw);
 818         if (asyw->set.ntfy ) wndw->func->ntfy_set (wndw, asyw);
 819         if (asyw->set.image) wndw->func->image_set(wndw, asyw);
 820         if (asyw->set.lut  ) wndw->func->lut      (wndw, asyw);
 821         if (asyw->set.point) wndw->func->point    (wndw, asyw);
 822
 823         return wndw->func->update(wndw, interlock);
 824 }
 825
 826 static void
 827 nv50_wndw_atomic_check_release(struct nv50_wndw *wndw,
 828                                struct nv50_wndw_atom *asyw,
 829                                struct nv50_head_atom *asyh)
 830 {
 831         struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
 832         NV_ATOMIC(drm, "%s release\n", wndw->plane.name);
 833         wndw->func->release(wndw, asyw, asyh);
 834         asyw->ntfy.handle = 0;
 835         asyw->sema.handle = 0;
 836 }
 837
 838 static int
 839 nv50_wndw_atomic_check_acquire(struct nv50_wndw *wndw,
 840                                struct nv50_wndw_atom *asyw,
 841                                struct nv50_head_atom *asyh)
 842 {
 843         struct nouveau_framebuffer *fb = nouveau_framebuffer(asyw->state.fb);
 844         struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
 845         int ret;
 846
 847         NV_ATOMIC(drm, "%s acquire\n", wndw->plane.name);
 848         asyw->clip.x1 = 0;
 849         asyw->clip.y1 = 0;
 850         asyw->clip.x2 = asyh->state.mode.hdisplay;
 851         asyw->clip.y2 = asyh->state.mode.vdisplay;
 852
 853         asyw->image.w = fb->base.width;
 854         asyw->image.h = fb->base.height;
 855         asyw->image.kind = (fb->nvbo->tile_flags & 0x0000ff00) >> 8;
 856         if (asyw->image.kind) {
 857                 asyw->image.layout = 0;
 858                 if (drm->device.info.chipset >= 0xc0)
 859                         asyw->image.block = fb->nvbo->tile_mode >> 4;
 860                 else
 861                         asyw->image.block = fb->nvbo->tile_mode;
 862                 asyw->image.pitch = (fb->base.pitches[0] / 4) << 4;
 863         } else {
 864                 asyw->image.layout = 1;
 865                 asyw->image.block  = 0;
 866                 asyw->image.pitch  = fb->base.pitches[0];
 867         }
 868
 869         ret = wndw->func->acquire(wndw, asyw, asyh);
 870         if (ret)
 871                 return ret;
 872
 873         if (asyw->set.image) {
 874                 if (!(asyw->image.mode = asyw->interval ? 0 : 1))
 875                         asyw->image.interval = asyw->interval;
 876                 else
 877                         asyw->image.interval = 0;
 878         }
 879
 880         return 0;
 881 }
 882
 883 static int
 884 nv50_wndw_atomic_check(struct drm_plane *plane, struct drm_plane_state *state)
 885 {
 886         struct nouveau_drm *drm = nouveau_drm(plane->dev);
 887         struct nv50_wndw *wndw = nv50_wndw(plane);
 888         struct nv50_wndw_atom *armw = &wndw->arm;
 889         struct nv50_wndw_atom *asyw = &wndw->asy;
 890         struct nv50_head_atom *harm = NULL, *asyh = NULL;
 891         bool varm = false, asyv = false, asym = false;
 892         int ret;
 893
 894         asyw->clr.mask = 0;
 895         asyw->set.mask = 0;
 896
 897         NV_ATOMIC(drm, "%s atomic_check\n", plane->name);
 898         if (asyw->state.crtc) {
 899                 asyh = &nv50_head(asyw->state.crtc)->asy;
 900                 if (IS_ERR(asyh))
 901                         return PTR_ERR(asyh);
 902                 asym = drm_atomic_crtc_needs_modeset(&asyh->state);
 903                 asyv = asyh->state.active;
 904         }
 905
 906         if (armw->state.crtc) {
 907                 harm = &nv50_head(armw->state.crtc)->asy;
 908                 if (IS_ERR(harm))
 909                         return PTR_ERR(harm);
 910                 varm = nv50_head(harm->state.crtc)->arm.state.active;
 911         }
 912
 913         if (asyv) {
 914                 asyw->point.x = asyw->state.crtc_x;
 915                 asyw->point.y = asyw->state.crtc_y;
 916                 if (memcmp(&armw->point, &asyw->point, sizeof(asyw->point)))
 917                         asyw->set.point = true;
 918
 919                 if (!varm || asym || armw->state.fb != asyw->state.fb) {
 920                         ret = nv50_wndw_atomic_check_acquire(wndw, asyw, asyh);
 921                         if (ret)
 922                                 return ret;
 923                 }
 924         } else
 925         if (varm) {
 926                 nv50_wndw_atomic_check_release(wndw, asyw, harm);
 927         } else {
 928                 return 0;
 929         }
 930
 931         if (!asyv || asym) {
 932                 asyw->clr.ntfy = armw->ntfy.handle != 0;
 933                 asyw->clr.sema = armw->sema.handle != 0;
 934                 if (wndw->func->image_clr)
 935                         asyw->clr.image = armw->image.handle != 0;
 936                 asyw->set.lut = wndw->func->lut && asyv;
 937         }
 938
 939         memcpy(armw, asyw, sizeof(*asyw));
 940         return 0;
 941 }
 942
 943 static void
 944 nv50_wndw_atomic_destroy_state(struct drm_plane *plane,
 945                                struct drm_plane_state *state)
 946 {
 947         struct nv50_wndw_atom *asyw = nv50_wndw_atom(state);
 948         __drm_atomic_helper_plane_destroy_state(&asyw->state);
 949         dma_fence_put(asyw->state.fence);
 950         kfree(asyw);
 951 }
 952
 953 static struct drm_plane_state *
 954 nv50_wndw_atomic_duplicate_state(struct drm_plane *plane)
 955 {
 956         struct nv50_wndw_atom *armw = nv50_wndw_atom(plane->state);
 957         struct nv50_wndw_atom *asyw;
 958         if (!(asyw = kmalloc(sizeof(*asyw), GFP_KERNEL)))
 959                 return NULL;
 960         __drm_atomic_helper_plane_duplicate_state(plane, &asyw->state);
 961         asyw->state.fence = NULL;
 962         asyw->interval = 1;
 963         asyw->sema = armw->sema;
 964         asyw->ntfy = armw->ntfy;
 965         asyw->image = armw->image;
 966         asyw->point = armw->point;
 967         asyw->lut = armw->lut;
 968         asyw->clr.mask = 0;
 969         asyw->set.mask = 0;
 970         return &asyw->state;
 971 }
 972
 973 static void
 974 nv50_wndw_reset(struct drm_plane *plane)
 975 {
 976         struct nv50_wndw_atom *asyw;
 977
 978         if (WARN_ON(!(asyw = kzalloc(sizeof(*asyw), GFP_KERNEL))))
 979                 return;
 980
 981         if (plane->state)
 982                 plane->funcs->atomic_destroy_state(plane, plane->state);
 983         plane->state = &asyw->state;
 984         plane->state->plane = plane;
 985         plane->state->rotation = DRM_ROTATE_0;
 986 }
 987
 988 static void
 989 nv50_wndw_destroy(struct drm_plane *plane)
 990 {
 991         struct nv50_wndw *wndw = nv50_wndw(plane);
 992         void *data;
 993         nvif_notify_fini(&wndw->notify);
 994         data = wndw->func->dtor(wndw);
 995         drm_plane_cleanup(&wndw->plane);
 996         kfree(data);
 997 }
 998
 999 static const struct drm_plane_funcs
1000 nv50_wndw = {
1001         .destroy = nv50_wndw_destroy,
1002         .reset = nv50_wndw_reset,
1003         .set_property = drm_atomic_helper_plane_set_property,
1004         .atomic_duplicate_state = nv50_wndw_atomic_duplicate_state,
1005         .atomic_destroy_state = nv50_wndw_atomic_destroy_state,
1006 };
1007
1008 static void
1009 nv50_wndw_fini(struct nv50_wndw *wndw)
1010 {
1011         nvif_notify_put(&wndw->notify);
1012 }
1013
1014 static void
1015 nv50_wndw_init(struct nv50_wndw *wndw)
1016 {
1017         nvif_notify_get(&wndw->notify);
1018 }
1019
1020 static int
1021 nv50_wndw_ctor(const struct nv50_wndw_func *func, struct drm_device *dev,
1022                enum drm_plane_type type, const char *name, int index,
1023                struct nv50_dmac *dmac, const u32 *format, int nformat,
1024                struct nv50_wndw *wndw)
1025 {
1026         int ret;
1027
1028         wndw->func = func;
1029         wndw->dmac = dmac;
1030
1031         ret = drm_universal_plane_init(dev, &wndw->plane, 0, &nv50_wndw, format,
1032                                        nformat, type, "%s-%d", name, index);
1033         if (ret)
1034                 return ret;
1035
1036         return 0;
1037 }
1038
1039 /******************************************************************************
1040  * Cursor plane
1041  *****************************************************************************/
1042 #define nv50_curs(p) container_of((p), struct nv50_curs, wndw)
1043
1044 struct nv50_curs {
1045         struct nv50_wndw wndw;
1046         struct nvif_object chan;
1047 };
1048
1049 static u32
1050 nv50_curs_update(struct nv50_wndw *wndw, u32 interlock)
1051 {
1052         struct nv50_curs *curs = nv50_curs(wndw);
1053         nvif_wr32(&curs->chan, 0x0080, 0x00000000);
1054         return 0;
1055 }
1056
1057 static void
1058 nv50_curs_point(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1059 {
1060         struct nv50_curs *curs = nv50_curs(wndw);
1061         nvif_wr32(&curs->chan, 0x0084, (asyw->point.y << 16) | asyw->point.x);
1062 }
1063
1064 static void
1065 nv50_curs_prepare(struct nv50_wndw *wndw, struct nv50_head_atom *asyh,
1066                   struct nv50_wndw_atom *asyw)
1067 {
1068         asyh->curs.handle = nv50_disp(wndw->plane.dev)->mast.base.vram.handle;
1069         asyh->curs.offset = asyw->image.offset;
1070         asyh->set.curs = asyh->curs.visible;
1071 }
1072
1073 static void
1074 nv50_curs_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
1075                   struct nv50_head_atom *asyh)
1076 {
1077         asyh->curs.visible = false;
1078 }
1079
1080 static int
1081 nv50_curs_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
1082                   struct nv50_head_atom *asyh)
1083 {
1084         int ret;
1085
1086         ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
1087                                            DRM_PLANE_HELPER_NO_SCALING,
1088                                            DRM_PLANE_HELPER_NO_SCALING,
1089                                            true, true);
1090         asyh->curs.visible = asyw->state.visible;
1091         if (ret || !asyh->curs.visible)
1092                 return ret;
1093
1094         switch (asyw->state.fb->width) {
1095         case 32: asyh->curs.layout = 0; break;
1096         case 64: asyh->curs.layout = 1; break;
1097         default:
1098                 return -EINVAL;
1099         }
1100
1101         if (asyw->state.fb->width != asyw->state.fb->height)
1102                 return -EINVAL;
1103
1104         switch (asyw->state.fb->pixel_format) {
1105         case DRM_FORMAT_ARGB8888: asyh->curs.format = 1; break;
1106         default:
1107                 WARN_ON(1);
1108                 return -EINVAL;
1109         }
1110
1111         return 0;
1112 }
1113
1114 static void *
1115 nv50_curs_dtor(struct nv50_wndw *wndw)
1116 {
1117         struct nv50_curs *curs = nv50_curs(wndw);
1118         nvif_object_fini(&curs->chan);
1119         return curs;
1120 }
1121
1122 static const u32
1123 nv50_curs_format[] = {
1124         DRM_FORMAT_ARGB8888,
1125 };
1126
1127 static const struct nv50_wndw_func
1128 nv50_curs = {
1129         .dtor = nv50_curs_dtor,
1130         .acquire = nv50_curs_acquire,
1131         .release = nv50_curs_release,
1132         .prepare = nv50_curs_prepare,
1133         .point = nv50_curs_point,
1134         .update = nv50_curs_update,
1135 };
1136
1137 static int
1138 nv50_curs_new(struct nouveau_drm *drm, struct nv50_head *head,
1139               struct nv50_curs **pcurs)
1140 {
1141         static const struct nvif_mclass curses[] = {
1142                 { GK104_DISP_CURSOR, 0 },
1143                 { GF110_DISP_CURSOR, 0 },
1144                 { GT214_DISP_CURSOR, 0 },
1145                 {   G82_DISP_CURSOR, 0 },
1146                 {  NV50_DISP_CURSOR, 0 },
1147                 {}
1148         };
1149         struct nv50_disp_cursor_v0 args = {
1150                 .head = head->base.index,
1151         };
1152         struct nv50_disp *disp = nv50_disp(drm->dev);
1153         struct nv50_curs *curs;
1154         int cid, ret;
1155
1156         cid = nvif_mclass(disp->disp, curses);
1157         if (cid < 0) {
1158                 NV_ERROR(drm, "No supported cursor immediate class\n");
1159                 return cid;
1160         }
1161
1162         if (!(curs = *pcurs = kzalloc(sizeof(*curs), GFP_KERNEL)))
1163                 return -ENOMEM;
1164
1165         ret = nv50_wndw_ctor(&nv50_curs, drm->dev, DRM_PLANE_TYPE_CURSOR,
1166                              "curs", head->base.index, &disp->mast.base,
1167                              nv50_curs_format, ARRAY_SIZE(nv50_curs_format),
1168                              &curs->wndw);
1169         if (ret) {
1170                 kfree(curs);
1171                 return ret;
1172         }
1173
1174         ret = nvif_object_init(disp->disp, 0, curses[cid].oclass, &args,
1175                                sizeof(args), &curs->chan);
1176         if (ret) {
1177                 NV_ERROR(drm, "curs%04x allocation failed: %d\n",
1178                          curses[cid].oclass, ret);
1179                 return ret;
1180         }
1181
1182         return 0;
1183 }
1184
1185 /******************************************************************************
1186  * Primary plane
1187  *****************************************************************************/
1188 #define nv50_base(p) container_of((p), struct nv50_base, wndw)
1189
1190 struct nv50_base {
1191         struct nv50_wndw wndw;
1192         struct nv50_sync chan;
1193         int id;
1194 };
1195
1196 static int
1197 nv50_base_notify(struct nvif_notify *notify)
1198 {
1199         return NVIF_NOTIFY_KEEP;
1200 }
1201
1202 static void
1203 nv50_base_lut(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1204 {
1205         struct nv50_base *base = nv50_base(wndw);
1206         u32 *push;
1207         if ((push = evo_wait(&base->chan, 2))) {
1208                 evo_mthd(push, 0x00e0, 1);
1209                 evo_data(push, asyw->lut.enable << 30);
1210                 evo_kick(push, &base->chan);
1211         }
1212 }
1213
1214 static void
1215 nv50_base_image_clr(struct nv50_wndw *wndw)
1216 {
1217         struct nv50_base *base = nv50_base(wndw);
1218         u32 *push;
1219         if ((push = evo_wait(&base->chan, 4))) {
1220                 evo_mthd(push, 0x0084, 1);
1221                 evo_data(push, 0x00000000);
1222                 evo_mthd(push, 0x00c0, 1);
1223                 evo_data(push, 0x00000000);
1224                 evo_kick(push, &base->chan);
1225         }
1226 }
1227
1228 static void
1229 nv50_base_image_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1230 {
1231         struct nv50_base *base = nv50_base(wndw);
1232         const s32 oclass = base->chan.base.base.user.oclass;
1233         u32 *push;
1234         if ((push = evo_wait(&base->chan, 10))) {
1235                 evo_mthd(push, 0x0084, 1);
1236                 evo_data(push, (asyw->image.mode << 8) |
1237                                (asyw->image.interval << 4));
1238                 evo_mthd(push, 0x00c0, 1);
1239                 evo_data(push, asyw->image.handle);
1240                 if (oclass < G82_DISP_BASE_CHANNEL_DMA) {
1241                         evo_mthd(push, 0x0800, 5);
1242                         evo_data(push, asyw->image.offset >> 8);
1243                         evo_data(push, 0x00000000);
1244                         evo_data(push, (asyw->image.h << 16) | asyw->image.w);
1245                         evo_data(push, (asyw->image.layout << 20) |
1246                                         asyw->image.pitch |
1247                                         asyw->image.block);
1248                         evo_data(push, (asyw->image.kind << 16) |
1249                                        (asyw->image.format << 8));
1250                 } else
1251                 if (oclass < GF110_DISP_BASE_CHANNEL_DMA) {
1252                         evo_mthd(push, 0x0800, 5);
1253                         evo_data(push, asyw->image.offset >> 8);
1254                         evo_data(push, 0x00000000);
1255                         evo_data(push, (asyw->image.h << 16) | asyw->image.w);
1256                         evo_data(push, (asyw->image.layout << 20) |
1257                                         asyw->image.pitch |
1258                                         asyw->image.block);
1259                         evo_data(push, asyw->image.format << 8);
1260                 } else {
1261                         evo_mthd(push, 0x0400, 5);
1262                         evo_data(push, asyw->image.offset >> 8);
1263                         evo_data(push, 0x00000000);
1264                         evo_data(push, (asyw->image.h << 16) | asyw->image.w);
1265                         evo_data(push, (asyw->image.layout << 24) |
1266                                         asyw->image.pitch |
1267                                         asyw->image.block);
1268                         evo_data(push, asyw->image.format << 8);
1269                 }
1270                 evo_kick(push, &base->chan);
1271         }
1272 }
1273
1274 static void
1275 nv50_base_ntfy_clr(struct nv50_wndw *wndw)
1276 {
1277         struct nv50_base *base = nv50_base(wndw);
1278         u32 *push;
1279         if ((push = evo_wait(&base->chan, 2))) {
1280                 evo_mthd(push, 0x00a4, 1);
1281                 evo_data(push, 0x00000000);
1282                 evo_kick(push, &base->chan);
1283         }
1284 }
1285
1286 static void
1287 nv50_base_ntfy_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1288 {
1289         struct nv50_base *base = nv50_base(wndw);
1290         u32 *push;
1291         if ((push = evo_wait(&base->chan, 3))) {
1292                 evo_mthd(push, 0x00a0, 2);
1293                 evo_data(push, (asyw->ntfy.awaken << 30) | asyw->ntfy.offset);
1294                 evo_data(push, asyw->ntfy.handle);
1295                 evo_kick(push, &base->chan);
1296         }
1297 }
1298
1299 static void
1300 nv50_base_sema_clr(struct nv50_wndw *wndw)
1301 {
1302         struct nv50_base *base = nv50_base(wndw);
1303         u32 *push;
1304         if ((push = evo_wait(&base->chan, 2))) {
1305                 evo_mthd(push, 0x0094, 1);
1306                 evo_data(push, 0x00000000);
1307                 evo_kick(push, &base->chan);
1308         }
1309 }
1310
1311 static void
1312 nv50_base_sema_set(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1313 {
1314         struct nv50_base *base = nv50_base(wndw);
1315         u32 *push;
1316         if ((push = evo_wait(&base->chan, 5))) {
1317                 evo_mthd(push, 0x0088, 4);
1318                 evo_data(push, asyw->sema.offset);
1319                 evo_data(push, asyw->sema.acquire);
1320                 evo_data(push, asyw->sema.release);
1321                 evo_data(push, asyw->sema.handle);
1322                 evo_kick(push, &base->chan);
1323         }
1324 }
1325
1326 static u32
1327 nv50_base_update(struct nv50_wndw *wndw, u32 interlock)
1328 {
1329         struct nv50_base *base = nv50_base(wndw);
1330         u32 *push;
1331
1332         if (!(push = evo_wait(&base->chan, 2)))
1333                 return 0;
1334         evo_mthd(push, 0x0080, 1);
1335         evo_data(push, interlock);
1336         evo_kick(push, &base->chan);
1337
1338         if (base->chan.base.base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA)
1339                 return interlock ? 2 << (base->id * 8) : 0;
1340         return interlock ? 2 << (base->id * 4) : 0;
1341 }
1342
1343 static int
1344 nv50_base_ntfy_wait_begun(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw)
1345 {
1346         struct nouveau_drm *drm = nouveau_drm(wndw->plane.dev);
1347         struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
1348         if (nvif_msec(&drm->device, 2000ULL,
1349                 u32 data = nouveau_bo_rd32(disp->sync, asyw->ntfy.offset / 4);
1350                 if ((data & 0xc0000000) == 0x40000000)
1351                         break;
1352                 usleep_range(1, 2);
1353         ) < 0)
1354                 return -ETIMEDOUT;
1355         return 0;
1356 }
1357
1358 static void
1359 nv50_base_release(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
1360                   struct nv50_head_atom *asyh)
1361 {
1362         asyh->base.cpp = 0;
1363 }
1364
1365 static int
1366 nv50_base_acquire(struct nv50_wndw *wndw, struct nv50_wndw_atom *asyw,
1367                   struct nv50_head_atom *asyh)
1368 {
1369         const u32 format = asyw->state.fb->pixel_format;
1370         const struct drm_format_info *info;
1371         int ret;
1372
1373         info = drm_format_info(format);
1374         if (!info || !info->depth)
1375                 return -EINVAL;
1376
1377         ret = drm_plane_helper_check_state(&asyw->state, &asyw->clip,
1378                                            DRM_PLANE_HELPER_NO_SCALING,
1379                                            DRM_PLANE_HELPER_NO_SCALING,
1380                                            false, true);
1381         if (ret)
1382                 return ret;
1383
1384         asyh->base.depth = info->depth;
1385         asyh->base.cpp = info->cpp[0];
1386         asyh->base.x = asyw->state.src.x1 >> 16;
1387         asyh->base.y = asyw->state.src.y1 >> 16;
1388         asyh->base.w = asyw->state.fb->width;
1389         asyh->base.h = asyw->state.fb->height;
1390
1391         switch (format) {
1392         case DRM_FORMAT_C8         : asyw->image.format = 0x1e; break;
1393         case DRM_FORMAT_RGB565     : asyw->image.format = 0xe8; break;
1394         case DRM_FORMAT_XRGB1555   :
1395         case DRM_FORMAT_ARGB1555   : asyw->image.format = 0xe9; break;
1396         case DRM_FORMAT_XRGB8888   :
1397         case DRM_FORMAT_ARGB8888   : asyw->image.format = 0xcf; break;
1398         case DRM_FORMAT_XBGR2101010:
1399         case DRM_FORMAT_ABGR2101010: asyw->image.format = 0xd1; break;
1400         case DRM_FORMAT_XBGR8888   :
1401         case DRM_FORMAT_ABGR8888   : asyw->image.format = 0xd5; break;
1402         default:
1403                 WARN_ON(1);
1404                 return -EINVAL;
1405         }
1406
1407         asyw->lut.enable = 1;
1408         asyw->set.image = true;
1409         return 0;
1410 }
1411
1412 static void *
1413 nv50_base_dtor(struct nv50_wndw *wndw)
1414 {
1415         struct nv50_disp *disp = nv50_disp(wndw->plane.dev);
1416         struct nv50_base *base = nv50_base(wndw);
1417         nv50_dmac_destroy(&base->chan.base, disp->disp);
1418         return base;
1419 }
1420
1421 static const u32
1422 nv50_base_format[] = {
1423         DRM_FORMAT_C8,
1424         DRM_FORMAT_RGB565,
1425         DRM_FORMAT_XRGB1555,
1426         DRM_FORMAT_ARGB1555,
1427         DRM_FORMAT_XRGB8888,
1428         DRM_FORMAT_ARGB8888,
1429         DRM_FORMAT_XBGR2101010,
1430         DRM_FORMAT_ABGR2101010,
1431         DRM_FORMAT_XBGR8888,
1432         DRM_FORMAT_ABGR8888,
1433 };
1434
1435 static const struct nv50_wndw_func
1436 nv50_base = {
1437         .dtor = nv50_base_dtor,
1438         .acquire = nv50_base_acquire,
1439         .release = nv50_base_release,
1440         .sema_set = nv50_base_sema_set,
1441         .sema_clr = nv50_base_sema_clr,
1442         .ntfy_set = nv50_base_ntfy_set,
1443         .ntfy_clr = nv50_base_ntfy_clr,
1444         .ntfy_wait_begun = nv50_base_ntfy_wait_begun,
1445         .image_set = nv50_base_image_set,
1446         .image_clr = nv50_base_image_clr,
1447         .lut = nv50_base_lut,
1448         .update = nv50_base_update,
1449 };
1450
1451 static int
1452 nv50_base_new(struct nouveau_drm *drm, struct nv50_head *head,
1453               struct nv50_base **pbase)
1454 {
1455         struct nv50_disp *disp = nv50_disp(drm->dev);
1456         struct nv50_base *base;
1457         int ret;
1458
1459         if (!(base = *pbase = kzalloc(sizeof(*base), GFP_KERNEL)))
1460                 return -ENOMEM;
1461         base->id = head->base.index;
1462         base->wndw.ntfy = EVO_FLIP_NTFY0(base->id);
1463         base->wndw.sema = EVO_FLIP_SEM0(base->id);
1464         base->wndw.data = 0x00000000;
1465
1466         ret = nv50_wndw_ctor(&nv50_base, drm->dev, DRM_PLANE_TYPE_PRIMARY,
1467                              "base", base->id, &base->chan.base,
1468                              nv50_base_format, ARRAY_SIZE(nv50_base_format),
1469                              &base->wndw);
1470         if (ret) {
1471                 kfree(base);
1472                 return ret;
1473         }
1474
1475         ret = nv50_base_create(&drm->device, disp->disp, base->id,
1476                                disp->sync->bo.offset, &base->chan);
1477         if (ret)
1478                 return ret;
1479
1480         return nvif_notify_init(&base->chan.base.base.user, nv50_base_notify,
1481                                 false,
1482                                 NV50_DISP_BASE_CHANNEL_DMA_V0_NTFY_UEVENT,
1483                                 &(struct nvif_notify_uevent_req) {},
1484                                 sizeof(struct nvif_notify_uevent_req),
1485                                 sizeof(struct nvif_notify_uevent_rep),
1486                                 &base->wndw.notify);
1487 }
1488
1489 /******************************************************************************
1490  * Page flipping channel
1491  *****************************************************************************/
1492 struct nouveau_bo *
1493 nv50_display_crtc_sema(struct drm_device *dev, int crtc)
1494 {
1495         return nv50_disp(dev)->sync;
1496 }
1497
1498 struct nv50_display_flip {
1499         struct nv50_disp *disp;
1500         struct nv50_base *base;
1501 };
1502
1503 static bool
1504 nv50_display_flip_wait(void *data)
1505 {
1506         struct nv50_display_flip *flip = data;
1507         if (nouveau_bo_rd32(flip->disp->sync, flip->base->wndw.sema / 4) ==
1508                                               flip->base->wndw.data)
1509                 return true;
1510         usleep_range(1, 2);
1511         return false;
1512 }
1513
1514 void
1515 nv50_display_flip_stop(struct drm_crtc *crtc)
1516 {
1517         struct nvif_device *device = &nouveau_drm(crtc->dev)->device;
1518         struct nv50_base *base = nv50_head(crtc)->_base;
1519         struct nv50_wndw *wndw = &base->wndw;
1520         struct nv50_wndw_atom *asyw = &wndw->asy;
1521         struct nv50_display_flip flip = {
1522                 .disp = nv50_disp(crtc->dev),
1523                 .base = base,
1524         };
1525
1526         asyw->state.crtc = NULL;
1527         asyw->state.fb = NULL;
1528         nv50_wndw_atomic_check(&wndw->plane, &asyw->state);
1529         nv50_wndw_flush_clr(wndw, 0, true, asyw);
1530
1531         nvif_msec(device, 2000,
1532                 if (nv50_display_flip_wait(&flip))
1533                         break;
1534         );
1535 }
1536
1537 int
1538 nv50_display_flip_next(struct drm_crtc *crtc, struct drm_framebuffer *fb,
1539                        struct nouveau_channel *chan, u32 swap_interval)
1540 {
1541         struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
1542         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
1543         struct nv50_head *head = nv50_head(crtc);
1544         struct nv50_base *base = nv50_head(crtc)->_base;
1545         struct nv50_wndw *wndw = &base->wndw;
1546         struct nv50_wndw_atom *asyw = &wndw->asy;
1547         int ret;
1548
1549         if (crtc->primary->fb->width != fb->width ||
1550             crtc->primary->fb->height != fb->height)
1551                 return -EINVAL;
1552
1553         if (chan == NULL)
1554                 evo_sync(crtc->dev);
1555
1556         if (chan && chan->user.oclass < G82_CHANNEL_GPFIFO) {
1557                 ret = RING_SPACE(chan, 8);
1558                 if (ret)
1559                         return ret;
1560
1561                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 2);
1562                 OUT_RING  (chan, NvEvoSema0 + nv_crtc->index);
1563                 OUT_RING  (chan, base->wndw.sema ^ 0x10);
1564                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_RELEASE, 1);
1565                 OUT_RING  (chan, base->wndw.data + 1);
1566                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_SEMAPHORE_OFFSET, 2);
1567                 OUT_RING  (chan, base->wndw.sema);
1568                 OUT_RING  (chan, base->wndw.data);
1569         } else
1570         if (chan && chan->user.oclass < FERMI_CHANNEL_GPFIFO) {
1571                 u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + base->wndw.sema;
1572                 ret = RING_SPACE(chan, 12);
1573                 if (ret)
1574                         return ret;
1575
1576                 BEGIN_NV04(chan, 0, NV11_SUBCHAN_DMA_SEMAPHORE, 1);
1577                 OUT_RING  (chan, chan->vram.handle);
1578                 BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
1579                 OUT_RING  (chan, upper_32_bits(addr ^ 0x10));
1580                 OUT_RING  (chan, lower_32_bits(addr ^ 0x10));
1581                 OUT_RING  (chan, base->wndw.data + 1);
1582                 OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG);
1583                 BEGIN_NV04(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
1584                 OUT_RING  (chan, upper_32_bits(addr));
1585                 OUT_RING  (chan, lower_32_bits(addr));
1586                 OUT_RING  (chan, base->wndw.data);
1587                 OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL);
1588         } else
1589         if (chan) {
1590                 u64 addr = nv84_fence_crtc(chan, nv_crtc->index) + base->wndw.sema;
1591                 ret = RING_SPACE(chan, 10);
1592                 if (ret)
1593                         return ret;
1594
1595                 BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
1596                 OUT_RING  (chan, upper_32_bits(addr ^ 0x10));
1597                 OUT_RING  (chan, lower_32_bits(addr ^ 0x10));
1598                 OUT_RING  (chan, base->wndw.data + 1);
1599                 OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_WRITE_LONG |
1600                                  NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD);
1601                 BEGIN_NVC0(chan, 0, NV84_SUBCHAN_SEMAPHORE_ADDRESS_HIGH, 4);
1602                 OUT_RING  (chan, upper_32_bits(addr));
1603                 OUT_RING  (chan, lower_32_bits(addr));
1604                 OUT_RING  (chan, base->wndw.data);
1605                 OUT_RING  (chan, NV84_SUBCHAN_SEMAPHORE_TRIGGER_ACQUIRE_EQUAL |
1606                                  NVC0_SUBCHAN_SEMAPHORE_TRIGGER_YIELD);
1607         }
1608
1609         if (chan) {
1610                 base->wndw.sema ^= 0x10;
1611                 base->wndw.data++;
1612                 FIRE_RING (chan);
1613         }
1614
1615         /* queue the flip */
1616         asyw->state.crtc = &head->base.base;
1617         asyw->state.fb = fb;
1618         asyw->interval = swap_interval;
1619         asyw->image.handle = nv_fb->r_handle;
1620         asyw->image.offset = nv_fb->nvbo->bo.offset;
1621         asyw->sema.handle = base->chan.base.sync.handle;
1622         asyw->sema.offset = base->wndw.sema;
1623         asyw->sema.acquire = base->wndw.data++;
1624         asyw->sema.release = base->wndw.data;
1625         nv50_wndw_atomic_check(&wndw->plane, &asyw->state);
1626         asyw->set.sema = true;
1627         nv50_wndw_flush_set(wndw, 0, asyw);
1628         nv50_wndw_wait_armed(wndw, asyw);
1629
1630         nouveau_bo_ref(nv_fb->nvbo, &head->image);
1631         return 0;
1632 }
1633
1634 /******************************************************************************
1635  * Head
1636  *****************************************************************************/
1637 static void
1638 nv50_head_procamp(struct nv50_head *head, struct nv50_head_atom *asyh)
1639 {
1640         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1641         u32 *push;
1642         if ((push = evo_wait(core, 2))) {
1643                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
1644                         evo_mthd(push, 0x08a8 + (head->base.index * 0x400), 1);
1645                 else
1646                         evo_mthd(push, 0x0498 + (head->base.index * 0x300), 1);
1647                 evo_data(push, (asyh->procamp.sat.sin << 20) |
1648                                (asyh->procamp.sat.cos << 8));
1649                 evo_kick(push, core);
1650         }
1651 }
1652
1653 static void
1654 nv50_head_dither(struct nv50_head *head, struct nv50_head_atom *asyh)
1655 {
1656         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1657         u32 *push;
1658         if ((push = evo_wait(core, 2))) {
1659                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
1660                         evo_mthd(push, 0x08a0 + (head->base.index * 0x0400), 1);
1661                 else
1662                 if (core->base.user.oclass < GK104_DISP_CORE_CHANNEL_DMA)
1663                         evo_mthd(push, 0x0490 + (head->base.index * 0x0300), 1);
1664                 else
1665                         evo_mthd(push, 0x04a0 + (head->base.index * 0x0300), 1);
1666                 evo_data(push, (asyh->dither.mode << 3) |
1667                                (asyh->dither.bits << 1) |
1668                                 asyh->dither.enable);
1669                 evo_kick(push, core);
1670         }
1671 }
1672
1673 static void
1674 nv50_head_ovly(struct nv50_head *head, struct nv50_head_atom *asyh)
1675 {
1676         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1677         u32 bounds = 0;
1678         u32 *push;
1679
1680         if (asyh->base.cpp) {
1681                 switch (asyh->base.cpp) {
1682                 case 8: bounds |= 0x00000500; break;
1683                 case 4: bounds |= 0x00000300; break;
1684                 case 2: bounds |= 0x00000100; break;
1685                 default:
1686                         WARN_ON(1);
1687                         break;
1688                 }
1689                 bounds |= 0x00000001;
1690         }
1691
1692         if ((push = evo_wait(core, 2))) {
1693                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
1694                         evo_mthd(push, 0x0904 + head->base.index * 0x400, 1);
1695                 else
1696                         evo_mthd(push, 0x04d4 + head->base.index * 0x300, 1);
1697                 evo_data(push, bounds);
1698                 evo_kick(push, core);
1699         }
1700 }
1701
1702 static void
1703 nv50_head_base(struct nv50_head *head, struct nv50_head_atom *asyh)
1704 {
1705         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1706         u32 bounds = 0;
1707         u32 *push;
1708
1709         if (asyh->base.cpp) {
1710                 switch (asyh->base.cpp) {
1711                 case 8: bounds |= 0x00000500; break;
1712                 case 4: bounds |= 0x00000300; break;
1713                 case 2: bounds |= 0x00000100; break;
1714                 case 1: bounds |= 0x00000000; break;
1715                 default:
1716                         WARN_ON(1);
1717                         break;
1718                 }
1719                 bounds |= 0x00000001;
1720         }
1721
1722         if ((push = evo_wait(core, 2))) {
1723                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
1724                         evo_mthd(push, 0x0900 + head->base.index * 0x400, 1);
1725                 else
1726                         evo_mthd(push, 0x04d0 + head->base.index * 0x300, 1);
1727                 evo_data(push, bounds);
1728                 evo_kick(push, core);
1729         }
1730 }
1731
1732 static void
1733 nv50_head_curs_clr(struct nv50_head *head)
1734 {
1735         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1736         u32 *push;
1737         if ((push = evo_wait(core, 4))) {
1738                 if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
1739                         evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
1740                         evo_data(push, 0x05000000);
1741                 } else
1742                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1743                         evo_mthd(push, 0x0880 + head->base.index * 0x400, 1);
1744                         evo_data(push, 0x05000000);
1745                         evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
1746                         evo_data(push, 0x00000000);
1747                 } else {
1748                         evo_mthd(push, 0x0480 + head->base.index * 0x300, 1);
1749                         evo_data(push, 0x05000000);
1750                         evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
1751                         evo_data(push, 0x00000000);
1752                 }
1753                 evo_kick(push, core);
1754         }
1755 }
1756
1757 static void
1758 nv50_head_curs_set(struct nv50_head *head, struct nv50_head_atom *asyh)
1759 {
1760         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1761         u32 *push;
1762         if ((push = evo_wait(core, 5))) {
1763                 if (core->base.user.oclass < G82_DISP_BASE_CHANNEL_DMA) {
1764                         evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
1765                         evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
1766                                                     (asyh->curs.format << 24));
1767                         evo_data(push, asyh->curs.offset >> 8);
1768                 } else
1769                 if (core->base.user.oclass < GF110_DISP_BASE_CHANNEL_DMA) {
1770                         evo_mthd(push, 0x0880 + head->base.index * 0x400, 2);
1771                         evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
1772                                                     (asyh->curs.format << 24));
1773                         evo_data(push, asyh->curs.offset >> 8);
1774                         evo_mthd(push, 0x089c + head->base.index * 0x400, 1);
1775                         evo_data(push, asyh->curs.handle);
1776                 } else {
1777                         evo_mthd(push, 0x0480 + head->base.index * 0x300, 2);
1778                         evo_data(push, 0x80000000 | (asyh->curs.layout << 26) |
1779                                                     (asyh->curs.format << 24));
1780                         evo_data(push, asyh->curs.offset >> 8);
1781                         evo_mthd(push, 0x048c + head->base.index * 0x300, 1);
1782                         evo_data(push, asyh->curs.handle);
1783                 }
1784                 evo_kick(push, core);
1785         }
1786 }
1787
1788 static void
1789 nv50_head_core_clr(struct nv50_head *head)
1790 {
1791         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1792         u32 *push;
1793         if ((push = evo_wait(core, 2))) {
1794                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA)
1795                         evo_mthd(push, 0x0874 + head->base.index * 0x400, 1);
1796                 else
1797                         evo_mthd(push, 0x0474 + head->base.index * 0x300, 1);
1798                 evo_data(push, 0x00000000);
1799                 evo_kick(push, core);
1800         }
1801 }
1802
1803 static void
1804 nv50_head_core_set(struct nv50_head *head, struct nv50_head_atom *asyh)
1805 {
1806         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1807         u32 *push;
1808         if ((push = evo_wait(core, 9))) {
1809                 if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
1810                         evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
1811                         evo_data(push, asyh->core.offset >> 8);
1812                         evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
1813                         evo_data(push, (asyh->core.h << 16) | asyh->core.w);
1814                         evo_data(push, asyh->core.layout << 20 |
1815                                        (asyh->core.pitch >> 8) << 8 |
1816                                        asyh->core.block);
1817                         evo_data(push, asyh->core.kind << 16 |
1818                                        asyh->core.format << 8);
1819                         evo_data(push, asyh->core.handle);
1820                         evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
1821                         evo_data(push, (asyh->core.y << 16) | asyh->core.x);
1822                 } else
1823                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1824                         evo_mthd(push, 0x0860 + head->base.index * 0x400, 1);
1825                         evo_data(push, asyh->core.offset >> 8);
1826                         evo_mthd(push, 0x0868 + head->base.index * 0x400, 4);
1827                         evo_data(push, (asyh->core.h << 16) | asyh->core.w);
1828                         evo_data(push, asyh->core.layout << 20 |
1829                                        (asyh->core.pitch >> 8) << 8 |
1830                                        asyh->core.block);
1831                         evo_data(push, asyh->core.format << 8);
1832                         evo_data(push, asyh->core.handle);
1833                         evo_mthd(push, 0x08c0 + head->base.index * 0x400, 1);
1834                         evo_data(push, (asyh->core.y << 16) | asyh->core.x);
1835                 } else {
1836                         evo_mthd(push, 0x0460 + head->base.index * 0x300, 1);
1837                         evo_data(push, asyh->core.offset >> 8);
1838                         evo_mthd(push, 0x0468 + head->base.index * 0x300, 4);
1839                         evo_data(push, (asyh->core.h << 16) | asyh->core.w);
1840                         evo_data(push, asyh->core.layout << 24 |
1841                                        (asyh->core.pitch >> 8) << 8 |
1842                                        asyh->core.block);
1843                         evo_data(push, asyh->core.format << 8);
1844                         evo_data(push, asyh->core.handle);
1845                         evo_mthd(push, 0x04b0 + head->base.index * 0x300, 1);
1846                         evo_data(push, (asyh->core.y << 16) | asyh->core.x);
1847                 }
1848                 evo_kick(push, core);
1849         }
1850 }
1851
1852 static void
1853 nv50_head_lut_clr(struct nv50_head *head)
1854 {
1855         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1856         u32 *push;
1857         if ((push = evo_wait(core, 4))) {
1858                 if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
1859                         evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
1860                         evo_data(push, 0x40000000);
1861                 } else
1862                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1863                         evo_mthd(push, 0x0840 + (head->base.index * 0x400), 1);
1864                         evo_data(push, 0x40000000);
1865                         evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
1866                         evo_data(push, 0x00000000);
1867                 } else {
1868                         evo_mthd(push, 0x0440 + (head->base.index * 0x300), 1);
1869                         evo_data(push, 0x03000000);
1870                         evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
1871                         evo_data(push, 0x00000000);
1872                 }
1873                 evo_kick(push, core);
1874         }
1875 }
1876
1877 static void
1878 nv50_head_lut_set(struct nv50_head *head, struct nv50_head_atom *asyh)
1879 {
1880         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1881         u32 *push;
1882         if ((push = evo_wait(core, 7))) {
1883                 if (core->base.user.oclass < G82_DISP_CORE_CHANNEL_DMA) {
1884                         evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
1885                         evo_data(push, 0xc0000000);
1886                         evo_data(push, asyh->lut.offset >> 8);
1887                 } else
1888                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1889                         evo_mthd(push, 0x0840 + (head->base.index * 0x400), 2);
1890                         evo_data(push, 0xc0000000);
1891                         evo_data(push, asyh->lut.offset >> 8);
1892                         evo_mthd(push, 0x085c + (head->base.index * 0x400), 1);
1893                         evo_data(push, asyh->lut.handle);
1894                 } else {
1895                         evo_mthd(push, 0x0440 + (head->base.index * 0x300), 4);
1896                         evo_data(push, 0x83000000);
1897                         evo_data(push, asyh->lut.offset >> 8);
1898                         evo_data(push, 0x00000000);
1899                         evo_data(push, 0x00000000);
1900                         evo_mthd(push, 0x045c + (head->base.index * 0x300), 1);
1901                         evo_data(push, asyh->lut.handle);
1902                 }
1903                 evo_kick(push, core);
1904         }
1905 }
1906
1907 static void
1908 nv50_head_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
1909 {
1910         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1911         struct nv50_head_mode *m = &asyh->mode;
1912         u32 *push;
1913         if ((push = evo_wait(core, 14))) {
1914                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1915                         evo_mthd(push, 0x0804 + (head->base.index * 0x400), 2);
1916                         evo_data(push, 0x00800000 | m->clock);
1917                         evo_data(push, m->interlace ? 0x00000002 : 0x00000000);
1918                         evo_mthd(push, 0x0810 + (head->base.index * 0x400), 7);
1919                         evo_data(push, 0x00000000);
1920                         evo_data(push, (m->v.active  << 16) | m->h.active );
1921                         evo_data(push, (m->v.synce   << 16) | m->h.synce  );
1922                         evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
1923                         evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
1924                         evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
1925                         evo_data(push, asyh->mode.v.blankus);
1926                         evo_mthd(push, 0x082c + (head->base.index * 0x400), 1);
1927                         evo_data(push, 0x00000000);
1928                 } else {
1929                         evo_mthd(push, 0x0410 + (head->base.index * 0x300), 6);
1930                         evo_data(push, 0x00000000);
1931                         evo_data(push, (m->v.active  << 16) | m->h.active );
1932                         evo_data(push, (m->v.synce   << 16) | m->h.synce  );
1933                         evo_data(push, (m->v.blanke  << 16) | m->h.blanke );
1934                         evo_data(push, (m->v.blanks  << 16) | m->h.blanks );
1935                         evo_data(push, (m->v.blank2e << 16) | m->v.blank2s);
1936                         evo_mthd(push, 0x042c + (head->base.index * 0x300), 2);
1937                         evo_data(push, 0x00000000); /* ??? */
1938                         evo_data(push, 0xffffff00);
1939                         evo_mthd(push, 0x0450 + (head->base.index * 0x300), 3);
1940                         evo_data(push, m->clock * 1000);
1941                         evo_data(push, 0x00200000); /* ??? */
1942                         evo_data(push, m->clock * 1000);
1943                 }
1944                 evo_kick(push, core);
1945         }
1946 }
1947
1948 static void
1949 nv50_head_view(struct nv50_head *head, struct nv50_head_atom *asyh)
1950 {
1951         struct nv50_dmac *core = &nv50_disp(head->base.base.dev)->mast.base;
1952         u32 *push;
1953         if ((push = evo_wait(core, 10))) {
1954                 if (core->base.user.oclass < GF110_DISP_CORE_CHANNEL_DMA) {
1955                         evo_mthd(push, 0x08a4 + (head->base.index * 0x400), 1);
1956                         evo_data(push, 0x00000000);
1957                         evo_mthd(push, 0x08c8 + (head->base.index * 0x400), 1);
1958                         evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
1959                         evo_mthd(push, 0x08d8 + (head->base.index * 0x400), 2);
1960                         evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
1961                         evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
1962                 } else {
1963                         evo_mthd(push, 0x0494 + (head->base.index * 0x300), 1);
1964                         evo_data(push, 0x00000000);
1965                         evo_mthd(push, 0x04b8 + (head->base.index * 0x300), 1);
1966                         evo_data(push, (asyh->view.iH << 16) | asyh->view.iW);
1967                         evo_mthd(push, 0x04c0 + (head->base.index * 0x300), 3);
1968                         evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
1969                         evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
1970                         evo_data(push, (asyh->view.oH << 16) | asyh->view.oW);
1971                 }
1972                 evo_kick(push, core);
1973         }
1974 }
1975
1976 static void
1977 nv50_head_flush_clr(struct nv50_head *head, struct nv50_head_atom *asyh, bool y)
1978 {
1979         if (asyh->clr.core && (!asyh->set.core || y))
1980                 nv50_head_lut_clr(head);
1981         if (asyh->clr.core && (!asyh->set.core || y))
1982                 nv50_head_core_clr(head);
1983         if (asyh->clr.curs && (!asyh->set.curs || y))
1984                 nv50_head_curs_clr(head);
1985 }
1986
1987 static void
1988 nv50_head_flush_set(struct nv50_head *head, struct nv50_head_atom *asyh)
1989 {
1990         if (asyh->set.view   ) nv50_head_view    (head, asyh);
1991         if (asyh->set.mode   ) nv50_head_mode    (head, asyh);
1992         if (asyh->set.core   ) nv50_head_lut_set (head, asyh);
1993         if (asyh->set.core   ) nv50_head_core_set(head, asyh);
1994         if (asyh->set.curs   ) nv50_head_curs_set(head, asyh);
1995         if (asyh->set.base   ) nv50_head_base    (head, asyh);
1996         if (asyh->set.ovly   ) nv50_head_ovly    (head, asyh);
1997         if (asyh->set.dither ) nv50_head_dither  (head, asyh);
1998         if (asyh->set.procamp) nv50_head_procamp (head, asyh);
1999 }
2000
2001 static void
2002 nv50_head_atomic_check_procamp(struct nv50_head_atom *armh,
2003                                struct nv50_head_atom *asyh,
2004                                struct nouveau_conn_atom *asyc)
2005 {
2006         const int vib = asyc->procamp.color_vibrance - 100;
2007         const int hue = asyc->procamp.vibrant_hue - 90;
2008         const int adj = (vib > 0) ? 50 : 0;
2009         asyh->procamp.sat.cos = ((vib * 2047 + adj) / 100) & 0xfff;
2010         asyh->procamp.sat.sin = ((hue * 2047) / 100) & 0xfff;
2011         asyh->set.procamp = true;
2012 }
2013
2014 static void
2015 nv50_head_atomic_check_dither(struct nv50_head_atom *armh,
2016                               struct nv50_head_atom *asyh,
2017                               struct nouveau_conn_atom *asyc)
2018 {
2019         struct drm_connector *connector = asyc->state.connector;
2020         u32 mode = 0x00;
2021
2022         if (asyc->dither.mode == DITHERING_MODE_AUTO) {
2023                 if (asyh->base.depth > connector->display_info.bpc * 3)
2024                         mode = DITHERING_MODE_DYNAMIC2X2;
2025         } else {
2026                 mode = asyc->dither.mode;
2027         }
2028
2029         if (asyc->dither.depth == DITHERING_DEPTH_AUTO) {
2030                 if (connector->display_info.bpc >= 8)
2031                         mode |= DITHERING_DEPTH_8BPC;
2032         } else {
2033                 mode |= asyc->dither.depth;
2034         }
2035
2036         asyh->dither.enable = mode;
2037         asyh->dither.bits = mode >> 1;
2038         asyh->dither.mode = mode >> 3;
2039         asyh->set.dither = true;
2040 }
2041
2042 static void
2043 nv50_head_atomic_check_view(struct nv50_head_atom *armh,
2044                             struct nv50_head_atom *asyh,
2045                             struct nouveau_conn_atom *asyc)
2046 {
2047         struct drm_connector *connector = asyc->state.connector;
2048         struct drm_display_mode *omode = &asyh->state.adjusted_mode;
2049         struct drm_display_mode *umode = &asyh->state.mode;
2050         int mode = asyc->scaler.mode;
2051         struct edid *edid;
2052
2053         if (connector->edid_blob_ptr)
2054                 edid = (struct edid *)connector->edid_blob_ptr->data;
2055         else
2056                 edid = NULL;
2057
2058         if (!asyc->scaler.full) {
2059                 if (mode == DRM_MODE_SCALE_NONE)
2060                         omode = umode;
2061         } else {
2062                 /* Non-EDID LVDS/eDP mode. */
2063                 mode = DRM_MODE_SCALE_FULLSCREEN;
2064         }
2065
2066         asyh->view.iW = umode->hdisplay;
2067         asyh->view.iH = umode->vdisplay;
2068         asyh->view.oW = omode->hdisplay;
2069         asyh->view.oH = omode->vdisplay;
2070         if (omode->flags & DRM_MODE_FLAG_DBLSCAN)
2071                 asyh->view.oH *= 2;
2072
2073         /* Add overscan compensation if necessary, will keep the aspect
2074          * ratio the same as the backend mode unless overridden by the
2075          * user setting both hborder and vborder properties.
2076          */
2077         if ((asyc->scaler.underscan.mode == UNDERSCAN_ON ||
2078             (asyc->scaler.underscan.mode == UNDERSCAN_AUTO &&
2079              drm_detect_hdmi_monitor(edid)))) {
2080                 u32 bX = asyc->scaler.underscan.hborder;
2081                 u32 bY = asyc->scaler.underscan.vborder;
2082                 u32 r = (asyh->view.oH << 19) / asyh->view.oW;
2083
2084                 if (bX) {
2085                         asyh->view.oW -= (bX * 2);
2086                         if (bY) asyh->view.oH -= (bY * 2);
2087                         else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
2088                 } else {
2089                         asyh->view.oW -= (asyh->view.oW >> 4) + 32;
2090                         if (bY) asyh->view.oH -= (bY * 2);
2091                         else    asyh->view.oH  = ((asyh->view.oW * r) + (r / 2)) >> 19;
2092                 }
2093         }
2094
2095         /* Handle CENTER/ASPECT scaling, taking into account the areas
2096          * removed already for overscan compensation.
2097          */
2098         switch (mode) {
2099         case DRM_MODE_SCALE_CENTER:
2100                 asyh->view.oW = min((u16)umode->hdisplay, asyh->view.oW);
2101                 asyh->view.oH = min((u16)umode->vdisplay, asyh->view.oH);
2102                 /* fall-through */
2103         case DRM_MODE_SCALE_ASPECT:
2104                 if (asyh->view.oH < asyh->view.oW) {
2105                         u32 r = (asyh->view.iW << 19) / asyh->view.iH;
2106                         asyh->view.oW = ((asyh->view.oH * r) + (r / 2)) >> 19;
2107                 } else {
2108                         u32 r = (asyh->view.iH << 19) / asyh->view.iW;
2109                         asyh->view.oH = ((asyh->view.oW * r) + (r / 2)) >> 19;
2110                 }
2111                 break;
2112         default:
2113                 break;
2114         }
2115
2116         asyh->set.view = true;
2117 }
2118
2119 static void
2120 nv50_head_atomic_check_mode(struct nv50_head *head, struct nv50_head_atom *asyh)
2121 {
2122         struct drm_display_mode *mode = &asyh->state.adjusted_mode;
2123         u32 ilace   = (mode->flags & DRM_MODE_FLAG_INTERLACE) ? 2 : 1;
2124         u32 vscan   = (mode->flags & DRM_MODE_FLAG_DBLSCAN) ? 2 : 1;
2125         u32 hbackp  =  mode->htotal - mode->hsync_end;
2126         u32 vbackp  = (mode->vtotal - mode->vsync_end) * vscan / ilace;
2127         u32 hfrontp =  mode->hsync_start - mode->hdisplay;
2128         u32 vfrontp = (mode->vsync_start - mode->vdisplay) * vscan / ilace;
2129         struct nv50_head_mode *m = &asyh->mode;
2130
2131         m->h.active = mode->htotal;
2132         m->h.synce  = mode->hsync_end - mode->hsync_start - 1;
2133         m->h.blanke = m->h.synce + hbackp;
2134         m->h.blanks = mode->htotal - hfrontp - 1;
2135
2136         m->v.active = mode->vtotal * vscan / ilace;
2137         m->v.synce  = ((mode->vsync_end - mode->vsync_start) * vscan / ilace) - 1;
2138         m->v.blanke = m->v.synce + vbackp;
2139         m->v.blanks = m->v.active - vfrontp - 1;
2140
2141         /*XXX: Safe underestimate, even "0" works */
2142         m->v.blankus = (m->v.active - mode->vdisplay - 2) * m->h.active;
2143         m->v.blankus *= 1000;
2144         m->v.blankus /= mode->clock;
2145
2146         if (mode->flags & DRM_MODE_FLAG_INTERLACE) {
2147                 m->v.blank2e =  m->v.active + m->v.synce + vbackp;
2148                 m->v.blank2s =  m->v.blank2e + (mode->vdisplay * vscan / ilace);
2149                 m->v.active  = (m->v.active * 2) + 1;
2150                 m->interlace = true;
2151         } else {
2152                 m->v.blank2e = 0;
2153                 m->v.blank2s = 1;
2154                 m->interlace = false;
2155         }
2156         m->clock = mode->clock;
2157
2158         drm_mode_set_crtcinfo(mode, CRTC_INTERLACE_HALVE_V);
2159         asyh->set.mode = true;
2160 }
2161
2162 static int
2163 nv50_head_atomic_check(struct drm_crtc *crtc, struct drm_crtc_state *state)
2164 {
2165         struct nouveau_drm *drm = nouveau_drm(crtc->dev);
2166         struct nv50_disp *disp = nv50_disp(crtc->dev);
2167         struct nv50_head *head = nv50_head(crtc);
2168         struct nv50_head_atom *armh = &head->arm;
2169         struct nv50_head_atom *asyh = nv50_head_atom(state);
2170
2171         NV_ATOMIC(drm, "%s atomic_check %d\n", crtc->name, asyh->state.active);
2172         asyh->clr.mask = 0;
2173         asyh->set.mask = 0;
2174
2175         if (asyh->state.active) {
2176                 if (asyh->state.mode_changed)
2177                         nv50_head_atomic_check_mode(head, asyh);
2178
2179                 if ((asyh->core.visible = (asyh->base.cpp != 0))) {
2180                         asyh->core.x = asyh->base.x;
2181                         asyh->core.y = asyh->base.y;
2182                         asyh->core.w = asyh->base.w;
2183                         asyh->core.h = asyh->base.h;
2184                 } else
2185                 if ((asyh->core.visible = asyh->curs.visible)) {
2186                         /*XXX: We need to either find some way of having the
2187                          *     primary base layer appear black, while still
2188                          *     being able to display the other layers, or we
2189                          *     need to allocate a dummy black surface here.
2190                          */
2191                         asyh->core.x = 0;
2192                         asyh->core.y = 0;
2193                         asyh->core.w = asyh->state.mode.hdisplay;
2194                         asyh->core.h = asyh->state.mode.vdisplay;
2195                 }
2196                 asyh->core.handle = disp->mast.base.vram.handle;
2197                 asyh->core.offset = 0;
2198                 asyh->core.format = 0xcf;
2199                 asyh->core.kind = 0;
2200                 asyh->core.layout = 1;
2201                 asyh->core.block = 0;
2202                 asyh->core.pitch = ALIGN(asyh->core.w, 64) * 4;
2203                 asyh->lut.handle = disp->mast.base.vram.handle;
2204                 asyh->lut.offset = head->base.lut.nvbo->bo.offset;
2205                 asyh->set.base = armh->base.cpp != asyh->base.cpp;
2206                 asyh->set.ovly = armh->ovly.cpp != asyh->ovly.cpp;
2207         } else {
2208                 asyh->core.visible = false;
2209                 asyh->curs.visible = false;
2210                 asyh->base.cpp = 0;
2211                 asyh->ovly.cpp = 0;
2212         }
2213
2214         if (!drm_atomic_crtc_needs_modeset(&asyh->state)) {
2215                 if (asyh->core.visible) {
2216                         if (memcmp(&armh->core, &asyh->core, sizeof(asyh->core)))
2217                                 asyh->set.core = true;
2218                 } else
2219                 if (armh->core.visible) {
2220                         asyh->clr.core = true;
2221                 }
2222
2223                 if (asyh->curs.visible) {
2224                         if (memcmp(&armh->curs, &asyh->curs, sizeof(asyh->curs)))
2225                                 asyh->set.curs = true;
2226                 } else
2227                 if (armh->curs.visible) {
2228                         asyh->clr.curs = true;
2229                 }
2230         } else {
2231                 asyh->clr.core = armh->core.visible;
2232                 asyh->clr.curs = armh->curs.visible;
2233                 asyh->set.core = asyh->core.visible;
2234                 asyh->set.curs = asyh->curs.visible;
2235         }
2236
2237         memcpy(armh, asyh, sizeof(*asyh));
2238         asyh->state.mode_changed = 0;
2239         return 0;
2240 }
2241
2242 /******************************************************************************
2243  * CRTC
2244  *****************************************************************************/
2245 static int
2246 nv50_crtc_set_dither(struct nouveau_crtc *nv_crtc, bool update)
2247 {
2248         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
2249         struct nv50_head *head = nv50_head(&nv_crtc->base);
2250         struct nv50_head_atom *asyh = &head->asy;
2251         struct nouveau_connector *nv_connector;
2252         struct nouveau_conn_atom asyc;
2253         u32 *push;
2254
2255         nv_connector = nouveau_crtc_connector_get(nv_crtc);
2256
2257         asyc.state.connector = &nv_connector->base;
2258         asyc.dither.mode = nv_connector->dithering_mode;
2259         asyc.dither.depth = nv_connector->dithering_depth;
2260         asyh->state.crtc = &nv_crtc->base;
2261         nv50_head_atomic_check(&head->base.base, &asyh->state);
2262         nv50_head_atomic_check_dither(&head->arm, asyh, &asyc);
2263         nv50_head_flush_set(head, asyh);
2264
2265         if (update) {
2266                 if ((push = evo_wait(mast, 2))) {
2267                         evo_mthd(push, 0x0080, 1);
2268                         evo_data(push, 0x00000000);
2269                         evo_kick(push, mast);
2270                 }
2271         }
2272
2273         return 0;
2274 }
2275
2276 static int
2277 nv50_crtc_set_scale(struct nouveau_crtc *nv_crtc, bool update)
2278 {
2279         struct nv50_head *head = nv50_head(&nv_crtc->base);
2280         struct nv50_head_atom *asyh = &head->asy;
2281         struct drm_crtc *crtc = &nv_crtc->base;
2282         struct nouveau_connector *nv_connector;
2283         struct nouveau_conn_atom asyc;
2284
2285         nv_connector = nouveau_crtc_connector_get(nv_crtc);
2286
2287         asyc.state.connector = &nv_connector->base;
2288         asyc.scaler.mode = nv_connector->scaling_mode;
2289         asyc.scaler.full = nv_connector->scaling_full;
2290         asyc.scaler.underscan.mode = nv_connector->underscan;
2291         asyc.scaler.underscan.hborder = nv_connector->underscan_hborder;
2292         asyc.scaler.underscan.vborder = nv_connector->underscan_vborder;
2293         nv50_head_atomic_check(&head->base.base, &asyh->state);
2294         nv50_head_atomic_check_view(&head->arm, asyh, &asyc);
2295         nv50_head_flush_set(head, asyh);
2296
2297         if (update) {
2298                 nv50_display_flip_stop(crtc);
2299                 nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
2300         }
2301
2302         return 0;
2303 }
2304
2305 static int
2306 nv50_crtc_set_color_vibrance(struct nouveau_crtc *nv_crtc, bool update)
2307 {
2308         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
2309         struct nv50_head *head = nv50_head(&nv_crtc->base);
2310         struct nv50_head_atom *asyh = &head->asy;
2311         struct nouveau_conn_atom asyc;
2312         u32 *push;
2313
2314         asyc.procamp.color_vibrance = nv_crtc->color_vibrance + 100;
2315         asyc.procamp.vibrant_hue = nv_crtc->vibrant_hue + 90;
2316         nv50_head_atomic_check(&head->base.base, &asyh->state);
2317         nv50_head_atomic_check_procamp(&head->arm, asyh, &asyc);
2318         nv50_head_flush_set(head, asyh);
2319
2320         if (update) {
2321                 if ((push = evo_wait(mast, 2))) {
2322                         evo_mthd(push, 0x0080, 1);
2323                         evo_data(push, 0x00000000);
2324                         evo_kick(push, mast);
2325                 }
2326         }
2327
2328         return 0;
2329 }
2330
2331 static int
2332 nv50_crtc_set_image(struct nouveau_crtc *nv_crtc, struct drm_framebuffer *fb,
2333                     int x, int y, bool update)
2334 {
2335         struct nouveau_framebuffer *nvfb = nouveau_framebuffer(fb);
2336         struct nv50_head *head = nv50_head(&nv_crtc->base);
2337         struct nv50_head_atom *asyh = &head->asy;
2338         struct nv50_wndw_atom *asyw = &head->_base->wndw.asy;
2339         const struct drm_format_info *info;
2340
2341         info = drm_format_info(nvfb->base.pixel_format);
2342         if (!info || !info->depth)
2343                 return -EINVAL;
2344
2345         asyh->base.depth = info->depth;
2346         asyh->base.cpp = info->cpp[0];
2347         asyh->base.x = x;
2348         asyh->base.y = y;
2349         asyh->base.w = nvfb->base.width;
2350         asyh->base.h = nvfb->base.height;
2351         asyw->state.src_x = x << 16;
2352         asyw->state.src_y = y << 16;
2353         nv50_head_atomic_check(&head->base.base, &asyh->state);
2354         nv50_head_flush_set(head, asyh);
2355
2356         if (update) {
2357                 struct nv50_mast *core = nv50_mast(nv_crtc->base.dev);
2358                 u32 *push = evo_wait(core, 2);
2359                 if (push) {
2360                         evo_mthd(push, 0x0080, 1);
2361                         evo_data(push, 0x00000000);
2362                         evo_kick(push, core);
2363                 }
2364         }
2365
2366         nv_crtc->fb.handle = nvfb->r_handle;
2367         return 0;
2368 }
2369
2370 static void
2371 nv50_crtc_cursor_show(struct nouveau_crtc *nv_crtc)
2372 {
2373         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
2374         struct nv50_head *head = nv50_head(&nv_crtc->base);
2375         struct nv50_head_atom *asyh = &head->asy;
2376
2377         asyh->curs.visible = true;
2378         asyh->curs.handle = mast->base.vram.handle;
2379         asyh->curs.offset = nv_crtc->cursor.nvbo->bo.offset;
2380         asyh->curs.layout = 1;
2381         asyh->curs.format = 1;
2382         nv50_head_atomic_check(&head->base.base, &asyh->state);
2383         nv50_head_flush_set(head, asyh);
2384 }
2385
2386 static void
2387 nv50_crtc_cursor_hide(struct nouveau_crtc *nv_crtc)
2388 {
2389         struct nv50_head *head = nv50_head(&nv_crtc->base);
2390         struct nv50_head_atom *asyh = &head->asy;
2391
2392         asyh->curs.visible = false;
2393         nv50_head_atomic_check(&head->base.base, &asyh->state);
2394         nv50_head_flush_clr(head, asyh, false);
2395 }
2396
2397 static void
2398 nv50_crtc_cursor_show_hide(struct nouveau_crtc *nv_crtc, bool show, bool update)
2399 {
2400         struct nv50_mast *mast = nv50_mast(nv_crtc->base.dev);
2401
2402         if (show && nv_crtc->cursor.nvbo && nv_crtc->base.enabled)
2403                 nv50_crtc_cursor_show(nv_crtc);
2404         else
2405                 nv50_crtc_cursor_hide(nv_crtc);
2406
2407         if (update) {
2408                 u32 *push = evo_wait(mast, 2);
2409                 if (push) {
2410                         evo_mthd(push, 0x0080, 1);
2411                         evo_data(push, 0x00000000);
2412                         evo_kick(push, mast);
2413                 }
2414         }
2415 }
2416
2417 static void
2418 nv50_crtc_dpms(struct drm_crtc *crtc, int mode)
2419 {
2420 }
2421
2422 static void
2423 nv50_crtc_prepare(struct drm_crtc *crtc)
2424 {
2425         struct nv50_head *head = nv50_head(crtc);
2426         struct nv50_head_atom *asyh = &head->asy;
2427
2428         nv50_display_flip_stop(crtc);
2429
2430         asyh->state.active = false;
2431         nv50_head_atomic_check(&head->base.base, &asyh->state);
2432         nv50_head_flush_clr(head, asyh, false);
2433 }
2434
2435 static void
2436 nv50_crtc_commit(struct drm_crtc *crtc)
2437 {
2438         struct nv50_head *head = nv50_head(crtc);
2439         struct nv50_head_atom *asyh = &head->asy;
2440
2441         asyh->state.active = true;
2442         nv50_head_atomic_check(&head->base.base, &asyh->state);
2443         nv50_head_flush_set(head, asyh);
2444
2445         nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
2446 }
2447
2448 static bool
2449 nv50_crtc_mode_fixup(struct drm_crtc *crtc, const struct drm_display_mode *mode,
2450                      struct drm_display_mode *adjusted_mode)
2451 {
2452         drm_mode_set_crtcinfo(adjusted_mode, CRTC_INTERLACE_HALVE_V);
2453         return true;
2454 }
2455
2456 static int
2457 nv50_crtc_swap_fbs(struct drm_crtc *crtc, struct drm_framebuffer *old_fb)
2458 {
2459         struct nouveau_framebuffer *nvfb = nouveau_framebuffer(crtc->primary->fb);
2460         struct nv50_head *head = nv50_head(crtc);
2461         int ret;
2462
2463         ret = nouveau_bo_pin(nvfb->nvbo, TTM_PL_FLAG_VRAM, true);
2464         if (ret == 0) {
2465                 if (head->image)
2466                         nouveau_bo_unpin(head->image);
2467                 nouveau_bo_ref(nvfb->nvbo, &head->image);
2468         }
2469
2470         return ret;
2471 }
2472
2473 static int
2474 nv50_crtc_mode_set(struct drm_crtc *crtc, struct drm_display_mode *umode,
2475                    struct drm_display_mode *mode, int x, int y,
2476                    struct drm_framebuffer *old_fb)
2477 {
2478         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2479         struct nouveau_connector *nv_connector;
2480         int ret;
2481         struct nv50_head *head = nv50_head(crtc);
2482         struct nv50_head_atom *asyh = &head->asy;
2483
2484         memcpy(&asyh->state.mode, umode, sizeof(*umode));
2485         memcpy(&asyh->state.adjusted_mode, mode, sizeof(*mode));
2486         asyh->state.active = true;
2487         asyh->state.mode_changed = true;
2488         nv50_head_atomic_check(&head->base.base, &asyh->state);
2489
2490         ret = nv50_crtc_swap_fbs(crtc, old_fb);
2491         if (ret)
2492                 return ret;
2493
2494         nv50_head_flush_set(head, asyh);
2495
2496         nv_connector = nouveau_crtc_connector_get(nv_crtc);
2497         nv50_crtc_set_dither(nv_crtc, false);
2498         nv50_crtc_set_scale(nv_crtc, false);
2499
2500         nv50_crtc_set_color_vibrance(nv_crtc, false);
2501         nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, false);
2502         return 0;
2503 }
2504
2505 static int
2506 nv50_crtc_mode_set_base(struct drm_crtc *crtc, int x, int y,
2507                         struct drm_framebuffer *old_fb)
2508 {
2509         struct nouveau_drm *drm = nouveau_drm(crtc->dev);
2510         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2511         int ret;
2512
2513         if (!crtc->primary->fb) {
2514                 NV_DEBUG(drm, "No FB bound\n");
2515                 return 0;
2516         }
2517
2518         ret = nv50_crtc_swap_fbs(crtc, old_fb);
2519         if (ret)
2520                 return ret;
2521
2522         nv50_display_flip_stop(crtc);
2523         nv50_crtc_set_image(nv_crtc, crtc->primary->fb, x, y, true);
2524         nv50_display_flip_next(crtc, crtc->primary->fb, NULL, 1);
2525         return 0;
2526 }
2527
2528 static int
2529 nv50_crtc_mode_set_base_atomic(struct drm_crtc *crtc,
2530                                struct drm_framebuffer *fb, int x, int y,
2531                                enum mode_set_atomic state)
2532 {
2533         WARN_ON(1);
2534         return 0;
2535 }
2536
2537 static void
2538 nv50_crtc_lut_load(struct drm_crtc *crtc)
2539 {
2540         struct nv50_disp *disp = nv50_disp(crtc->dev);
2541         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2542         void __iomem *lut = nvbo_kmap_obj_iovirtual(nv_crtc->lut.nvbo);
2543         int i;
2544
2545         for (i = 0; i < 256; i++) {
2546                 u16 r = nv_crtc->lut.r[i] >> 2;
2547                 u16 g = nv_crtc->lut.g[i] >> 2;
2548                 u16 b = nv_crtc->lut.b[i] >> 2;
2549
2550                 if (disp->disp->oclass < GF110_DISP) {
2551                         writew(r + 0x0000, lut + (i * 0x08) + 0);
2552                         writew(g + 0x0000, lut + (i * 0x08) + 2);
2553                         writew(b + 0x0000, lut + (i * 0x08) + 4);
2554                 } else {
2555                         writew(r + 0x6000, lut + (i * 0x20) + 0);
2556                         writew(g + 0x6000, lut + (i * 0x20) + 2);
2557                         writew(b + 0x6000, lut + (i * 0x20) + 4);
2558                 }
2559         }
2560 }
2561
2562 static void
2563 nv50_crtc_disable(struct drm_crtc *crtc)
2564 {
2565         struct nv50_head *head = nv50_head(crtc);
2566         evo_sync(crtc->dev);
2567         if (head->image)
2568                 nouveau_bo_unpin(head->image);
2569         nouveau_bo_ref(NULL, &head->image);
2570 }
2571
2572 static int
2573 nv50_crtc_cursor_set(struct drm_crtc *crtc, struct drm_file *file_priv,
2574                      uint32_t handle, uint32_t width, uint32_t height)
2575 {
2576         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2577         struct drm_gem_object *gem = NULL;
2578         struct nouveau_bo *nvbo = NULL;
2579         int ret = 0;
2580
2581         if (handle) {
2582                 if (width != 64 || height != 64)
2583                         return -EINVAL;
2584
2585                 gem = drm_gem_object_lookup(file_priv, handle);
2586                 if (unlikely(!gem))
2587                         return -ENOENT;
2588                 nvbo = nouveau_gem_object(gem);
2589
2590                 ret = nouveau_bo_pin(nvbo, TTM_PL_FLAG_VRAM, true);
2591         }
2592
2593         if (ret == 0) {
2594                 if (nv_crtc->cursor.nvbo)
2595                         nouveau_bo_unpin(nv_crtc->cursor.nvbo);
2596                 nouveau_bo_ref(nvbo, &nv_crtc->cursor.nvbo);
2597         }
2598         drm_gem_object_unreference_unlocked(gem);
2599
2600         nv50_crtc_cursor_show_hide(nv_crtc, true, true);
2601         return ret;
2602 }
2603
2604 static int
2605 nv50_crtc_cursor_move(struct drm_crtc *crtc, int x, int y)
2606 {
2607         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2608         struct nv50_wndw *wndw = &nv50_head(crtc)->_curs->wndw;
2609         struct nv50_wndw_atom *asyw = &wndw->asy;
2610
2611         asyw->point.x = x;
2612         asyw->point.y = y;
2613         asyw->set.point = true;
2614         nv50_wndw_flush_set(wndw, 0, asyw);
2615
2616         nv_crtc->cursor_saved_x = x;
2617         nv_crtc->cursor_saved_y = y;
2618         return 0;
2619 }
2620
2621 static int
2622 nv50_crtc_gamma_set(struct drm_crtc *crtc, u16 *r, u16 *g, u16 *b,
2623                     uint32_t size)
2624 {
2625         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2626         u32 i;
2627
2628         for (i = 0; i < size; i++) {
2629                 nv_crtc->lut.r[i] = r[i];
2630                 nv_crtc->lut.g[i] = g[i];
2631                 nv_crtc->lut.b[i] = b[i];
2632         }
2633
2634         nv50_crtc_lut_load(crtc);
2635
2636         return 0;
2637 }
2638
2639 static void
2640 nv50_crtc_cursor_restore(struct nouveau_crtc *nv_crtc, int x, int y)
2641 {
2642         nv50_crtc_cursor_move(&nv_crtc->base, x, y);
2643
2644         nv50_crtc_cursor_show_hide(nv_crtc, true, true);
2645 }
2646
2647 static void
2648 nv50_crtc_destroy(struct drm_crtc *crtc)
2649 {
2650         struct nouveau_crtc *nv_crtc = nouveau_crtc(crtc);
2651         struct nv50_disp *disp = nv50_disp(crtc->dev);
2652         struct nv50_head *head = nv50_head(crtc);
2653
2654         nv50_dmac_destroy(&head->ovly.base, disp->disp);
2655         nv50_pioc_destroy(&head->oimm.base);
2656
2657         /*XXX: this shouldn't be necessary, but the core doesn't call
2658          *     disconnect() during the cleanup paths
2659          */
2660         if (head->image)
2661                 nouveau_bo_unpin(head->image);
2662         nouveau_bo_ref(NULL, &head->image);
2663
2664         /*XXX: ditto */
2665         if (nv_crtc->cursor.nvbo)
2666                 nouveau_bo_unpin(nv_crtc->cursor.nvbo);
2667         nouveau_bo_ref(NULL, &nv_crtc->cursor.nvbo);
2668
2669         nouveau_bo_unmap(nv_crtc->lut.nvbo);
2670         if (nv_crtc->lut.nvbo)
2671                 nouveau_bo_unpin(nv_crtc->lut.nvbo);
2672         nouveau_bo_ref(NULL, &nv_crtc->lut.nvbo);
2673
2674         drm_crtc_cleanup(crtc);
2675         kfree(crtc);
2676 }
2677
2678 static const struct drm_crtc_helper_funcs nv50_crtc_hfunc = {
2679         .dpms = nv50_crtc_dpms,
2680         .prepare = nv50_crtc_prepare,
2681         .commit = nv50_crtc_commit,
2682         .mode_fixup = nv50_crtc_mode_fixup,
2683         .mode_set = nv50_crtc_mode_set,
2684         .mode_set_base = nv50_crtc_mode_set_base,
2685         .mode_set_base_atomic = nv50_crtc_mode_set_base_atomic,
2686         .load_lut = nv50_crtc_lut_load,
2687         .disable = nv50_crtc_disable,
2688 };
2689
2690 static const struct drm_crtc_funcs nv50_crtc_func = {
2691         .cursor_set = nv50_crtc_cursor_set,
2692         .cursor_move = nv50_crtc_cursor_move,
2693         .gamma_set = nv50_crtc_gamma_set,
2694         .set_config = nouveau_crtc_set_config,
2695         .destroy = nv50_crtc_destroy,
2696         .page_flip = nouveau_crtc_page_flip,
2697 };
2698
2699 static int
2700 nv50_crtc_create(struct drm_device *dev, int index)
2701 {
2702         struct nouveau_drm *drm = nouveau_drm(dev);
2703         struct nvif_device *device = &drm->device;
2704         struct nv50_disp *disp = nv50_disp(dev);
2705         struct nv50_head *head;
2706         struct nv50_base *base;
2707         struct nv50_curs *curs;
2708         struct drm_crtc *crtc;
2709         int ret, i;
2710
2711         head = kzalloc(sizeof(*head), GFP_KERNEL);
2712         if (!head)
2713                 return -ENOMEM;
2714
2715         head->base.index = index;
2716         head->base.color_vibrance = 50;
2717         head->base.vibrant_hue = 0;
2718         head->base.cursor.set_pos = nv50_crtc_cursor_restore;
2719         for (i = 0; i < 256; i++) {
2720                 head->base.lut.r[i] = i << 8;
2721                 head->base.lut.g[i] = i << 8;
2722                 head->base.lut.b[i] = i << 8;
2723         }
2724
2725         ret = nv50_base_new(drm, head, &base);
2726         if (ret == 0)
2727                 ret = nv50_curs_new(drm, head, &curs);
2728         if (ret) {
2729                 kfree(head);
2730                 return ret;
2731         }
2732
2733         crtc = &head->base.base;
2734         head->_base = base;
2735         head->_curs = curs;
2736
2737         drm_crtc_init(dev, crtc, &nv50_crtc_func);
2738         drm_crtc_helper_add(crtc, &nv50_crtc_hfunc);
2739         drm_mode_crtc_set_gamma_size(crtc, 256);
2740
2741         ret = nouveau_bo_new(dev, 8192, 0x100, TTM_PL_FLAG_VRAM,
2742                              0, 0x0000, NULL, NULL, &head->base.lut.nvbo);
2743         if (!ret) {
2744                 ret = nouveau_bo_pin(head->base.lut.nvbo, TTM_PL_FLAG_VRAM, true);
2745                 if (!ret) {
2746                         ret = nouveau_bo_map(head->base.lut.nvbo);
2747                         if (ret)
2748                                 nouveau_bo_unpin(head->base.lut.nvbo);
2749                 }
2750                 if (ret)
2751                         nouveau_bo_ref(NULL, &head->base.lut.nvbo);
2752         }
2753
2754         if (ret)
2755                 goto out;
2756
2757         /* allocate overlay resources */
2758         ret = nv50_oimm_create(device, disp->disp, index, &head->oimm);
2759         if (ret)
2760                 goto out;
2761
2762         ret = nv50_ovly_create(device, disp->disp, index, disp->sync->bo.offset,
2763                                &head->ovly);
2764         if (ret)
2765                 goto out;
2766
2767 out:
2768         if (ret)
2769                 nv50_crtc_destroy(crtc);
2770         return ret;
2771 }
2772
2773 /******************************************************************************
2774  * Output path helpers
2775  *****************************************************************************/
2776 static int
2777 nv50_outp_atomic_check_view(struct drm_encoder *encoder,
2778                             struct drm_crtc_state *crtc_state,
2779                             struct drm_connector_state *conn_state,
2780                             struct drm_display_mode *native_mode)
2781 {
2782         struct drm_display_mode *adjusted_mode = &crtc_state->adjusted_mode;
2783         struct drm_display_mode *mode = &crtc_state->mode;
2784         struct drm_connector *connector = conn_state->connector;
2785         struct nouveau_conn_atom *asyc = nouveau_conn_atom(conn_state);
2786         struct nouveau_drm *drm = nouveau_drm(encoder->dev);
2787
2788         NV_ATOMIC(drm, "%s atomic_check\n", encoder->name);
2789         asyc->scaler.full = false;
2790         if (!native_mode)
2791                 return 0;
2792
2793         if (asyc->scaler.mode == DRM_MODE_SCALE_NONE) {
2794                 switch (connector->connector_type) {
2795                 case DRM_MODE_CONNECTOR_LVDS:
2796                 case DRM_MODE_CONNECTOR_eDP:
2797                         /* Force use of scaler for non-EDID modes. */
2798                         if (adjusted_mode->type & DRM_MODE_TYPE_DRIVER)
2799                                 break;
2800                         mode = native_mode;
2801                         asyc->scaler.full = true;
2802                         break;
2803                 default:
2804                         break;
2805                 }
2806         } else {
2807                 mode = native_mode;
2808         }
2809
2810         if (!drm_mode_equal(adjusted_mode, mode)) {
2811                 drm_mode_copy(adjusted_mode, mode);
2812                 crtc_state->mode_changed = true;
2813         }
2814
2815         return 0;
2816 }
2817
2818 static bool
2819 nv50_encoder_mode_fixup(struct drm_encoder *encoder,
2820                         const struct drm_display_mode *mode,
2821                         struct drm_display_mode *adjusted_mode)
2822 {
2823         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2824         struct nouveau_connector *nv_connector;
2825
2826         nv_connector = nouveau_encoder_connector_get(nv_encoder);
2827         if (nv_connector && nv_connector->native_mode) {
2828                 struct nouveau_conn_atom *asyc
2829                         = nouveau_conn_atom(nv_connector->base.state);
2830                 struct drm_crtc_state crtc_state = {
2831                         .mode = *mode,
2832                         .adjusted_mode = *adjusted_mode,
2833                 };
2834
2835                 nv50_outp_atomic_check_view(encoder, &crtc_state, &asyc->state,
2836                                             nv_connector->native_mode);
2837                 nv_connector->scaling_full = asyc->scaler.full;
2838                 drm_mode_copy(adjusted_mode, &crtc_state.adjusted_mode);
2839         }
2840
2841         return true;
2842 }
2843
2844 /******************************************************************************
2845  * DAC
2846  *****************************************************************************/
2847 static void
2848 nv50_dac_dpms(struct drm_encoder *encoder, int mode)
2849 {
2850         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2851         struct nv50_disp *disp = nv50_disp(encoder->dev);
2852         struct {
2853                 struct nv50_disp_mthd_v1 base;
2854                 struct nv50_disp_dac_pwr_v0 pwr;
2855         } args = {
2856                 .base.version = 1,
2857                 .base.method = NV50_DISP_MTHD_V1_DAC_PWR,
2858                 .base.hasht  = nv_encoder->dcb->hasht,
2859                 .base.hashm  = nv_encoder->dcb->hashm,
2860                 .pwr.state = 1,
2861                 .pwr.data  = 1,
2862                 .pwr.vsync = (mode != DRM_MODE_DPMS_SUSPEND &&
2863                               mode != DRM_MODE_DPMS_OFF),
2864                 .pwr.hsync = (mode != DRM_MODE_DPMS_STANDBY &&
2865                               mode != DRM_MODE_DPMS_OFF),
2866         };
2867
2868         nvif_mthd(disp->disp, 0, &args, sizeof(args));
2869 }
2870
2871 static void
2872 nv50_dac_disconnect(struct drm_encoder *encoder)
2873 {
2874         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2875         struct nv50_mast *mast = nv50_mast(encoder->dev);
2876         const int or = nv_encoder->or;
2877         u32 *push;
2878
2879         if (nv_encoder->crtc) {
2880                 nv50_crtc_prepare(nv_encoder->crtc);
2881
2882                 push = evo_wait(mast, 4);
2883                 if (push) {
2884                         if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
2885                                 evo_mthd(push, 0x0400 + (or * 0x080), 1);
2886                                 evo_data(push, 0x00000000);
2887                         } else {
2888                                 evo_mthd(push, 0x0180 + (or * 0x020), 1);
2889                                 evo_data(push, 0x00000000);
2890                         }
2891                         evo_kick(push, mast);
2892                 }
2893         }
2894
2895         nv_encoder->crtc = NULL;
2896 }
2897
2898 static void
2899 nv50_dac_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
2900                   struct drm_display_mode *adjusted_mode)
2901 {
2902         struct nv50_mast *mast = nv50_mast(encoder->dev);
2903         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2904         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
2905         u32 *push;
2906
2907         nv50_dac_dpms(encoder, DRM_MODE_DPMS_ON);
2908
2909         push = evo_wait(mast, 8);
2910         if (push) {
2911                 if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
2912                         u32 syncs = 0x00000000;
2913
2914                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
2915                                 syncs |= 0x00000001;
2916                         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
2917                                 syncs |= 0x00000002;
2918
2919                         evo_mthd(push, 0x0400 + (nv_encoder->or * 0x080), 2);
2920                         evo_data(push, 1 << nv_crtc->index);
2921                         evo_data(push, syncs);
2922                 } else {
2923                         u32 magic = 0x31ec6000 | (nv_crtc->index << 25);
2924                         u32 syncs = 0x00000001;
2925
2926                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
2927                                 syncs |= 0x00000008;
2928                         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
2929                                 syncs |= 0x00000010;
2930
2931                         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
2932                                 magic |= 0x00000001;
2933
2934                         evo_mthd(push, 0x0404 + (nv_crtc->index * 0x300), 2);
2935                         evo_data(push, syncs);
2936                         evo_data(push, magic);
2937                         evo_mthd(push, 0x0180 + (nv_encoder->or * 0x020), 1);
2938                         evo_data(push, 1 << nv_crtc->index);
2939                 }
2940
2941                 evo_kick(push, mast);
2942         }
2943
2944         nv_encoder->crtc = encoder->crtc;
2945 }
2946
2947 static enum drm_connector_status
2948 nv50_dac_detect(struct drm_encoder *encoder, struct drm_connector *connector)
2949 {
2950         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
2951         struct nv50_disp *disp = nv50_disp(encoder->dev);
2952         struct {
2953                 struct nv50_disp_mthd_v1 base;
2954                 struct nv50_disp_dac_load_v0 load;
2955         } args = {
2956                 .base.version = 1,
2957                 .base.method = NV50_DISP_MTHD_V1_DAC_LOAD,
2958                 .base.hasht  = nv_encoder->dcb->hasht,
2959                 .base.hashm  = nv_encoder->dcb->hashm,
2960         };
2961         int ret;
2962
2963         args.load.data = nouveau_drm(encoder->dev)->vbios.dactestval;
2964         if (args.load.data == 0)
2965                 args.load.data = 340;
2966
2967         ret = nvif_mthd(disp->disp, 0, &args, sizeof(args));
2968         if (ret || !args.load.load)
2969                 return connector_status_disconnected;
2970
2971         return connector_status_connected;
2972 }
2973
2974 static const struct drm_encoder_helper_funcs
2975 nv50_dac_help = {
2976         .dpms = nv50_dac_dpms,
2977         .mode_fixup = nv50_encoder_mode_fixup,
2978         .prepare = nv50_dac_disconnect,
2979         .mode_set = nv50_dac_mode_set,
2980         .disable = nv50_dac_disconnect,
2981         .get_crtc = nv50_display_crtc_get,
2982         .detect = nv50_dac_detect
2983 };
2984
2985 static void
2986 nv50_dac_destroy(struct drm_encoder *encoder)
2987 {
2988         drm_encoder_cleanup(encoder);
2989         kfree(encoder);
2990 }
2991
2992 static const struct drm_encoder_funcs
2993 nv50_dac_func = {
2994         .destroy = nv50_dac_destroy,
2995 };
2996
2997 static int
2998 nv50_dac_create(struct drm_connector *connector, struct dcb_output *dcbe)
2999 {
3000         struct nouveau_drm *drm = nouveau_drm(connector->dev);
3001         struct nvkm_i2c *i2c = nvxx_i2c(&drm->device);
3002         struct nvkm_i2c_bus *bus;
3003         struct nouveau_encoder *nv_encoder;
3004         struct drm_encoder *encoder;
3005         int type = DRM_MODE_ENCODER_DAC;
3006
3007         nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
3008         if (!nv_encoder)
3009                 return -ENOMEM;
3010         nv_encoder->dcb = dcbe;
3011         nv_encoder->or = ffs(dcbe->or) - 1;
3012
3013         bus = nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
3014         if (bus)
3015                 nv_encoder->i2c = &bus->i2c;
3016
3017         encoder = to_drm_encoder(nv_encoder);
3018         encoder->possible_crtcs = dcbe->heads;
3019         encoder->possible_clones = 0;
3020         drm_encoder_init(connector->dev, encoder, &nv50_dac_func, type,
3021                          "dac-%04x-%04x", dcbe->hasht, dcbe->hashm);
3022         drm_encoder_helper_add(encoder, &nv50_dac_help);
3023
3024         drm_mode_connector_attach_encoder(connector, encoder);
3025         return 0;
3026 }
3027
3028 /******************************************************************************
3029  * Audio
3030  *****************************************************************************/
3031 static void
3032 nv50_audio_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
3033 {
3034         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3035         struct nv50_disp *disp = nv50_disp(encoder->dev);
3036         struct {
3037                 struct nv50_disp_mthd_v1 base;
3038                 struct nv50_disp_sor_hda_eld_v0 eld;
3039         } args = {
3040                 .base.version = 1,
3041                 .base.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
3042                 .base.hasht   = nv_encoder->dcb->hasht,
3043                 .base.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
3044                                 (0x0100 << nv_crtc->index),
3045         };
3046
3047         nvif_mthd(disp->disp, 0, &args, sizeof(args));
3048 }
3049
3050 static void
3051 nv50_audio_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
3052 {
3053         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3054         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
3055         struct nouveau_connector *nv_connector;
3056         struct nv50_disp *disp = nv50_disp(encoder->dev);
3057         struct __packed {
3058                 struct {
3059                         struct nv50_disp_mthd_v1 mthd;
3060                         struct nv50_disp_sor_hda_eld_v0 eld;
3061                 } base;
3062                 u8 data[sizeof(nv_connector->base.eld)];
3063         } args = {
3064                 .base.mthd.version = 1,
3065                 .base.mthd.method  = NV50_DISP_MTHD_V1_SOR_HDA_ELD,
3066                 .base.mthd.hasht   = nv_encoder->dcb->hasht,
3067                 .base.mthd.hashm   = (0xf0ff & nv_encoder->dcb->hashm) |
3068                                      (0x0100 << nv_crtc->index),
3069         };
3070
3071         nv_connector = nouveau_encoder_connector_get(nv_encoder);
3072         if (!drm_detect_monitor_audio(nv_connector->edid))
3073                 return;
3074
3075         drm_edid_to_eld(&nv_connector->base, nv_connector->edid);
3076         memcpy(args.data, nv_connector->base.eld, sizeof(args.data));
3077
3078         nvif_mthd(disp->disp, 0, &args,
3079                   sizeof(args.base) + drm_eld_size(args.data));
3080 }
3081
3082 /******************************************************************************
3083  * HDMI
3084  *****************************************************************************/
3085 static void
3086 nv50_hdmi_disable(struct drm_encoder *encoder, struct nouveau_crtc *nv_crtc)
3087 {
3088         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3089         struct nv50_disp *disp = nv50_disp(encoder->dev);
3090         struct {
3091                 struct nv50_disp_mthd_v1 base;
3092                 struct nv50_disp_sor_hdmi_pwr_v0 pwr;
3093         } args = {
3094                 .base.version = 1,
3095                 .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
3096                 .base.hasht  = nv_encoder->dcb->hasht,
3097                 .base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
3098                                (0x0100 << nv_crtc->index),
3099         };
3100
3101         nvif_mthd(disp->disp, 0, &args, sizeof(args));
3102 }
3103
3104 static void
3105 nv50_hdmi_enable(struct drm_encoder *encoder, struct drm_display_mode *mode)
3106 {
3107         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3108         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
3109         struct nv50_disp *disp = nv50_disp(encoder->dev);
3110         struct {
3111                 struct nv50_disp_mthd_v1 base;
3112                 struct nv50_disp_sor_hdmi_pwr_v0 pwr;
3113         } args = {
3114                 .base.version = 1,
3115                 .base.method = NV50_DISP_MTHD_V1_SOR_HDMI_PWR,
3116                 .base.hasht  = nv_encoder->dcb->hasht,
3117                 .base.hashm  = (0xf0ff & nv_encoder->dcb->hashm) |
3118                                (0x0100 << nv_crtc->index),
3119                 .pwr.state = 1,
3120                 .pwr.rekey = 56, /* binary driver, and tegra, constant */
3121         };
3122         struct nouveau_connector *nv_connector;
3123         u32 max_ac_packet;
3124
3125         nv_connector = nouveau_encoder_connector_get(nv_encoder);
3126         if (!drm_detect_hdmi_monitor(nv_connector->edid))
3127                 return;
3128
3129         max_ac_packet  = mode->htotal - mode->hdisplay;
3130         max_ac_packet -= args.pwr.rekey;
3131         max_ac_packet -= 18; /* constant from tegra */
3132         args.pwr.max_ac_packet = max_ac_packet / 32;
3133
3134         nvif_mthd(disp->disp, 0, &args, sizeof(args));
3135         nv50_audio_enable(encoder, mode);
3136 }
3137
3138 /******************************************************************************
3139  * MST
3140  *****************************************************************************/
3141 struct nv50_mstm {
3142         struct nouveau_encoder *outp;
3143
3144         struct drm_dp_mst_topology_mgr mgr;
3145 };
3146
3147 static int
3148 nv50_mstm_enable(struct nv50_mstm *mstm, u8 dpcd, int state)
3149 {
3150         struct nouveau_encoder *outp = mstm->outp;
3151         struct {
3152                 struct nv50_disp_mthd_v1 base;
3153                 struct nv50_disp_sor_dp_mst_link_v0 mst;
3154         } args = {
3155                 .base.version = 1,
3156                 .base.method = NV50_DISP_MTHD_V1_SOR_DP_MST_LINK,
3157                 .base.hasht = outp->dcb->hasht,
3158                 .base.hashm = outp->dcb->hashm,
3159                 .mst.state = state,
3160         };
3161         struct nouveau_drm *drm = nouveau_drm(outp->base.base.dev);
3162         struct nvif_object *disp = &drm->display->disp;
3163         int ret;
3164
3165         if (dpcd >= 0x12) {
3166                 ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CTRL, &dpcd);
3167                 if (ret < 0)
3168                         return ret;
3169
3170                 dpcd &= ~DP_MST_EN;
3171                 if (state)
3172                         dpcd |= DP_MST_EN;
3173
3174                 ret = drm_dp_dpcd_writeb(mstm->mgr.aux, DP_MSTM_CTRL, dpcd);
3175                 if (ret < 0)
3176                         return ret;
3177         }
3178
3179         return nvif_mthd(disp, 0, &args, sizeof(args));
3180 }
3181
3182 int
3183 nv50_mstm_detect(struct nv50_mstm *mstm, u8 dpcd[8], int allow)
3184 {
3185         int ret, state = 0;
3186
3187         if (!mstm)
3188                 return 0;
3189
3190         if (dpcd[0] >= 0x12 && allow) {
3191                 ret = drm_dp_dpcd_readb(mstm->mgr.aux, DP_MSTM_CAP, &dpcd[1]);
3192                 if (ret < 0)
3193                         return ret;
3194
3195                 state = dpcd[1] & DP_MST_CAP;
3196         }
3197
3198         ret = nv50_mstm_enable(mstm, dpcd[0], state);
3199         if (ret)
3200                 return ret;
3201
3202         ret = drm_dp_mst_topology_mgr_set_mst(&mstm->mgr, state);
3203         if (ret)
3204                 return nv50_mstm_enable(mstm, dpcd[0], 0);
3205
3206         return mstm->mgr.mst_state;
3207 }
3208
3209 static void
3210 nv50_mstm_del(struct nv50_mstm **pmstm)
3211 {
3212         struct nv50_mstm *mstm = *pmstm;
3213         if (mstm) {
3214                 kfree(*pmstm);
3215                 *pmstm = NULL;
3216         }
3217 }
3218
3219 static int
3220 nv50_mstm_new(struct nouveau_encoder *outp, struct drm_dp_aux *aux, int aux_max,
3221               int conn_base_id, struct nv50_mstm **pmstm)
3222 {
3223         const int max_payloads = hweight8(outp->dcb->heads);
3224         struct drm_device *dev = outp->base.base.dev;
3225         struct nv50_mstm *mstm;
3226         int ret;
3227
3228         if (!(mstm = *pmstm = kzalloc(sizeof(*mstm), GFP_KERNEL)))
3229                 return -ENOMEM;
3230         mstm->outp = outp;
3231
3232         ret = drm_dp_mst_topology_mgr_init(&mstm->mgr, dev->dev, aux, aux_max,
3233                                            max_payloads, conn_base_id);
3234         if (ret)
3235                 return ret;
3236
3237         return 0;
3238 }
3239
3240 /******************************************************************************
3241  * SOR
3242  *****************************************************************************/
3243 static void
3244 nv50_sor_dpms(struct drm_encoder *encoder, int mode)
3245 {
3246         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3247         struct nv50_disp *disp = nv50_disp(encoder->dev);
3248         struct {
3249                 struct nv50_disp_mthd_v1 base;
3250                 struct nv50_disp_sor_pwr_v0 pwr;
3251         } args = {
3252                 .base.version = 1,
3253                 .base.method = NV50_DISP_MTHD_V1_SOR_PWR,
3254                 .base.hasht  = nv_encoder->dcb->hasht,
3255                 .base.hashm  = nv_encoder->dcb->hashm,
3256                 .pwr.state = mode == DRM_MODE_DPMS_ON,
3257         };
3258         struct {
3259                 struct nv50_disp_mthd_v1 base;
3260                 struct nv50_disp_sor_dp_pwr_v0 pwr;
3261         } link = {
3262                 .base.version = 1,
3263                 .base.method = NV50_DISP_MTHD_V1_SOR_DP_PWR,
3264                 .base.hasht  = nv_encoder->dcb->hasht,
3265                 .base.hashm  = nv_encoder->dcb->hashm,
3266                 .pwr.state = mode == DRM_MODE_DPMS_ON,
3267         };
3268         struct drm_device *dev = encoder->dev;
3269         struct drm_encoder *partner;
3270
3271         nv_encoder->last_dpms = mode;
3272
3273         list_for_each_entry(partner, &dev->mode_config.encoder_list, head) {
3274                 struct nouveau_encoder *nv_partner = nouveau_encoder(partner);
3275
3276                 if (partner->encoder_type != DRM_MODE_ENCODER_TMDS)
3277                         continue;
3278
3279                 if (nv_partner != nv_encoder &&
3280                     nv_partner->dcb->or == nv_encoder->dcb->or) {
3281                         if (nv_partner->last_dpms == DRM_MODE_DPMS_ON)
3282                                 return;
3283                         break;
3284                 }
3285         }
3286
3287         if (nv_encoder->dcb->type == DCB_OUTPUT_DP) {
3288                 args.pwr.state = 1;
3289                 nvif_mthd(disp->disp, 0, &args, sizeof(args));
3290                 nvif_mthd(disp->disp, 0, &link, sizeof(link));
3291         } else {
3292                 nvif_mthd(disp->disp, 0, &args, sizeof(args));
3293         }
3294 }
3295
3296 static void
3297 nv50_sor_ctrl(struct nouveau_encoder *nv_encoder, u32 mask, u32 data)
3298 {
3299         struct nv50_mast *mast = nv50_mast(nv_encoder->base.base.dev);
3300         u32 temp = (nv_encoder->ctrl & ~mask) | (data & mask), *push;
3301         if (temp != nv_encoder->ctrl && (push = evo_wait(mast, 2))) {
3302                 if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
3303                         evo_mthd(push, 0x0600 + (nv_encoder->or * 0x40), 1);
3304                         evo_data(push, (nv_encoder->ctrl = temp));
3305                 } else {
3306                         evo_mthd(push, 0x0200 + (nv_encoder->or * 0x20), 1);
3307                         evo_data(push, (nv_encoder->ctrl = temp));
3308                 }
3309                 evo_kick(push, mast);
3310         }
3311 }
3312
3313 static void
3314 nv50_sor_disconnect(struct drm_encoder *encoder)
3315 {
3316         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3317         struct nouveau_crtc *nv_crtc = nouveau_crtc(nv_encoder->crtc);
3318
3319         nv_encoder->last_dpms = DRM_MODE_DPMS_OFF;
3320         nv_encoder->crtc = NULL;
3321
3322         if (nv_crtc) {
3323                 nv50_crtc_prepare(&nv_crtc->base);
3324                 nv50_sor_ctrl(nv_encoder, 1 << nv_crtc->index, 0);
3325                 nv50_audio_disable(encoder, nv_crtc);
3326                 nv50_hdmi_disable(&nv_encoder->base.base, nv_crtc);
3327         }
3328 }
3329
3330 static void
3331 nv50_sor_mode_set(struct drm_encoder *encoder, struct drm_display_mode *umode,
3332                   struct drm_display_mode *mode)
3333 {
3334         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3335         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
3336         struct {
3337                 struct nv50_disp_mthd_v1 base;
3338                 struct nv50_disp_sor_lvds_script_v0 lvds;
3339         } lvds = {
3340                 .base.version = 1,
3341                 .base.method  = NV50_DISP_MTHD_V1_SOR_LVDS_SCRIPT,
3342                 .base.hasht   = nv_encoder->dcb->hasht,
3343                 .base.hashm   = nv_encoder->dcb->hashm,
3344         };
3345         struct nv50_disp *disp = nv50_disp(encoder->dev);
3346         struct nv50_mast *mast = nv50_mast(encoder->dev);
3347         struct drm_device *dev = encoder->dev;
3348         struct nouveau_drm *drm = nouveau_drm(dev);
3349         struct nouveau_connector *nv_connector;
3350         struct nvbios *bios = &drm->vbios;
3351         u32 mask, ctrl;
3352         u8 owner = 1 << nv_crtc->index;
3353         u8 proto = 0xf;
3354         u8 depth = 0x0;
3355
3356         nv_connector = nouveau_encoder_connector_get(nv_encoder);
3357         nv_encoder->crtc = encoder->crtc;
3358
3359         switch (nv_encoder->dcb->type) {
3360         case DCB_OUTPUT_TMDS:
3361                 if (nv_encoder->dcb->sorconf.link & 1) {
3362                         proto = 0x1;
3363                         /* Only enable dual-link if:
3364                          *  - Need to (i.e. rate > 165MHz)
3365                          *  - DCB says we can
3366                          *  - Not an HDMI monitor, since there's no dual-link
3367                          *    on HDMI.
3368                          */
3369                         if (mode->clock >= 165000 &&
3370                             nv_encoder->dcb->duallink_possible &&
3371                             !drm_detect_hdmi_monitor(nv_connector->edid))
3372                                 proto |= 0x4;
3373                 } else {
3374                         proto = 0x2;
3375                 }
3376
3377                 nv50_hdmi_enable(&nv_encoder->base.base, mode);
3378                 break;
3379         case DCB_OUTPUT_LVDS:
3380                 proto = 0x0;
3381
3382                 if (bios->fp_no_ddc) {
3383                         if (bios->fp.dual_link)
3384                                 lvds.lvds.script |= 0x0100;
3385                         if (bios->fp.if_is_24bit)
3386                                 lvds.lvds.script |= 0x0200;
3387                 } else {
3388                         if (nv_connector->type == DCB_CONNECTOR_LVDS_SPWG) {
3389                                 if (((u8 *)nv_connector->edid)[121] == 2)
3390                                         lvds.lvds.script |= 0x0100;
3391                         } else
3392                         if (mode->clock >= bios->fp.duallink_transition_clk) {
3393                                 lvds.lvds.script |= 0x0100;
3394                         }
3395
3396                         if (lvds.lvds.script & 0x0100) {
3397                                 if (bios->fp.strapless_is_24bit & 2)
3398                                         lvds.lvds.script |= 0x0200;
3399                         } else {
3400                                 if (bios->fp.strapless_is_24bit & 1)
3401                                         lvds.lvds.script |= 0x0200;
3402                         }
3403
3404                         if (nv_connector->base.display_info.bpc == 8)
3405                                 lvds.lvds.script |= 0x0200;
3406                 }
3407
3408                 nvif_mthd(disp->disp, 0, &lvds, sizeof(lvds));
3409                 break;
3410         case DCB_OUTPUT_DP:
3411                 if (nv_connector->base.display_info.bpc == 6)
3412                         depth = 0x2;
3413                 else
3414                 if (nv_connector->base.display_info.bpc == 8)
3415                         depth = 0x5;
3416                 else
3417                         depth = 0x6;
3418
3419                 if (nv_encoder->dcb->sorconf.link & 1)
3420                         proto = 0x8;
3421                 else
3422                         proto = 0x9;
3423
3424                 nv50_audio_enable(encoder, mode);
3425                 break;
3426         default:
3427                 BUG_ON(1);
3428                 break;
3429         }
3430
3431         nv50_sor_dpms(&nv_encoder->base.base, DRM_MODE_DPMS_ON);
3432
3433         if (nv50_vers(mast) >= GF110_DISP) {
3434                 u32 *push = evo_wait(mast, 3);
3435                 if (push) {
3436                         u32 magic = 0x31ec6000 | (nv_crtc->index << 25);
3437                         u32 syncs = 0x00000001;
3438
3439                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
3440                                 syncs |= 0x00000008;
3441                         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
3442                                 syncs |= 0x00000010;
3443
3444                         if (mode->flags & DRM_MODE_FLAG_INTERLACE)
3445                                 magic |= 0x00000001;
3446
3447                         evo_mthd(push, 0x0404 + (nv_crtc->index * 0x300), 2);
3448                         evo_data(push, syncs | (depth << 6));
3449                         evo_data(push, magic);
3450                         evo_kick(push, mast);
3451                 }
3452
3453                 ctrl = proto << 8;
3454                 mask = 0x00000f00;
3455         } else {
3456                 ctrl = (depth << 16) | (proto << 8);
3457                 if (mode->flags & DRM_MODE_FLAG_NHSYNC)
3458                         ctrl |= 0x00001000;
3459                 if (mode->flags & DRM_MODE_FLAG_NVSYNC)
3460                         ctrl |= 0x00002000;
3461                 mask = 0x000f3f00;
3462         }
3463
3464         nv50_sor_ctrl(nv_encoder, mask | owner, ctrl | owner);
3465 }
3466
3467 static const struct drm_encoder_helper_funcs
3468 nv50_sor_help = {
3469         .dpms = nv50_sor_dpms,
3470         .mode_fixup = nv50_encoder_mode_fixup,
3471         .prepare = nv50_sor_disconnect,
3472         .mode_set = nv50_sor_mode_set,
3473         .disable = nv50_sor_disconnect,
3474         .get_crtc = nv50_display_crtc_get,
3475 };
3476
3477 static void
3478 nv50_sor_destroy(struct drm_encoder *encoder)
3479 {
3480         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3481         nv50_mstm_del(&nv_encoder->dp.mstm);
3482         drm_encoder_cleanup(encoder);
3483         kfree(encoder);
3484 }
3485
3486 static const struct drm_encoder_funcs
3487 nv50_sor_func = {
3488         .destroy = nv50_sor_destroy,
3489 };
3490
3491 static int
3492 nv50_sor_create(struct drm_connector *connector, struct dcb_output *dcbe)
3493 {
3494         struct nouveau_connector *nv_connector = nouveau_connector(connector);
3495         struct nouveau_drm *drm = nouveau_drm(connector->dev);
3496         struct nvkm_i2c *i2c = nvxx_i2c(&drm->device);
3497         struct nouveau_encoder *nv_encoder;
3498         struct drm_encoder *encoder;
3499         int type, ret;
3500
3501         switch (dcbe->type) {
3502         case DCB_OUTPUT_LVDS: type = DRM_MODE_ENCODER_LVDS; break;
3503         case DCB_OUTPUT_TMDS:
3504         case DCB_OUTPUT_DP:
3505         default:
3506                 type = DRM_MODE_ENCODER_TMDS;
3507                 break;
3508         }
3509
3510         nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
3511         if (!nv_encoder)
3512                 return -ENOMEM;
3513         nv_encoder->dcb = dcbe;
3514         nv_encoder->or = ffs(dcbe->or) - 1;
3515         nv_encoder->last_dpms = DRM_MODE_DPMS_OFF;
3516
3517         encoder = to_drm_encoder(nv_encoder);
3518         encoder->possible_crtcs = dcbe->heads;
3519         encoder->possible_clones = 0;
3520         drm_encoder_init(connector->dev, encoder, &nv50_sor_func, type,
3521                          "sor-%04x-%04x", dcbe->hasht, dcbe->hashm);
3522         drm_encoder_helper_add(encoder, &nv50_sor_help);
3523
3524         drm_mode_connector_attach_encoder(connector, encoder);
3525
3526         if (dcbe->type == DCB_OUTPUT_DP) {
3527                 struct nvkm_i2c_aux *aux =
3528                         nvkm_i2c_aux_find(i2c, dcbe->i2c_index);
3529                 if (aux) {
3530                         nv_encoder->i2c = &aux->i2c;
3531                         nv_encoder->aux = aux;
3532                 }
3533
3534                 /*TODO: Use DP Info Table to check for support. */
3535                 if (nv50_disp(encoder->dev)->disp->oclass >= GF110_DISP) {
3536                         ret = nv50_mstm_new(nv_encoder, &nv_connector->aux, 16,
3537                                             nv_connector->base.base.id,
3538                                             &nv_encoder->dp.mstm);
3539                         if (ret)
3540                                 return ret;
3541                 }
3542         } else {
3543                 struct nvkm_i2c_bus *bus =
3544                         nvkm_i2c_bus_find(i2c, dcbe->i2c_index);
3545                 if (bus)
3546                         nv_encoder->i2c = &bus->i2c;
3547         }
3548
3549         return 0;
3550 }
3551
3552 /******************************************************************************
3553  * PIOR
3554  *****************************************************************************/
3555 static void
3556 nv50_pior_dpms(struct drm_encoder *encoder, int mode)
3557 {
3558         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3559         struct nv50_disp *disp = nv50_disp(encoder->dev);
3560         struct {
3561                 struct nv50_disp_mthd_v1 base;
3562                 struct nv50_disp_pior_pwr_v0 pwr;
3563         } args = {
3564                 .base.version = 1,
3565                 .base.method = NV50_DISP_MTHD_V1_PIOR_PWR,
3566                 .base.hasht  = nv_encoder->dcb->hasht,
3567                 .base.hashm  = nv_encoder->dcb->hashm,
3568                 .pwr.state = mode == DRM_MODE_DPMS_ON,
3569                 .pwr.type = nv_encoder->dcb->type,
3570         };
3571
3572         nvif_mthd(disp->disp, 0, &args, sizeof(args));
3573 }
3574
3575 static bool
3576 nv50_pior_mode_fixup(struct drm_encoder *encoder,
3577                      const struct drm_display_mode *mode,
3578                      struct drm_display_mode *adjusted_mode)
3579 {
3580         if (!nv50_encoder_mode_fixup(encoder, mode, adjusted_mode))
3581                 return false;
3582         adjusted_mode->clock *= 2;
3583         return true;
3584 }
3585
3586 static void
3587 nv50_pior_disconnect(struct drm_encoder *encoder)
3588 {
3589         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3590         struct nv50_mast *mast = nv50_mast(encoder->dev);
3591         const int or = nv_encoder->or;
3592         u32 *push;
3593
3594         if (nv_encoder->crtc) {
3595                 nv50_crtc_prepare(nv_encoder->crtc);
3596
3597                 push = evo_wait(mast, 4);
3598                 if (push) {
3599                         if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
3600                                 evo_mthd(push, 0x0700 + (or * 0x040), 1);
3601                                 evo_data(push, 0x00000000);
3602                         }
3603                         evo_kick(push, mast);
3604                 }
3605         }
3606
3607         nv_encoder->crtc = NULL;
3608 }
3609
3610 static void
3611 nv50_pior_mode_set(struct drm_encoder *encoder, struct drm_display_mode *mode,
3612                    struct drm_display_mode *adjusted_mode)
3613 {
3614         struct nv50_mast *mast = nv50_mast(encoder->dev);
3615         struct nouveau_encoder *nv_encoder = nouveau_encoder(encoder);
3616         struct nouveau_crtc *nv_crtc = nouveau_crtc(encoder->crtc);
3617         struct nouveau_connector *nv_connector;
3618         u8 owner = 1 << nv_crtc->index;
3619         u8 proto, depth;
3620         u32 *push;
3621
3622         nv_connector = nouveau_encoder_connector_get(nv_encoder);
3623         switch (nv_connector->base.display_info.bpc) {
3624         case 10: depth = 0x6; break;
3625         case  8: depth = 0x5; break;
3626         case  6: depth = 0x2; break;
3627         default: depth = 0x0; break;
3628         }
3629
3630         switch (nv_encoder->dcb->type) {
3631         case DCB_OUTPUT_TMDS:
3632         case DCB_OUTPUT_DP:
3633                 proto = 0x0;
3634                 break;
3635         default:
3636                 BUG_ON(1);
3637                 break;
3638         }
3639
3640         nv50_pior_dpms(encoder, DRM_MODE_DPMS_ON);
3641
3642         push = evo_wait(mast, 8);
3643         if (push) {
3644                 if (nv50_vers(mast) < GF110_DISP_CORE_CHANNEL_DMA) {
3645                         u32 ctrl = (depth << 16) | (proto << 8) | owner;
3646                         if (mode->flags & DRM_MODE_FLAG_NHSYNC)
3647                                 ctrl |= 0x00001000;
3648                         if (mode->flags & DRM_MODE_FLAG_NVSYNC)
3649                                 ctrl |= 0x00002000;
3650                         evo_mthd(push, 0x0700 + (nv_encoder->or * 0x040), 1);
3651                         evo_data(push, ctrl);
3652                 }
3653
3654                 evo_kick(push, mast);
3655         }
3656
3657         nv_encoder->crtc = encoder->crtc;
3658 }
3659
3660 static const struct drm_encoder_helper_funcs
3661 nv50_pior_help = {
3662         .dpms = nv50_pior_dpms,
3663         .mode_fixup = nv50_pior_mode_fixup,
3664         .prepare = nv50_pior_disconnect,
3665         .mode_set = nv50_pior_mode_set,
3666         .disable = nv50_pior_disconnect,
3667         .get_crtc = nv50_display_crtc_get,
3668 };
3669
3670 static void
3671 nv50_pior_destroy(struct drm_encoder *encoder)
3672 {
3673         drm_encoder_cleanup(encoder);
3674         kfree(encoder);
3675 }
3676
3677 static const struct drm_encoder_funcs
3678 nv50_pior_func = {
3679         .destroy = nv50_pior_destroy,
3680 };
3681
3682 static int
3683 nv50_pior_create(struct drm_connector *connector, struct dcb_output *dcbe)
3684 {
3685         struct nouveau_drm *drm = nouveau_drm(connector->dev);
3686         struct nvkm_i2c *i2c = nvxx_i2c(&drm->device);
3687         struct nvkm_i2c_bus *bus = NULL;
3688         struct nvkm_i2c_aux *aux = NULL;
3689         struct i2c_adapter *ddc;
3690         struct nouveau_encoder *nv_encoder;
3691         struct drm_encoder *encoder;
3692         int type;
3693
3694         switch (dcbe->type) {
3695         case DCB_OUTPUT_TMDS:
3696                 bus  = nvkm_i2c_bus_find(i2c, NVKM_I2C_BUS_EXT(dcbe->extdev));
3697                 ddc  = bus ? &bus->i2c : NULL;
3698                 type = DRM_MODE_ENCODER_TMDS;
3699                 break;
3700         case DCB_OUTPUT_DP:
3701                 aux  = nvkm_i2c_aux_find(i2c, NVKM_I2C_AUX_EXT(dcbe->extdev));
3702                 ddc  = aux ? &aux->i2c : NULL;
3703                 type = DRM_MODE_ENCODER_TMDS;
3704                 break;
3705         default:
3706                 return -ENODEV;
3707         }
3708
3709         nv_encoder = kzalloc(sizeof(*nv_encoder), GFP_KERNEL);
3710         if (!nv_encoder)
3711                 return -ENOMEM;
3712         nv_encoder->dcb = dcbe;
3713         nv_encoder->or = ffs(dcbe->or) - 1;
3714         nv_encoder->i2c = ddc;
3715         nv_encoder->aux = aux;
3716
3717         encoder = to_drm_encoder(nv_encoder);
3718         encoder->possible_crtcs = dcbe->heads;
3719         encoder->possible_clones = 0;
3720         drm_encoder_init(connector->dev, encoder, &nv50_pior_func, type,
3721                          "pior-%04x-%04x", dcbe->hasht, dcbe->hashm);
3722         drm_encoder_helper_add(encoder, &nv50_pior_help);
3723
3724         drm_mode_connector_attach_encoder(connector, encoder);
3725         return 0;
3726 }
3727
3728 /******************************************************************************
3729  * Framebuffer
3730  *****************************************************************************/
3731
3732 static void
3733 nv50_fb_dtor(struct drm_framebuffer *fb)
3734 {
3735 }
3736
3737 static int
3738 nv50_fb_ctor(struct drm_framebuffer *fb)
3739 {
3740         struct nouveau_framebuffer *nv_fb = nouveau_framebuffer(fb);
3741         struct nouveau_drm *drm = nouveau_drm(fb->dev);
3742         struct nouveau_bo *nvbo = nv_fb->nvbo;
3743         struct nv50_disp *disp = nv50_disp(fb->dev);
3744         u8 kind = nouveau_bo_tile_layout(nvbo) >> 8;
3745         u8 tile = nvbo->tile_mode;
3746         struct drm_crtc *crtc;
3747
3748         if (drm->device.info.chipset >= 0xc0)
3749                 tile >>= 4; /* yep.. */
3750
3751         switch (fb->depth) {
3752         case  8: nv_fb->r_format = 0x1e00; break;
3753         case 15: nv_fb->r_format = 0xe900; break;
3754         case 16: nv_fb->r_format = 0xe800; break;
3755         case 24:
3756         case 32: nv_fb->r_format = 0xcf00; break;
3757         case 30: nv_fb->r_format = 0xd100; break;
3758         default:
3759                  NV_ERROR(drm, "unknown depth %d\n", fb->depth);
3760                  return -EINVAL;
3761         }
3762
3763         if (disp->disp->oclass < G82_DISP) {
3764                 nv_fb->r_pitch   = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
3765                                             (fb->pitches[0] | 0x00100000);
3766                 nv_fb->r_format |= kind << 16;
3767         } else
3768         if (disp->disp->oclass < GF110_DISP) {
3769                 nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
3770                                            (fb->pitches[0] | 0x00100000);
3771         } else {
3772                 nv_fb->r_pitch  = kind ? (((fb->pitches[0] / 4) << 4) | tile) :
3773                                            (fb->pitches[0] | 0x01000000);
3774         }
3775         nv_fb->r_handle = 0xffff0000 | kind;
3776
3777         list_for_each_entry(crtc, &drm->dev->mode_config.crtc_list, head) {
3778                 struct nv50_head *head = nv50_head(crtc);
3779                 struct nv50_dmac_ctxdma *ctxdma;
3780
3781                 ctxdma = nv50_dmac_ctxdma_new(&head->_base->chan.base,
3782                                               nv_fb->r_handle, nv_fb);
3783                 if (IS_ERR(ctxdma))
3784                         return PTR_ERR(ctxdma);
3785         }
3786
3787         return 0;
3788 }
3789
3790 /******************************************************************************
3791  * Init
3792  *****************************************************************************/
3793
3794 void
3795 nv50_display_fini(struct drm_device *dev)
3796 {
3797         struct drm_plane *plane;
3798
3799         drm_for_each_plane(plane, dev) {
3800                 struct nv50_wndw *wndw = nv50_wndw(plane);
3801                 if (plane->funcs != &nv50_wndw)
3802                         continue;
3803                 nv50_wndw_fini(wndw);
3804         }
3805 }
3806
3807 int
3808 nv50_display_init(struct drm_device *dev)
3809 {
3810         struct nv50_disp *disp = nv50_disp(dev);
3811         struct drm_encoder *encoder;
3812         struct drm_plane *plane;
3813         struct drm_crtc *crtc;
3814         u32 *push;
3815
3816         push = evo_wait(nv50_mast(dev), 32);
3817         if (!push)
3818                 return -EBUSY;
3819
3820         list_for_each_entry(crtc, &dev->mode_config.crtc_list, head) {
3821                 struct nv50_wndw *wndw = &nv50_head(crtc)->_base->wndw;
3822
3823                 nv50_crtc_lut_load(crtc);
3824                 nouveau_bo_wr32(disp->sync, wndw->sema / 4, wndw->data);
3825         }
3826
3827         evo_mthd(push, 0x0088, 1);
3828         evo_data(push, nv50_mast(dev)->base.sync.handle);
3829         evo_kick(push, nv50_mast(dev));
3830
3831         list_for_each_entry(encoder, &dev->mode_config.encoder_list, head) {
3832                 if (encoder->encoder_type != DRM_MODE_ENCODER_DPMST) {
3833                         const struct drm_encoder_helper_funcs *help;
3834                         struct nouveau_encoder *nv_encoder;
3835
3836                         nv_encoder = nouveau_encoder(encoder);
3837                         if (nv_encoder->dcb->type == DCB_OUTPUT_DP)
3838                                 nv_encoder->dcb->type = DCB_OUTPUT_EOL;
3839
3840                         help = encoder->helper_private;
3841                         if (help && help->dpms)
3842                                 help->dpms(encoder, DRM_MODE_DPMS_ON);
3843
3844                         if (nv_encoder->dcb->type == DCB_OUTPUT_EOL)
3845                                 nv_encoder->dcb->type = DCB_OUTPUT_DP;
3846                 }
3847         }
3848
3849         drm_for_each_plane(plane, dev) {
3850                 struct nv50_wndw *wndw = nv50_wndw(plane);
3851                 if (plane->funcs != &nv50_wndw)
3852                         continue;
3853                 nv50_wndw_init(wndw);
3854         }
3855
3856         return 0;
3857 }
3858
3859 void
3860 nv50_display_destroy(struct drm_device *dev)
3861 {
3862         struct nv50_disp *disp = nv50_disp(dev);
3863
3864         nv50_dmac_destroy(&disp->mast.base, disp->disp);
3865
3866         nouveau_bo_unmap(disp->sync);
3867         if (disp->sync)
3868                 nouveau_bo_unpin(disp->sync);
3869         nouveau_bo_ref(NULL, &disp->sync);
3870
3871         nouveau_display(dev)->priv = NULL;
3872         kfree(disp);
3873 }
3874
3875 int
3876 nv50_display_create(struct drm_device *dev)
3877 {
3878         struct nvif_device *device = &nouveau_drm(dev)->device;
3879         struct nouveau_drm *drm = nouveau_drm(dev);
3880         struct dcb_table *dcb = &drm->vbios.dcb;
3881         struct drm_connector *connector, *tmp;
3882         struct nv50_disp *disp;
3883         struct dcb_output *dcbe;
3884         int crtcs, ret, i;
3885
3886         disp = kzalloc(sizeof(*disp), GFP_KERNEL);
3887         if (!disp)
3888                 return -ENOMEM;
3889
3890         nouveau_display(dev)->priv = disp;
3891         nouveau_display(dev)->dtor = nv50_display_destroy;
3892         nouveau_display(dev)->init = nv50_display_init;
3893         nouveau_display(dev)->fini = nv50_display_fini;
3894         nouveau_display(dev)->fb_ctor = nv50_fb_ctor;
3895         nouveau_display(dev)->fb_dtor = nv50_fb_dtor;
3896         disp->disp = &nouveau_display(dev)->disp;
3897
3898         /* small shared memory area we use for notifiers and semaphores */
3899         ret = nouveau_bo_new(dev, 4096, 0x1000, TTM_PL_FLAG_VRAM,
3900                              0, 0x0000, NULL, NULL, &disp->sync);
3901         if (!ret) {
3902                 ret = nouveau_bo_pin(disp->sync, TTM_PL_FLAG_VRAM, true);
3903                 if (!ret) {
3904                         ret = nouveau_bo_map(disp->sync);
3905                         if (ret)
3906                                 nouveau_bo_unpin(disp->sync);
3907                 }
3908                 if (ret)
3909                         nouveau_bo_ref(NULL, &disp->sync);
3910         }
3911
3912         if (ret)
3913                 goto out;
3914
3915         /* allocate master evo channel */
3916         ret = nv50_core_create(device, disp->disp, disp->sync->bo.offset,
3917                               &disp->mast);
3918         if (ret)
3919                 goto out;
3920
3921         /* create crtc objects to represent the hw heads */
3922         if (disp->disp->oclass >= GF110_DISP)
3923                 crtcs = nvif_rd32(&device->object, 0x022448);
3924         else
3925                 crtcs = 2;
3926
3927         for (i = 0; i < crtcs; i++) {
3928                 ret = nv50_crtc_create(dev, i);
3929                 if (ret)
3930                         goto out;
3931         }
3932
3933         /* create encoder/connector objects based on VBIOS DCB table */
3934         for (i = 0, dcbe = &dcb->entry[0]; i < dcb->entries; i++, dcbe++) {
3935                 connector = nouveau_connector_create(dev, dcbe->connector);
3936                 if (IS_ERR(connector))
3937                         continue;
3938
3939                 if (dcbe->location == DCB_LOC_ON_CHIP) {
3940                         switch (dcbe->type) {
3941                         case DCB_OUTPUT_TMDS:
3942                         case DCB_OUTPUT_LVDS:
3943                         case DCB_OUTPUT_DP:
3944                                 ret = nv50_sor_create(connector, dcbe);
3945                                 break;
3946                         case DCB_OUTPUT_ANALOG:
3947                                 ret = nv50_dac_create(connector, dcbe);
3948                                 break;
3949                         default:
3950                                 ret = -ENODEV;
3951                                 break;
3952                         }
3953                 } else {
3954                         ret = nv50_pior_create(connector, dcbe);
3955                 }
3956
3957                 if (ret) {
3958                         NV_WARN(drm, "failed to create encoder %d/%d/%d: %d\n",
3959                                      dcbe->location, dcbe->type,
3960                                      ffs(dcbe->or) - 1, ret);
3961                         ret = 0;
3962                 }
3963         }
3964
3965         /* cull any connectors we created that don't have an encoder */
3966         list_for_each_entry_safe(connector, tmp, &dev->mode_config.connector_list, head) {
3967                 if (connector->encoder_ids[0])
3968                         continue;
3969
3970                 NV_WARN(drm, "%s has no encoders, removing\n",
3971                         connector->name);
3972                 connector->funcs->destroy(connector);
3973         }
3974
3975 out:
3976         if (ret)
3977                 nv50_display_destroy(dev);
3978         return ret;
3979 }