From 0a96099691c8cd1ac0744ef30b6846869dc2b566 Mon Sep 17 00:00:00 2001 From: Ben Skeggs Date: Tue, 21 Jul 2020 11:34:07 +1000 Subject: [PATCH] drm/nouveau/kms/nv50-: implement proper push buffer control logic We had a, what was supposed to be temporary, hack in the KMS code where we'd completely drain an EVO/NVD channel's push buffer when wrapping to the start again, instead of treating it as a ring buffer. Let's fix that, finally. Signed-off-by: Ben Skeggs --- drivers/gpu/drm/nouveau/dispnv50/disp.c | 157 ++++++++++-------- drivers/gpu/drm/nouveau/dispnv50/disp.h | 4 + drivers/gpu/drm/nouveau/include/nvif/object.h | 13 ++ .../drm/nouveau/nvkm/engine/disp/channv50.h | 2 + .../drm/nouveau/nvkm/engine/disp/coregf119.c | 4 +- .../drm/nouveau/nvkm/engine/disp/coregp102.c | 2 +- .../drm/nouveau/nvkm/engine/disp/coregv100.c | 3 +- .../drm/nouveau/nvkm/engine/disp/corenv50.c | 4 +- .../drm/nouveau/nvkm/engine/disp/dmacgf119.c | 4 +- .../drm/nouveau/nvkm/engine/disp/dmacgp102.c | 2 +- .../drm/nouveau/nvkm/engine/disp/dmacgv100.c | 4 +- .../drm/nouveau/nvkm/engine/disp/dmacnv50.c | 4 +- 12 files changed, 125 insertions(+), 78 deletions(-) diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.c b/drivers/gpu/drm/nouveau/dispnv50/disp.c index 368a2a97a900..0b3751c0d2a4 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.c +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.c @@ -41,6 +41,8 @@ #include #include +#include + #include #include #include @@ -48,6 +50,8 @@ #include #include +#include + #include "nouveau_drv.h" #include "nouveau_dma.h" #include "nouveau_gem.h" @@ -120,21 +124,93 @@ static void nv50_dmac_kick(struct nvif_push *push) { struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push); - evo_kick(push->cur, dmac); - push->bgn = push->cur = push->end; + + dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr; + if (dmac->put != dmac->cur) { + /* Push buffer fetches are not coherent with BAR1, we need to ensure + * writes have been flushed right through to VRAM before writing PUT. + */ + if (dmac->push->mem.type & NVIF_MEM_VRAM) { + struct nvif_device *device = dmac->base.device; + nvif_wr32(&device->object, 0x070000, 0x00000001); + nvif_msec(device, 2000, + if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002)) + break; + ); + } + + NVIF_WV32(&dmac->base.user, NV507C, PUT, PTR, dmac->cur); + dmac->put = dmac->cur; + } + + push->bgn = push->cur; +} + +static int +nv50_dmac_free(struct nv50_dmac *dmac) +{ + u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR); + if (get > dmac->cur) /* NVIDIA stay 5 away from GET, do the same. */ + return get - dmac->cur - 5; + return dmac->max - dmac->cur; +} + +static int +nv50_dmac_wind(struct nv50_dmac *dmac) +{ + /* Wait for GET to depart from the beginning of the push buffer to + * prevent writing PUT == GET, which would be ignored by HW. + */ + u32 get = NVIF_RV32(&dmac->base.user, NV507C, GET, PTR); + if (get == 0) { + /* Corner-case, HW idle, but non-committed work pending. */ + if (dmac->put == 0) + nv50_dmac_kick(dmac->push); + + if (nvif_msec(dmac->base.device, 2000, + if (NVIF_TV32(&dmac->base.user, NV507C, GET, PTR, >, 0)) + break; + ) < 0) + return -ETIMEDOUT; + } + + PUSH_RSVD(dmac->push, PUSH_JUMP(dmac->push, 0)); + dmac->cur = 0; + return 0; } static int nv50_dmac_wait(struct nvif_push *push, u32 size) { struct nv50_dmac *dmac = container_of(push, typeof(*dmac), _push); - u32 *ptr = evo_wait(dmac, size); - if (!ptr) + int free; + + if (WARN_ON(size > dmac->max)) + return -EINVAL; + + dmac->cur = push->cur - (u32 *)dmac->_push.mem.object.map.ptr; + if (dmac->cur + size >= dmac->max) { + int ret = nv50_dmac_wind(dmac); + if (ret) + return ret; + + push->cur = dmac->_push.mem.object.map.ptr; + push->cur = push->cur + dmac->cur; + nv50_dmac_kick(push); + } + + if (nvif_msec(dmac->base.device, 2000, + if ((free = nv50_dmac_free(dmac)) >= size) + break; + ) < 0) { + WARN_ON(1); return -ETIMEDOUT; + } - push->bgn = ptr; - push->cur = ptr; - push->end = ptr + size; + push->bgn = dmac->_push.mem.object.map.ptr; + push->bgn = push->bgn + dmac->cur; + push->cur = push->bgn; + push->end = push->cur + free; return 0; } @@ -171,6 +247,10 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, dmac->_push.wait = nv50_dmac_wait; dmac->_push.kick = nv50_dmac_kick; dmac->push = &dmac->_push; + dmac->push->bgn = dmac->_push.mem.object.map.ptr; + dmac->push->cur = dmac->push->bgn; + dmac->push->end = dmac->push->bgn; + dmac->max = 0x1000/4 - 1; args->pushbuf = nvif_handle(&dmac->_push.mem.object); @@ -209,69 +289,6 @@ nv50_dmac_create(struct nvif_device *device, struct nvif_object *disp, return ret; } -/****************************************************************************** - * EVO channel helpers - *****************************************************************************/ -static void -evo_flush(struct nv50_dmac *dmac) -{ - /* Push buffer fetches are not coherent with BAR1, we need to ensure - * writes have been flushed right through to VRAM before writing PUT. - */ - if (dmac->push->mem.type & NVIF_MEM_VRAM) { - struct nvif_device *device = dmac->base.device; - nvif_wr32(&device->object, 0x070000, 0x00000001); - nvif_msec(device, 2000, - if (!(nvif_rd32(&device->object, 0x070000) & 0x00000002)) - break; - ); - } -} - -u32 * -evo_wait(struct nv50_dmac *evoc, int nr) -{ - struct nv50_dmac *dmac = evoc; - struct nvif_device *device = dmac->base.device; - u32 put; - - if (dmac->push->cur != dmac->push->bgn) - PUSH_KICK(dmac->push); - - put = nvif_rd32(&dmac->base.user, 0x0000) / 4; - - mutex_lock(&dmac->lock); - if (put + nr >= (PAGE_SIZE / 4) - 8) { - dmac->ptr[put] = 0x20000000; - evo_flush(dmac); - - nvif_wr32(&dmac->base.user, 0x0000, 0x00000000); - if (nvif_msec(device, 2000, - if (!nvif_rd32(&dmac->base.user, 0x0004)) - break; - ) < 0) { - mutex_unlock(&dmac->lock); - pr_err("nouveau: evo channel stalled\n"); - return NULL; - } - - put = 0; - } - - return dmac->ptr + put; -} - -void -evo_kick(u32 *push, struct nv50_dmac *evoc) -{ - struct nv50_dmac *dmac = evoc; - - evo_flush(dmac); - - nvif_wr32(&dmac->base.user, 0x0000, (push - dmac->ptr) << 2); - mutex_unlock(&dmac->lock); -} - /****************************************************************************** * Output path helpers *****************************************************************************/ diff --git a/drivers/gpu/drm/nouveau/dispnv50/disp.h b/drivers/gpu/drm/nouveau/dispnv50/disp.h index fbf573a01f58..92bddc083617 100644 --- a/drivers/gpu/drm/nouveau/dispnv50/disp.h +++ b/drivers/gpu/drm/nouveau/dispnv50/disp.h @@ -73,6 +73,10 @@ struct nv50_dmac { * grabbed by evo_wait (if the pushbuf reservation is successful) and * dropped again by evo_kick. */ struct mutex lock; + + u32 cur; + u32 put; + u32 max; }; struct nv50_outp_atom { diff --git a/drivers/gpu/drm/nouveau/include/nvif/object.h b/drivers/gpu/drm/nouveau/include/nvif/object.h index a39d2bf7064b..1e4c158d20fa 100644 --- a/drivers/gpu/drm/nouveau/include/nvif/object.h +++ b/drivers/gpu/drm/nouveau/include/nvif/object.h @@ -116,6 +116,19 @@ struct nvif_mclass { _cid; \ }) +#define NVIF_RD32_(p,o,dr) nvif_rd32((p), (o) + (dr)) +#define NVIF_WR32_(p,o,dr,f) nvif_wr32((p), (o) + (dr), (f)) +#define NVIF_RD32(p,A...) DRF_RD(NVIF_RD32_, (p), 0, ##A) +#define NVIF_RV32(p,A...) DRF_RV(NVIF_RD32_, (p), 0, ##A) +#define NVIF_TV32(p,A...) DRF_TV(NVIF_RD32_, (p), 0, ##A) +#define NVIF_TD32(p,A...) DRF_TD(NVIF_RD32_, (p), 0, ##A) +#define NVIF_WR32(p,A...) DRF_WR( NVIF_WR32_, (p), 0, ##A) +#define NVIF_WV32(p,A...) DRF_WV( NVIF_WR32_, (p), 0, ##A) +#define NVIF_WD32(p,A...) DRF_WD( NVIF_WR32_, (p), 0, ##A) +#define NVIF_MR32(p,A...) DRF_MR(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A) +#define NVIF_MV32(p,A...) DRF_MV(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A) +#define NVIF_MD32(p,A...) DRF_MD(NVIF_RD32_, NVIF_WR32_, u32, (p), 0, ##A) + /*XXX*/ #include #define nvxx_object(a) ({ \ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h index e55054b7329f..9cf2cfe2010c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/channv50.h @@ -21,6 +21,8 @@ struct nv50_disp_chan { struct nvkm_memory *memory; u64 push; + + u32 suspend_put; }; struct nv50_disp_chan_func { diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c index d162b9cf4eac..689e3cdd959a 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregf119.c @@ -182,6 +182,8 @@ gf119_disp_core_fini(struct nv50_disp_chan *chan) nvkm_error(subdev, "core fini: %08x\n", nvkm_rd32(device, 0x610490)); } + + chan->suspend_put = nvkm_rd32(device, 0x640000); } static int @@ -195,7 +197,7 @@ gf119_disp_core_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610498, 0x00010000); nvkm_wr32(device, 0x61049c, 0x00000001); nvkm_mask(device, 0x610490, 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000, 0x00000000); + nvkm_wr32(device, 0x640000, chan->suspend_put); nvkm_wr32(device, 0x610490, 0x01000013); /* wait for it to go inactive */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c index 5b7f993c73c7..1b435beef3bf 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregp102.c @@ -36,7 +36,7 @@ gp102_disp_core_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x611498, 0x00010000); nvkm_wr32(device, 0x61149c, 0x00000001); nvkm_mask(device, 0x610490, 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000, 0x00000000); + nvkm_wr32(device, 0x640000, chan->suspend_put); nvkm_wr32(device, 0x610490, 0x01000013); /* wait for it to go inactive */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c index 4592d0e69fec..e20a48f201f6 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/coregv100.c @@ -167,6 +167,7 @@ gv100_disp_core_fini(struct nv50_disp_chan *chan) nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000000); gv100_disp_core_idle(chan); nvkm_mask(device, 0x6104e0, 0x00000002, 0x00000000); + chan->suspend_put = nvkm_rd32(device, 0x680000); } static int @@ -181,7 +182,7 @@ gv100_disp_core_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610b2c, 0x00000040); nvkm_mask(device, 0x6104e0, 0x00000010, 0x00000010); - nvkm_wr32(device, 0x680000, 0x00000000); + nvkm_wr32(device, 0x680000, chan->suspend_put); nvkm_wr32(device, 0x6104e0, 0x00000013); return gv100_disp_core_idle(chan); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c index 55db9a22b4be..660310b27f9c 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/corenv50.c @@ -179,6 +179,8 @@ nv50_disp_core_fini(struct nv50_disp_chan *chan) nvkm_error(subdev, "core fini: %08x\n", nvkm_rd32(device, 0x610200)); } + + chan->suspend_put = nvkm_rd32(device, 0x640000); } static int @@ -198,7 +200,7 @@ nv50_disp_core_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610208, 0x00010000); nvkm_wr32(device, 0x61020c, 0x00000000); nvkm_mask(device, 0x610200, 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000, 0x00000000); + nvkm_wr32(device, 0x640000, chan->suspend_put); nvkm_wr32(device, 0x610200, 0x01000013); /* wait for it to go inactive */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c index edf7dd0d931d..76425e8586da 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgf119.c @@ -53,6 +53,8 @@ gf119_disp_dmac_fini(struct nv50_disp_chan *chan) nvkm_error(subdev, "ch %d fini: %08x\n", user, nvkm_rd32(device, 0x610490 + (ctrl * 0x10))); } + + chan->suspend_put = nvkm_rd32(device, 0x640000 + (ctrl * 0x1000)); } static int @@ -68,7 +70,7 @@ gf119_disp_dmac_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610498 + (ctrl * 0x0010), 0x00010000); nvkm_wr32(device, 0x61049c + (ctrl * 0x0010), 0x00000001); nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000); + nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put); nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013); /* wait for it to go inactive */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c index f21a433199aa..da258df268d7 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgp102.c @@ -38,7 +38,7 @@ gp102_disp_dmac_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x611498 + (ctrl * 0x0010), 0x00010000); nvkm_wr32(device, 0x61149c + (ctrl * 0x0010), 0x00000001); nvkm_mask(device, 0x610490 + (ctrl * 0x0010), 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000); + nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put); nvkm_wr32(device, 0x610490 + (ctrl * 0x0010), 0x00000013); /* wait for it to go inactive */ diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c index eac0e42da354..fdb624ac6b87 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacgv100.c @@ -50,10 +50,12 @@ void gv100_disp_dmac_fini(struct nv50_disp_chan *chan) { struct nvkm_device *device = chan->disp->base.engine.subdev.device; + const u32 uoff = (chan->chid.ctrl - 1) * 0x1000; const u32 coff = chan->chid.ctrl * 0x04; nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000000); gv100_disp_dmac_idle(chan); nvkm_mask(device, 0x6104e0 + coff, 0x00000002, 0x00000000); + chan->suspend_put = nvkm_rd32(device, 0x690000 + uoff); } int @@ -71,7 +73,7 @@ gv100_disp_dmac_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610b2c + poff, 0x00000040); nvkm_mask(device, 0x6104e0 + coff, 0x00000010, 0x00000010); - nvkm_wr32(device, 0x690000 + uoff, 0x00000000); + nvkm_wr32(device, 0x690000 + uoff, chan->suspend_put); nvkm_wr32(device, 0x6104e0 + coff, 0x00000013); return gv100_disp_dmac_idle(chan); } diff --git a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c index 9e8a9d7a9b68..d0a7da96d62b 100644 --- a/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c +++ b/drivers/gpu/drm/nouveau/nvkm/engine/disp/dmacnv50.c @@ -94,6 +94,8 @@ nv50_disp_dmac_fini(struct nv50_disp_chan *chan) nvkm_error(subdev, "ch %d fini timeout, %08x\n", user, nvkm_rd32(device, 0x610200 + (ctrl * 0x10))); } + + chan->suspend_put = nvkm_rd32(device, 0x640000 + (ctrl * 0x1000)); } static int @@ -109,7 +111,7 @@ nv50_disp_dmac_init(struct nv50_disp_chan *chan) nvkm_wr32(device, 0x610208 + (ctrl * 0x0010), 0x00010000); nvkm_wr32(device, 0x61020c + (ctrl * 0x0010), ctrl); nvkm_mask(device, 0x610200 + (ctrl * 0x0010), 0x00000010, 0x00000010); - nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), 0x00000000); + nvkm_wr32(device, 0x640000 + (ctrl * 0x1000), chan->suspend_put); nvkm_wr32(device, 0x610200 + (ctrl * 0x0010), 0x00000013); /* wait for it to go inactive */ -- 2.20.1