drivers/gpu/drm/nouveau/nouveau_fence.c

   1 /*
   2  * Copyright (C) 2007 Ben Skeggs.
   3  * All Rights Reserved.
   4  *
   5  * Permission is hereby granted, free of charge, to any person obtaining
   6  * a copy of this software and associated documentation files (the
   7  * "Software"), to deal in the Software without restriction, including
   8  * without limitation the rights to use, copy, modify, merge, publish,
   9  * distribute, sublicense, and/or sell copies of the Software, and to
  10  * permit persons to whom the Software is furnished to do so, subject to
  11  * the following conditions:
  12  *
  13  * The above copyright notice and this permission notice (including the
  14  * next paragraph) shall be included in all copies or substantial
  15  * portions of the Software.
  16  *
  17  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20  * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21  * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22  * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23  * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24  *
  25  */
  26
  27 #include "drmP.h"
  28 #include "drm.h"
  29
  30 #include "nouveau_drv.h"
  31 #include "nouveau_ramht.h"
  32 #include "nouveau_dma.h"
  33
  34 #define USE_REFCNT(dev) (nouveau_private(dev)->chipset >= 0x10)
  35 #define USE_SEMA(dev) (nouveau_private(dev)->chipset >= 0x17)
  36
  37 struct nouveau_fence {
  38         struct nouveau_channel *channel;
  39         struct kref refcount;
  40         struct list_head entry;
  41
  42         uint32_t sequence;
  43         bool signalled;
  44
  45         void (*work)(void *priv, bool signalled);
  46         void *priv;
  47 };
  48
  49 struct nouveau_semaphore {
  50         struct kref ref;
  51         struct drm_device *dev;
  52         struct drm_mm_node *mem;
  53 };
  54
  55 static inline struct nouveau_fence *
  56 nouveau_fence(void *sync_obj)
  57 {
  58         return (struct nouveau_fence *)sync_obj;
  59 }
  60
  61 static void
  62 nouveau_fence_del(struct kref *ref)
  63 {
  64         struct nouveau_fence *fence =
  65                 container_of(ref, struct nouveau_fence, refcount);
  66
  67         kfree(fence);
  68 }
  69
  70 void
  71 nouveau_fence_update(struct nouveau_channel *chan)
  72 {
  73         struct drm_device *dev = chan->dev;
  74         struct nouveau_fence *tmp, *fence;
  75         uint32_t sequence;
  76
  77         spin_lock(&chan->fence.lock);
  78
  79         if (USE_REFCNT(dev))
  80                 sequence = nvchan_rd32(chan, 0x48);
  81         else
  82                 sequence = atomic_read(&chan->fence.last_sequence_irq);
  83
  84         if (chan->fence.sequence_ack == sequence)
  85                 goto out;
  86         chan->fence.sequence_ack = sequence;
  87
  88         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
  89                 sequence = fence->sequence;
  90                 fence->signalled = true;
  91                 list_del(&fence->entry);
  92
  93                 if (unlikely(fence->work))
  94                         fence->work(fence->priv, true);
  95
  96                 kref_put(&fence->refcount, nouveau_fence_del);
  97
  98                 if (sequence == chan->fence.sequence_ack)
  99                         break;
 100         }
 101 out:
 102         spin_unlock(&chan->fence.lock);
 103 }
 104
 105 int
 106 nouveau_fence_new(struct nouveau_channel *chan, struct nouveau_fence **pfence,
 107                   bool emit)
 108 {
 109         struct nouveau_fence *fence;
 110         int ret = 0;
 111
 112         fence = kzalloc(sizeof(*fence), GFP_KERNEL);
 113         if (!fence)
 114                 return -ENOMEM;
 115         kref_init(&fence->refcount);
 116         fence->channel = chan;
 117
 118         if (emit)
 119                 ret = nouveau_fence_emit(fence);
 120
 121         if (ret)
 122                 nouveau_fence_unref((void *)&fence);
 123         *pfence = fence;
 124         return ret;
 125 }
 126
 127 struct nouveau_channel *
 128 nouveau_fence_channel(struct nouveau_fence *fence)
 129 {
 130         return fence ? fence->channel : NULL;
 131 }
 132
 133 int
 134 nouveau_fence_emit(struct nouveau_fence *fence)
 135 {
 136         struct nouveau_channel *chan = fence->channel;
 137         struct drm_device *dev = chan->dev;
 138         int ret;
 139
 140         ret = RING_SPACE(chan, 2);
 141         if (ret)
 142                 return ret;
 143
 144         if (unlikely(chan->fence.sequence == chan->fence.sequence_ack - 1)) {
 145                 nouveau_fence_update(chan);
 146
 147                 BUG_ON(chan->fence.sequence ==
 148                        chan->fence.sequence_ack - 1);
 149         }
 150
 151         fence->sequence = ++chan->fence.sequence;
 152
 153         kref_get(&fence->refcount);
 154         spin_lock(&chan->fence.lock);
 155         list_add_tail(&fence->entry, &chan->fence.pending);
 156         spin_unlock(&chan->fence.lock);
 157
 158         BEGIN_RING(chan, NvSubSw, USE_REFCNT(dev) ? 0x0050 : 0x0150, 1);
 159         OUT_RING(chan, fence->sequence);
 160         FIRE_RING(chan);
 161
 162         return 0;
 163 }
 164
 165 void
 166 nouveau_fence_work(struct nouveau_fence *fence,
 167                    void (*work)(void *priv, bool signalled),
 168                    void *priv)
 169 {
 170         BUG_ON(fence->work);
 171
 172         spin_lock(&fence->channel->fence.lock);
 173
 174         if (fence->signalled) {
 175                 work(priv, true);
 176         } else {
 177                 fence->work = work;
 178                 fence->priv = priv;
 179         }
 180
 181         spin_unlock(&fence->channel->fence.lock);
 182 }
 183
 184 void
 185 nouveau_fence_unref(void **sync_obj)
 186 {
 187         struct nouveau_fence *fence = nouveau_fence(*sync_obj);
 188
 189         if (fence)
 190                 kref_put(&fence->refcount, nouveau_fence_del);
 191         *sync_obj = NULL;
 192 }
 193
 194 void *
 195 nouveau_fence_ref(void *sync_obj)
 196 {
 197         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 198
 199         kref_get(&fence->refcount);
 200         return sync_obj;
 201 }
 202
 203 bool
 204 nouveau_fence_signalled(void *sync_obj, void *sync_arg)
 205 {
 206         struct nouveau_fence *fence = nouveau_fence(sync_obj);
 207         struct nouveau_channel *chan = fence->channel;
 208
 209         if (fence->signalled)
 210                 return true;
 211
 212         nouveau_fence_update(chan);
 213         return fence->signalled;
 214 }
 215
 216 int
 217 nouveau_fence_wait(void *sync_obj, void *sync_arg, bool lazy, bool intr)
 218 {
 219         unsigned long timeout = jiffies + (3 * DRM_HZ);
 220         int ret = 0;
 221
 222         while (1) {
 223                 if (nouveau_fence_signalled(sync_obj, sync_arg))
 224                         break;
 225
 226                 if (time_after_eq(jiffies, timeout)) {
 227                         ret = -EBUSY;
 228                         break;
 229                 }
 230
 231                 __set_current_state(intr ? TASK_INTERRUPTIBLE
 232                         : TASK_UNINTERRUPTIBLE);
 233                 if (lazy)
 234                         schedule_timeout(1);
 235
 236                 if (intr && signal_pending(current)) {
 237                         ret = -ERESTARTSYS;
 238                         break;
 239                 }
 240         }
 241
 242         __set_current_state(TASK_RUNNING);
 243
 244         return ret;
 245 }
 246
 247 static struct nouveau_semaphore *
 248 alloc_semaphore(struct drm_device *dev)
 249 {
 250         struct drm_nouveau_private *dev_priv = dev->dev_private;
 251         struct nouveau_semaphore *sema;
 252         int ret;
 253
 254         if (!USE_SEMA(dev))
 255                 return NULL;
 256
 257         sema = kmalloc(sizeof(*sema), GFP_KERNEL);
 258         if (!sema)
 259                 goto fail;
 260
 261         ret = drm_mm_pre_get(&dev_priv->fence.heap);
 262         if (ret)
 263                 goto fail;
 264
 265         spin_lock(&dev_priv->fence.lock);
 266         sema->mem = drm_mm_search_free(&dev_priv->fence.heap, 4, 0, 0);
 267         if (sema->mem)
 268                 sema->mem = drm_mm_get_block_atomic(sema->mem, 4, 0);
 269         spin_unlock(&dev_priv->fence.lock);
 270
 271         if (!sema->mem)
 272                 goto fail;
 273
 274         kref_init(&sema->ref);
 275         sema->dev = dev;
 276         nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 0);
 277
 278         return sema;
 279 fail:
 280         kfree(sema);
 281         return NULL;
 282 }
 283
 284 static void
 285 free_semaphore(struct kref *ref)
 286 {
 287         struct nouveau_semaphore *sema =
 288                 container_of(ref, struct nouveau_semaphore, ref);
 289         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 290
 291         spin_lock(&dev_priv->fence.lock);
 292         drm_mm_put_block(sema->mem);
 293         spin_unlock(&dev_priv->fence.lock);
 294
 295         kfree(sema);
 296 }
 297
 298 static void
 299 semaphore_work(void *priv, bool signalled)
 300 {
 301         struct nouveau_semaphore *sema = priv;
 302         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 303
 304         if (unlikely(!signalled))
 305                 nouveau_bo_wr32(dev_priv->fence.bo, sema->mem->start / 4, 1);
 306
 307         kref_put(&sema->ref, free_semaphore);
 308 }
 309
 310 static int
 311 emit_semaphore(struct nouveau_channel *chan, int method,
 312                struct nouveau_semaphore *sema)
 313 {
 314         struct drm_nouveau_private *dev_priv = sema->dev->dev_private;
 315         struct nouveau_fence *fence;
 316         bool smart = (dev_priv->card_type >= NV_50);
 317         int ret;
 318
 319         ret = RING_SPACE(chan, smart ? 8 : 4);
 320         if (ret)
 321                 return ret;
 322
 323         if (smart) {
 324                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 325                 OUT_RING(chan, NvSema);
 326         }
 327         BEGIN_RING(chan, NvSubSw, NV_SW_SEMAPHORE_OFFSET, 1);
 328         OUT_RING(chan, sema->mem->start);
 329
 330         if (smart && method == NV_SW_SEMAPHORE_ACQUIRE) {
 331                 /*
 332                  * NV50 tries to be too smart and context-switch
 333                  * between semaphores instead of doing a "first come,
 334                  * first served" strategy like previous cards
 335                  * do.
 336                  *
 337                  * That's bad because the ACQUIRE latency can get as
 338                  * large as the PFIFO context time slice in the
 339                  * typical DRI2 case where you have several
 340                  * outstanding semaphores at the same moment.
 341                  *
 342                  * If we're going to ACQUIRE, force the card to
 343                  * context switch before, just in case the matching
 344                  * RELEASE is already scheduled to be executed in
 345                  * another channel.
 346                  */
 347                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 348                 OUT_RING(chan, 0);
 349         }
 350
 351         BEGIN_RING(chan, NvSubSw, method, 1);
 352         OUT_RING(chan, 1);
 353
 354         if (smart && method == NV_SW_SEMAPHORE_RELEASE) {
 355                 /*
 356                  * Force the card to context switch, there may be
 357                  * another channel waiting for the semaphore we just
 358                  * released.
 359                  */
 360                 BEGIN_RING(chan, NvSubSw, NV_SW_YIELD, 1);
 361                 OUT_RING(chan, 0);
 362         }
 363
 364         /* Delay semaphore destruction until its work is done */
 365         ret = nouveau_fence_new(chan, &fence, true);
 366         if (ret)
 367                 return ret;
 368
 369         kref_get(&sema->ref);
 370         nouveau_fence_work(fence, semaphore_work, sema);
 371         nouveau_fence_unref((void *)&fence);
 372
 373         return 0;
 374 }
 375
 376 int
 377 nouveau_fence_sync(struct nouveau_fence *fence,
 378                    struct nouveau_channel *wchan)
 379 {
 380         struct nouveau_channel *chan = nouveau_fence_channel(fence);
 381         struct drm_device *dev = wchan->dev;
 382         struct nouveau_semaphore *sema;
 383         int ret;
 384
 385         if (likely(!fence || chan == wchan ||
 386                    nouveau_fence_signalled(fence, NULL)))
 387                 return 0;
 388
 389         sema = alloc_semaphore(dev);
 390         if (!sema) {
 391                 /* Early card or broken userspace, fall back to
 392                  * software sync. */
 393                 return nouveau_fence_wait(fence, NULL, false, false);
 394         }
 395
 396         /* Make wchan wait until it gets signalled */
 397         ret = emit_semaphore(wchan, NV_SW_SEMAPHORE_ACQUIRE, sema);
 398         if (ret)
 399                 goto out;
 400
 401         /* Signal the semaphore from chan */
 402         ret = emit_semaphore(chan, NV_SW_SEMAPHORE_RELEASE, sema);
 403 out:
 404         kref_put(&sema->ref, free_semaphore);
 405         return ret;
 406 }
 407
 408 int
 409 nouveau_fence_flush(void *sync_obj, void *sync_arg)
 410 {
 411         return 0;
 412 }
 413
 414 int
 415 nouveau_fence_channel_init(struct nouveau_channel *chan)
 416 {
 417         struct drm_device *dev = chan->dev;
 418         struct drm_nouveau_private *dev_priv = dev->dev_private;
 419         struct nouveau_gpuobj *obj = NULL;
 420         int ret;
 421
 422         /* Create an NV_SW object for various sync purposes */
 423         ret = nouveau_gpuobj_sw_new(chan, NV_SW, &obj);
 424         if (ret)
 425                 return ret;
 426
 427         ret = nouveau_ramht_insert(chan, NvSw, obj);
 428         nouveau_gpuobj_ref(NULL, &obj);
 429         if (ret)
 430                 return ret;
 431
 432         ret = RING_SPACE(chan, 2);
 433         if (ret)
 434                 return ret;
 435         BEGIN_RING(chan, NvSubSw, 0, 1);
 436         OUT_RING(chan, NvSw);
 437
 438         /* Create a DMA object for the shared cross-channel sync area. */
 439         if (USE_SEMA(dev)) {
 440                 struct drm_mm_node *mem = dev_priv->fence.bo->bo.mem.mm_node;
 441
 442                 ret = nouveau_gpuobj_dma_new(chan, NV_CLASS_DMA_IN_MEMORY,
 443                                              mem->start << PAGE_SHIFT,
 444                                              mem->size << PAGE_SHIFT,
 445                                              NV_DMA_ACCESS_RW,
 446                                              NV_DMA_TARGET_VIDMEM, &obj);
 447                 if (ret)
 448                         return ret;
 449
 450                 ret = nouveau_ramht_insert(chan, NvSema, obj);
 451                 nouveau_gpuobj_ref(NULL, &obj);
 452                 if (ret)
 453                         return ret;
 454
 455                 ret = RING_SPACE(chan, 2);
 456                 if (ret)
 457                         return ret;
 458                 BEGIN_RING(chan, NvSubSw, NV_SW_DMA_SEMAPHORE, 1);
 459                 OUT_RING(chan, NvSema);
 460         }
 461
 462         FIRE_RING(chan);
 463
 464         INIT_LIST_HEAD(&chan->fence.pending);
 465         spin_lock_init(&chan->fence.lock);
 466         atomic_set(&chan->fence.last_sequence_irq, 0);
 467
 468         return 0;
 469 }
 470
 471 void
 472 nouveau_fence_channel_fini(struct nouveau_channel *chan)
 473 {
 474         struct nouveau_fence *tmp, *fence;
 475
 476         list_for_each_entry_safe(fence, tmp, &chan->fence.pending, entry) {
 477                 fence->signalled = true;
 478                 list_del(&fence->entry);
 479
 480                 if (unlikely(fence->work))
 481                         fence->work(fence->priv, false);
 482
 483                 kref_put(&fence->refcount, nouveau_fence_del);
 484         }
 485 }
 486
 487 int
 488 nouveau_fence_init(struct drm_device *dev)
 489 {
 490         struct drm_nouveau_private *dev_priv = dev->dev_private;
 491         int ret;
 492
 493         /* Create a shared VRAM heap for cross-channel sync. */
 494         if (USE_SEMA(dev)) {
 495                 ret = nouveau_bo_new(dev, NULL, 4096, 0, TTM_PL_FLAG_VRAM,
 496                                      0, 0, false, true, &dev_priv->fence.bo);
 497                 if (ret)
 498                         return ret;
 499
 500                 ret = nouveau_bo_pin(dev_priv->fence.bo, TTM_PL_FLAG_VRAM);
 501                 if (ret)
 502                         goto fail;
 503
 504                 ret = nouveau_bo_map(dev_priv->fence.bo);
 505                 if (ret)
 506                         goto fail;
 507
 508                 ret = drm_mm_init(&dev_priv->fence.heap, 0,
 509                                   dev_priv->fence.bo->bo.mem.size);
 510                 if (ret)
 511                         goto fail;
 512
 513                 spin_lock_init(&dev_priv->fence.lock);
 514         }
 515
 516         return 0;
 517 fail:
 518         nouveau_bo_unmap(dev_priv->fence.bo);
 519         nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 520         return ret;
 521 }
 522
 523 void
 524 nouveau_fence_fini(struct drm_device *dev)
 525 {
 526         struct drm_nouveau_private *dev_priv = dev->dev_private;
 527
 528         if (USE_SEMA(dev)) {
 529                 drm_mm_takedown(&dev_priv->fence.heap);
 530                 nouveau_bo_unmap(dev_priv->fence.bo);
 531                 nouveau_bo_unpin(dev_priv->fence.bo);
 532                 nouveau_bo_ref(NULL, &dev_priv->fence.bo);
 533         }
 534 }