drivers/gpu/drm/i915/gem/i915_gem_domain.c

   1 /*
   2  * SPDX-License-Identifier: MIT
   3  *
   4  * Copyright © 2014-2016 Intel Corporation
   5  */
   6
   7 #include "display/intel_frontbuffer.h"
   8
   9 #include "i915_drv.h"
  10 #include "i915_gem_clflush.h"
  11 #include "i915_gem_gtt.h"
  12 #include "i915_gem_ioctls.h"
  13 #include "i915_gem_object.h"
  14 #include "i915_vma.h"
  15 #include "i915_gem_lmem.h"
  16 #include "i915_gem_mman.h"
  17
  18 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
  19 {
  20         /*
  21          * We manually flush the CPU domain so that we can override and
  22          * force the flush for the display, and perform it asyncrhonously.
  23          */
  24         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
  25         if (obj->cache_dirty)
  26                 i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
  27         obj->write_domain = 0;
  28 }
  29
  30 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
  31 {
  32         if (!i915_gem_object_is_framebuffer(obj))
  33                 return;
  34
  35         i915_gem_object_lock(obj, NULL);
  36         __i915_gem_object_flush_for_display(obj);
  37         i915_gem_object_unlock(obj);
  38 }
  39
  40 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)
  41 {
  42         if (i915_gem_object_is_framebuffer(obj))
  43                 __i915_gem_object_flush_for_display(obj);
  44 }
  45
  46 /**
  47  * Moves a single object to the WC read, and possibly write domain.
  48  * @obj: object to act on
  49  * @write: ask for write access or read only
  50  *
  51  * This function returns when the move is complete, including waiting on
  52  * flushes to occur.
  53  */
  54 int
  55 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
  56 {
  57         int ret;
  58
  59         assert_object_held(obj);
  60
  61         ret = i915_gem_object_wait(obj,
  62                                    I915_WAIT_INTERRUPTIBLE |
  63                                    (write ? I915_WAIT_ALL : 0),
  64                                    MAX_SCHEDULE_TIMEOUT);
  65         if (ret)
  66                 return ret;
  67
  68         if (obj->write_domain == I915_GEM_DOMAIN_WC)
  69                 return 0;
  70
  71         /* Flush and acquire obj->pages so that we are coherent through
  72          * direct access in memory with previous cached writes through
  73          * shmemfs and that our cache domain tracking remains valid.
  74          * For example, if the obj->filp was moved to swap without us
  75          * being notified and releasing the pages, we would mistakenly
  76          * continue to assume that the obj remained out of the CPU cached
  77          * domain.
  78          */
  79         ret = i915_gem_object_pin_pages(obj);
  80         if (ret)
  81                 return ret;
  82
  83         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
  84
  85         /* Serialise direct access to this object with the barriers for
  86          * coherent writes from the GPU, by effectively invalidating the
  87          * WC domain upon first access.
  88          */
  89         if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
  90                 mb();
  91
  92         /* It should now be out of any other write domains, and we can update
  93          * the domain values for our changes.
  94          */
  95         GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
  96         obj->read_domains |= I915_GEM_DOMAIN_WC;
  97         if (write) {
  98                 obj->read_domains = I915_GEM_DOMAIN_WC;
  99                 obj->write_domain = I915_GEM_DOMAIN_WC;
 100                 obj->mm.dirty = true;
 101         }
 102
 103         i915_gem_object_unpin_pages(obj);
 104         return 0;
 105 }
 106
 107 /**
 108  * Moves a single object to the GTT read, and possibly write domain.
 109  * @obj: object to act on
 110  * @write: ask for write access or read only
 111  *
 112  * This function returns when the move is complete, including waiting on
 113  * flushes to occur.
 114  */
 115 int
 116 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
 117 {
 118         int ret;
 119
 120         assert_object_held(obj);
 121
 122         ret = i915_gem_object_wait(obj,
 123                                    I915_WAIT_INTERRUPTIBLE |
 124                                    (write ? I915_WAIT_ALL : 0),
 125                                    MAX_SCHEDULE_TIMEOUT);
 126         if (ret)
 127                 return ret;
 128
 129         if (obj->write_domain == I915_GEM_DOMAIN_GTT)
 130                 return 0;
 131
 132         /* Flush and acquire obj->pages so that we are coherent through
 133          * direct access in memory with previous cached writes through
 134          * shmemfs and that our cache domain tracking remains valid.
 135          * For example, if the obj->filp was moved to swap without us
 136          * being notified and releasing the pages, we would mistakenly
 137          * continue to assume that the obj remained out of the CPU cached
 138          * domain.
 139          */
 140         ret = i915_gem_object_pin_pages(obj);
 141         if (ret)
 142                 return ret;
 143
 144         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
 145
 146         /* Serialise direct access to this object with the barriers for
 147          * coherent writes from the GPU, by effectively invalidating the
 148          * GTT domain upon first access.
 149          */
 150         if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
 151                 mb();
 152
 153         /* It should now be out of any other write domains, and we can update
 154          * the domain values for our changes.
 155          */
 156         GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
 157         obj->read_domains |= I915_GEM_DOMAIN_GTT;
 158         if (write) {
 159                 struct i915_vma *vma;
 160
 161                 obj->read_domains = I915_GEM_DOMAIN_GTT;
 162                 obj->write_domain = I915_GEM_DOMAIN_GTT;
 163                 obj->mm.dirty = true;
 164
 165                 spin_lock(&obj->vma.lock);
 166                 for_each_ggtt_vma(vma, obj)
 167                         if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
 168                                 i915_vma_set_ggtt_write(vma);
 169                 spin_unlock(&obj->vma.lock);
 170         }
 171
 172         i915_gem_object_unpin_pages(obj);
 173         return 0;
 174 }
 175
 176 /**
 177  * Changes the cache-level of an object across all VMA.
 178  * @obj: object to act on
 179  * @cache_level: new cache level to set for the object
 180  *
 181  * After this function returns, the object will be in the new cache-level
 182  * across all GTT and the contents of the backing storage will be coherent,
 183  * with respect to the new cache-level. In order to keep the backing storage
 184  * coherent for all users, we only allow a single cache level to be set
 185  * globally on the object and prevent it from being changed whilst the
 186  * hardware is reading from the object. That is if the object is currently
 187  * on the scanout it will be set to uncached (or equivalent display
 188  * cache coherency) and all non-MOCS GPU access will also be uncached so
 189  * that all direct access to the scanout remains coherent.
 190  */
 191 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
 192                                     enum i915_cache_level cache_level)
 193 {
 194         int ret;
 195
 196         if (obj->cache_level == cache_level)
 197                 return 0;
 198
 199         ret = i915_gem_object_wait(obj,
 200                                    I915_WAIT_INTERRUPTIBLE |
 201                                    I915_WAIT_ALL,
 202                                    MAX_SCHEDULE_TIMEOUT);
 203         if (ret)
 204                 return ret;
 205
 206         /* Always invalidate stale cachelines */
 207         if (obj->cache_level != cache_level) {
 208                 i915_gem_object_set_cache_coherency(obj, cache_level);
 209                 obj->cache_dirty = true;
 210         }
 211
 212         /* The cache-level will be applied when each vma is rebound. */
 213         return i915_gem_object_unbind(obj,
 214                                       I915_GEM_OBJECT_UNBIND_ACTIVE |
 215                                       I915_GEM_OBJECT_UNBIND_BARRIER);
 216 }
 217
 218 int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
 219                                struct drm_file *file)
 220 {
 221         struct drm_i915_gem_caching *args = data;
 222         struct drm_i915_gem_object *obj;
 223         int err = 0;
 224
 225         rcu_read_lock();
 226         obj = i915_gem_object_lookup_rcu(file, args->handle);
 227         if (!obj) {
 228                 err = -ENOENT;
 229                 goto out;
 230         }
 231
 232         switch (obj->cache_level) {
 233         case I915_CACHE_LLC:
 234         case I915_CACHE_L3_LLC:
 235                 args->caching = I915_CACHING_CACHED;
 236                 break;
 237
 238         case I915_CACHE_WT:
 239                 args->caching = I915_CACHING_DISPLAY;
 240                 break;
 241
 242         default:
 243                 args->caching = I915_CACHING_NONE;
 244                 break;
 245         }
 246 out:
 247         rcu_read_unlock();
 248         return err;
 249 }
 250
 251 int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
 252                                struct drm_file *file)
 253 {
 254         struct drm_i915_private *i915 = to_i915(dev);
 255         struct drm_i915_gem_caching *args = data;
 256         struct drm_i915_gem_object *obj;
 257         enum i915_cache_level level;
 258         int ret = 0;
 259
 260         switch (args->caching) {
 261         case I915_CACHING_NONE:
 262                 level = I915_CACHE_NONE;
 263                 break;
 264         case I915_CACHING_CACHED:
 265                 /*
 266                  * Due to a HW issue on BXT A stepping, GPU stores via a
 267                  * snooped mapping may leave stale data in a corresponding CPU
 268                  * cacheline, whereas normally such cachelines would get
 269                  * invalidated.
 270                  */
 271                 if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
 272                         return -ENODEV;
 273
 274                 level = I915_CACHE_LLC;
 275                 break;
 276         case I915_CACHING_DISPLAY:
 277                 level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
 278                 break;
 279         default:
 280                 return -EINVAL;
 281         }
 282
 283         obj = i915_gem_object_lookup(file, args->handle);
 284         if (!obj)
 285                 return -ENOENT;
 286
 287         /*
 288          * The caching mode of proxy object is handled by its generator, and
 289          * not allowed to be changed by userspace.
 290          */
 291         if (i915_gem_object_is_proxy(obj)) {
 292                 ret = -ENXIO;
 293                 goto out;
 294         }
 295
 296         ret = i915_gem_object_lock_interruptible(obj, NULL);
 297         if (ret)
 298                 goto out;
 299
 300         ret = i915_gem_object_set_cache_level(obj, level);
 301         i915_gem_object_unlock(obj);
 302
 303 out:
 304         i915_gem_object_put(obj);
 305         return ret;
 306 }
 307
 308 /*
 309  * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
 310  * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
 311  * (for pageflips). We only flush the caches while preparing the buffer for
 312  * display, the callers are responsible for frontbuffer flush.
 313  */
 314 struct i915_vma *
 315 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
 316                                      u32 alignment,
 317                                      const struct i915_ggtt_view *view,
 318                                      unsigned int flags)
 319 {
 320         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 321         struct i915_gem_ww_ctx ww;
 322         struct i915_vma *vma;
 323         int ret;
 324
 325         /* Frame buffer must be in LMEM (no migration yet) */
 326         if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
 327                 return ERR_PTR(-EINVAL);
 328
 329         i915_gem_ww_ctx_init(&ww, true);
 330 retry:
 331         ret = i915_gem_object_lock(obj, &ww);
 332         if (ret)
 333                 goto err;
 334         /*
 335          * The display engine is not coherent with the LLC cache on gen6.  As
 336          * a result, we make sure that the pinning that is about to occur is
 337          * done with uncached PTEs. This is lowest common denominator for all
 338          * chipsets.
 339          *
 340          * However for gen6+, we could do better by using the GFDT bit instead
 341          * of uncaching, which would allow us to flush all the LLC-cached data
 342          * with that bit in the PTE to main memory with just one PIPE_CONTROL.
 343          */
 344         ret = i915_gem_object_set_cache_level(obj,
 345                                               HAS_WT(i915) ?
 346                                               I915_CACHE_WT : I915_CACHE_NONE);
 347         if (ret)
 348                 goto err;
 349
 350         /*
 351          * As the user may map the buffer once pinned in the display plane
 352          * (e.g. libkms for the bootup splash), we have to ensure that we
 353          * always use map_and_fenceable for all scanout buffers. However,
 354          * it may simply be too big to fit into mappable, in which case
 355          * put it anyway and hope that userspace can cope (but always first
 356          * try to preserve the existing ABI).
 357          */
 358         vma = ERR_PTR(-ENOSPC);
 359         if ((flags & PIN_MAPPABLE) == 0 &&
 360             (!view || view->type == I915_GGTT_VIEW_NORMAL))
 361                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0, alignment,
 362                                                   flags | PIN_MAPPABLE |
 363                                                   PIN_NONBLOCK);
 364         if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK))
 365                 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, view, 0,
 366                                                   alignment, flags);
 367         if (IS_ERR(vma)) {
 368                 ret = PTR_ERR(vma);
 369                 goto err;
 370         }
 371
 372         vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
 373
 374         i915_gem_object_flush_if_display_locked(obj);
 375
 376 err:
 377         if (ret == -EDEADLK) {
 378                 ret = i915_gem_ww_ctx_backoff(&ww);
 379                 if (!ret)
 380                         goto retry;
 381         }
 382         i915_gem_ww_ctx_fini(&ww);
 383
 384         if (ret)
 385                 return ERR_PTR(ret);
 386
 387         return vma;
 388 }
 389
 390 static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
 391 {
 392         struct drm_i915_private *i915 = to_i915(obj->base.dev);
 393         struct i915_vma *vma;
 394
 395         if (list_empty(&obj->vma.list))
 396                 return;
 397
 398         mutex_lock(&i915->ggtt.vm.mutex);
 399         spin_lock(&obj->vma.lock);
 400         for_each_ggtt_vma(vma, obj) {
 401                 if (!drm_mm_node_allocated(&vma->node))
 402                         continue;
 403
 404                 GEM_BUG_ON(vma->vm != &i915->ggtt.vm);
 405                 list_move_tail(&vma->vm_link, &vma->vm->bound_list);
 406         }
 407         spin_unlock(&obj->vma.lock);
 408         mutex_unlock(&i915->ggtt.vm.mutex);
 409
 410         if (i915_gem_object_is_shrinkable(obj)) {
 411                 unsigned long flags;
 412
 413                 spin_lock_irqsave(&i915->mm.obj_lock, flags);
 414
 415                 if (obj->mm.madv == I915_MADV_WILLNEED &&
 416                     !atomic_read(&obj->mm.shrink_pin))
 417                         list_move_tail(&obj->mm.link, &i915->mm.shrink_list);
 418
 419                 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
 420         }
 421 }
 422
 423 void
 424 i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
 425 {
 426         /* Bump the LRU to try and avoid premature eviction whilst flipping  */
 427         i915_gem_object_bump_inactive_ggtt(vma->obj);
 428
 429         i915_vma_unpin(vma);
 430 }
 431
 432 /**
 433  * Moves a single object to the CPU read, and possibly write domain.
 434  * @obj: object to act on
 435  * @write: requesting write or read-only access
 436  *
 437  * This function returns when the move is complete, including waiting on
 438  * flushes to occur.
 439  */
 440 int
 441 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
 442 {
 443         int ret;
 444
 445         assert_object_held(obj);
 446
 447         ret = i915_gem_object_wait(obj,
 448                                    I915_WAIT_INTERRUPTIBLE |
 449                                    (write ? I915_WAIT_ALL : 0),
 450                                    MAX_SCHEDULE_TIMEOUT);
 451         if (ret)
 452                 return ret;
 453
 454         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 455
 456         /* Flush the CPU cache if it's still invalid. */
 457         if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
 458                 i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
 459                 obj->read_domains |= I915_GEM_DOMAIN_CPU;
 460         }
 461
 462         /* It should now be out of any other write domains, and we can update
 463          * the domain values for our changes.
 464          */
 465         GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
 466
 467         /* If we're writing through the CPU, then the GPU read domains will
 468          * need to be invalidated at next use.
 469          */
 470         if (write)
 471                 __start_cpu_write(obj);
 472
 473         return 0;
 474 }
 475
 476 /**
 477  * Called when user space prepares to use an object with the CPU, either
 478  * through the mmap ioctl's mapping or a GTT mapping.
 479  * @dev: drm device
 480  * @data: ioctl data blob
 481  * @file: drm file
 482  */
 483 int
 484 i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
 485                           struct drm_file *file)
 486 {
 487         struct drm_i915_gem_set_domain *args = data;
 488         struct drm_i915_gem_object *obj;
 489         u32 read_domains = args->read_domains;
 490         u32 write_domain = args->write_domain;
 491         int err;
 492
 493         /* Only handle setting domains to types used by the CPU. */
 494         if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
 495                 return -EINVAL;
 496
 497         /*
 498          * Having something in the write domain implies it's in the read
 499          * domain, and only that read domain.  Enforce that in the request.
 500          */
 501         if (write_domain && read_domains != write_domain)
 502                 return -EINVAL;
 503
 504         if (!read_domains)
 505                 return 0;
 506
 507         obj = i915_gem_object_lookup(file, args->handle);
 508         if (!obj)
 509                 return -ENOENT;
 510
 511         /*
 512          * Already in the desired write domain? Nothing for us to do!
 513          *
 514          * We apply a little bit of cunning here to catch a broader set of
 515          * no-ops. If obj->write_domain is set, we must be in the same
 516          * obj->read_domains, and only that domain. Therefore, if that
 517          * obj->write_domain matches the request read_domains, we are
 518          * already in the same read/write domain and can skip the operation,
 519          * without having to further check the requested write_domain.
 520          */
 521         if (READ_ONCE(obj->write_domain) == read_domains) {
 522                 err = 0;
 523                 goto out;
 524         }
 525
 526         /*
 527          * Try to flush the object off the GPU without holding the lock.
 528          * We will repeat the flush holding the lock in the normal manner
 529          * to catch cases where we are gazumped.
 530          */
 531         err = i915_gem_object_wait(obj,
 532                                    I915_WAIT_INTERRUPTIBLE |
 533                                    I915_WAIT_PRIORITY |
 534                                    (write_domain ? I915_WAIT_ALL : 0),
 535                                    MAX_SCHEDULE_TIMEOUT);
 536         if (err)
 537                 goto out;
 538
 539         /*
 540          * Proxy objects do not control access to the backing storage, ergo
 541          * they cannot be used as a means to manipulate the cache domain
 542          * tracking for that backing storage. The proxy object is always
 543          * considered to be outside of any cache domain.
 544          */
 545         if (i915_gem_object_is_proxy(obj)) {
 546                 err = -ENXIO;
 547                 goto out;
 548         }
 549
 550         /*
 551          * Flush and acquire obj->pages so that we are coherent through
 552          * direct access in memory with previous cached writes through
 553          * shmemfs and that our cache domain tracking remains valid.
 554          * For example, if the obj->filp was moved to swap without us
 555          * being notified and releasing the pages, we would mistakenly
 556          * continue to assume that the obj remained out of the CPU cached
 557          * domain.
 558          */
 559         err = i915_gem_object_pin_pages(obj);
 560         if (err)
 561                 goto out;
 562
 563         err = i915_gem_object_lock_interruptible(obj, NULL);
 564         if (err)
 565                 goto out_unpin;
 566
 567         if (read_domains & I915_GEM_DOMAIN_WC)
 568                 err = i915_gem_object_set_to_wc_domain(obj, write_domain);
 569         else if (read_domains & I915_GEM_DOMAIN_GTT)
 570                 err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
 571         else
 572                 err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
 573
 574         /* And bump the LRU for this access */
 575         i915_gem_object_bump_inactive_ggtt(obj);
 576
 577         i915_gem_object_unlock(obj);
 578
 579         if (write_domain)
 580                 i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 581
 582 out_unpin:
 583         i915_gem_object_unpin_pages(obj);
 584 out:
 585         i915_gem_object_put(obj);
 586         return err;
 587 }
 588
 589 /*
 590  * Pins the specified object's pages and synchronizes the object with
 591  * GPU accesses. Sets needs_clflush to non-zero if the caller should
 592  * flush the object from the CPU cache.
 593  */
 594 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
 595                                  unsigned int *needs_clflush)
 596 {
 597         int ret;
 598
 599         *needs_clflush = 0;
 600         if (!i915_gem_object_has_struct_page(obj))
 601                 return -ENODEV;
 602
 603         assert_object_held(obj);
 604
 605         ret = i915_gem_object_wait(obj,
 606                                    I915_WAIT_INTERRUPTIBLE,
 607                                    MAX_SCHEDULE_TIMEOUT);
 608         if (ret)
 609                 return ret;
 610
 611         ret = i915_gem_object_pin_pages(obj);
 612         if (ret)
 613                 return ret;
 614
 615         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
 616             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 617                 ret = i915_gem_object_set_to_cpu_domain(obj, false);
 618                 if (ret)
 619                         goto err_unpin;
 620                 else
 621                         goto out;
 622         }
 623
 624         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 625
 626         /* If we're not in the cpu read domain, set ourself into the gtt
 627          * read domain and manually flush cachelines (if required). This
 628          * optimizes for the case when the gpu will dirty the data
 629          * anyway again before the next pread happens.
 630          */
 631         if (!obj->cache_dirty &&
 632             !(obj->read_domains & I915_GEM_DOMAIN_CPU))
 633                 *needs_clflush = CLFLUSH_BEFORE;
 634
 635 out:
 636         /* return with the pages pinned */
 637         return 0;
 638
 639 err_unpin:
 640         i915_gem_object_unpin_pages(obj);
 641         return ret;
 642 }
 643
 644 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
 645                                   unsigned int *needs_clflush)
 646 {
 647         int ret;
 648
 649         *needs_clflush = 0;
 650         if (!i915_gem_object_has_struct_page(obj))
 651                 return -ENODEV;
 652
 653         assert_object_held(obj);
 654
 655         ret = i915_gem_object_wait(obj,
 656                                    I915_WAIT_INTERRUPTIBLE |
 657                                    I915_WAIT_ALL,
 658                                    MAX_SCHEDULE_TIMEOUT);
 659         if (ret)
 660                 return ret;
 661
 662         ret = i915_gem_object_pin_pages(obj);
 663         if (ret)
 664                 return ret;
 665
 666         if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
 667             !static_cpu_has(X86_FEATURE_CLFLUSH)) {
 668                 ret = i915_gem_object_set_to_cpu_domain(obj, true);
 669                 if (ret)
 670                         goto err_unpin;
 671                 else
 672                         goto out;
 673         }
 674
 675         i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
 676
 677         /* If we're not in the cpu write domain, set ourself into the
 678          * gtt write domain and manually flush cachelines (as required).
 679          * This optimizes for the case when the gpu will use the data
 680          * right away and we therefore have to clflush anyway.
 681          */
 682         if (!obj->cache_dirty) {
 683                 *needs_clflush |= CLFLUSH_AFTER;
 684
 685                 /*
 686                  * Same trick applies to invalidate partially written
 687                  * cachelines read before writing.
 688                  */
 689                 if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
 690                         *needs_clflush |= CLFLUSH_BEFORE;
 691         }
 692
 693 out:
 694         i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 695         obj->mm.dirty = true;
 696         /* return with the pages pinned */
 697         return 0;
 698
 699 err_unpin:
 700         i915_gem_object_unpin_pages(obj);
 701         return ret;
 702 }