#include <linux/pci.h>
#include <linux/dma-buf.h>
+#define RQ_BUG_ON(expr)
+
static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
-static __must_check int
-i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
- bool readonly);
static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj);
-
+i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
+static void
+i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
static void i915_gem_write_fence(struct drm_device *dev, int reg,
struct drm_i915_gem_object *obj);
static void i915_gem_object_update_fence(struct drm_i915_gem_object *obj,
ret = i915_gem_object_wait_rendering(obj, true);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
ret = i915_gem_object_get_pages(obj);
ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
-
- i915_gem_object_retire(obj);
}
/* Same trick applies to invalidate partially written cachelines read
* before writing. */
return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
}
-static int __i915_spin_request(struct drm_i915_gem_request *rq)
+static int __i915_spin_request(struct drm_i915_gem_request *req)
{
unsigned long timeout;
- if (i915_gem_request_get_ring(rq)->irq_refcount)
+ if (i915_gem_request_get_ring(req)->irq_refcount)
return -EBUSY;
timeout = jiffies + 1;
while (!need_resched()) {
- if (i915_gem_request_completed(rq, true))
+ if (i915_gem_request_completed(req, true))
return 0;
if (time_after_eq(jiffies, timeout))
cpu_relax_lowlatency();
}
- if (i915_gem_request_completed(rq, false))
+ if (i915_gem_request_completed(req, false))
return 0;
return -EAGAIN;
unsigned reset_counter,
bool interruptible,
s64 *timeout,
- struct drm_i915_file_private *file_priv)
+ struct intel_rps_client *rps)
{
struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
struct drm_device *dev = ring->dev;
WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
+ if (list_empty(&req->list))
+ return 0;
+
if (i915_gem_request_completed(req, true))
return 0;
timeout_expire = timeout ?
jiffies + nsecs_to_jiffies_timeout((u64)*timeout) : 0;
- if (INTEL_INFO(dev)->gen >= 6)
- gen6_rps_boost(dev_priv, file_priv);
+ if (INTEL_INFO(dev_priv)->gen >= 6)
+ gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
/* Record current time in case interrupted by signal, or wedged */
trace_i915_gem_request_wait_begin(req);
return ret;
}
+static inline void
+i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
+{
+ struct drm_i915_file_private *file_priv = request->file_priv;
+
+ if (!file_priv)
+ return;
+
+ spin_lock(&file_priv->mm.lock);
+ list_del(&request->client_list);
+ request->file_priv = NULL;
+ spin_unlock(&file_priv->mm.lock);
+}
+
+static void i915_gem_request_retire(struct drm_i915_gem_request *request)
+{
+ trace_i915_gem_request_retire(request);
+
+ /* We know the GPU must have read the request to have
+ * sent us the seqno + interrupt, so use the position
+ * of tail of the request to update the last known position
+ * of the GPU head.
+ *
+ * Note this requires that we are always called in request
+ * completion order.
+ */
+ request->ringbuf->last_retired_head = request->postfix;
+
+ list_del_init(&request->list);
+ i915_gem_request_remove_from_client(request);
+
+ put_pid(request->pid);
+
+ i915_gem_request_unreference(request);
+}
+
+static void
+__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
+{
+ struct intel_engine_cs *engine = req->ring;
+ struct drm_i915_gem_request *tmp;
+
+ lockdep_assert_held(&engine->dev->struct_mutex);
+
+ if (list_empty(&req->list))
+ return;
+
+ do {
+ tmp = list_first_entry(&engine->request_list,
+ typeof(*tmp), list);
+
+ i915_gem_request_retire(tmp);
+ } while (tmp != req);
+
+ WARN_ON(i915_verify_lists(engine->dev));
+}
+
/**
* Waits for a request to be signaled, and cleans up the
* request and object lists appropriately for that event.
struct drm_device *dev;
struct drm_i915_private *dev_priv;
bool interruptible;
- unsigned reset_counter;
int ret;
BUG_ON(req == NULL);
if (ret)
return ret;
- reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
- ret = __i915_wait_request(req, reset_counter,
+ ret = __i915_wait_request(req,
+ atomic_read(&dev_priv->gpu_error.reset_counter),
interruptible, NULL, NULL);
- i915_gem_request_unreference(req);
- return ret;
-}
-
-static int
-i915_gem_object_wait_rendering__tail(struct drm_i915_gem_object *obj)
-{
- if (!obj->active)
- return 0;
-
- /* Manually manage the write flush as we may have not yet
- * retired the buffer.
- *
- * Note that the last_write_req is always the earlier of
- * the two (read/write) requests, so if we haved successfully waited,
- * we know we have passed the last write.
- */
- i915_gem_request_assign(&obj->last_write_req, NULL);
+ if (ret)
+ return ret;
+ __i915_gem_request_retire__upto(req);
return 0;
}
* Ensures that all rendering to the object has completed and the object is
* safe to unbind from the GTT or access from the CPU.
*/
-static __must_check int
+int
i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
bool readonly)
{
- struct drm_i915_gem_request *req;
- int ret;
+ int ret, i;
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
- ret = i915_wait_request(req);
- if (ret)
- return ret;
+ if (readonly) {
+ if (obj->last_write_req != NULL) {
+ ret = i915_wait_request(obj->last_write_req);
+ if (ret)
+ return ret;
+
+ i = obj->last_write_req->ring->id;
+ if (obj->last_read_req[i] == obj->last_write_req)
+ i915_gem_object_retire__read(obj, i);
+ else
+ i915_gem_object_retire__write(obj);
+ }
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ ret = i915_wait_request(obj->last_read_req[i]);
+ if (ret)
+ return ret;
+
+ i915_gem_object_retire__read(obj, i);
+ }
+ RQ_BUG_ON(obj->active);
+ }
- return i915_gem_object_wait_rendering__tail(obj);
+ return 0;
+}
+
+static void
+i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
+ struct drm_i915_gem_request *req)
+{
+ int ring = req->ring->id;
+
+ if (obj->last_read_req[ring] == req)
+ i915_gem_object_retire__read(obj, ring);
+ else if (obj->last_write_req == req)
+ i915_gem_object_retire__write(obj);
+
+ __i915_gem_request_retire__upto(req);
}
/* A nonblocking variant of the above wait. This is a highly dangerous routine
*/
static __must_check int
i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
- struct drm_i915_file_private *file_priv,
+ struct intel_rps_client *rps,
bool readonly)
{
- struct drm_i915_gem_request *req;
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct drm_i915_gem_request *requests[I915_NUM_RINGS];
unsigned reset_counter;
- int ret;
+ int ret, i, n = 0;
BUG_ON(!mutex_is_locked(&dev->struct_mutex));
BUG_ON(!dev_priv->mm.interruptible);
- req = readonly ? obj->last_write_req : obj->last_read_req;
- if (!req)
+ if (!obj->active)
return 0;
ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
if (ret)
return ret;
- ret = i915_gem_check_olr(req);
- if (ret)
- return ret;
-
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ if (readonly) {
+ struct drm_i915_gem_request *req;
+
+ req = obj->last_write_req;
+ if (req == NULL)
+ return 0;
+
+ ret = i915_gem_check_olr(req);
+ if (ret)
+ goto err;
+
+ requests[n++] = i915_gem_request_reference(req);
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct drm_i915_gem_request *req;
+
+ req = obj->last_read_req[i];
+ if (req == NULL)
+ continue;
+
+ ret = i915_gem_check_olr(req);
+ if (ret)
+ goto err;
+
+ requests[n++] = i915_gem_request_reference(req);
+ }
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true, NULL, file_priv);
+ for (i = 0; ret == 0 && i < n; i++)
+ ret = __i915_wait_request(requests[i], reset_counter, true,
+ NULL, rps);
mutex_lock(&dev->struct_mutex);
- i915_gem_request_unreference(req);
- if (ret)
- return ret;
- return i915_gem_object_wait_rendering__tail(obj);
+err:
+ for (i = 0; i < n; i++) {
+ if (ret == 0)
+ i915_gem_object_retire_request(obj, requests[i]);
+ i915_gem_request_unreference(requests[i]);
+ }
+
+ return ret;
+}
+
+static struct intel_rps_client *to_rps_client(struct drm_file *file)
+{
+ struct drm_i915_file_private *fpriv = file->driver_priv;
+ return &fpriv->rps;
}
/**
* to catch cases where we are gazumped.
*/
ret = i915_gem_object_wait_rendering__nonblocking(obj,
- file->driver_priv,
+ to_rps_client(file),
!write_domain);
if (ret)
goto unref;
struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
struct drm_device *dev = obj->base.dev;
struct drm_i915_private *dev_priv = dev->dev_private;
+ struct i915_ggtt_view view = i915_ggtt_view_normal;
pgoff_t page_offset;
unsigned long pfn;
int ret = 0;
goto unlock;
}
- /* Now bind it into the GTT if needed */
- ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE);
+ /* Use a partial view if the object is bigger than the aperture. */
+ if (obj->base.size >= dev_priv->gtt.mappable_end &&
+ obj->tiling_mode == I915_TILING_NONE) {
+ static const unsigned int chunk_size = 256; // 1 MiB
+
+ memset(&view, 0, sizeof(view));
+ view.type = I915_GGTT_VIEW_PARTIAL;
+ view.params.partial.offset = rounddown(page_offset, chunk_size);
+ view.params.partial.size =
+ min_t(unsigned int,
+ chunk_size,
+ (vma->vm_end - vma->vm_start)/PAGE_SIZE -
+ view.params.partial.offset);
+ }
+
+ /* Now pin it into the GTT if needed */
+ ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
if (ret)
goto unlock;
goto unpin;
/* Finally, remap it using the new GTT offset */
- pfn = dev_priv->gtt.mappable_base + i915_gem_obj_ggtt_offset(obj);
+ pfn = dev_priv->gtt.mappable_base +
+ i915_gem_obj_ggtt_offset_view(obj, &view);
pfn >>= PAGE_SHIFT;
- if (!obj->fault_mappable) {
- unsigned long size = min_t(unsigned long,
- vma->vm_end - vma->vm_start,
- obj->base.size);
- int i;
+ if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
+ /* Overriding existing pages in partial view does not cause
+ * us any trouble as TLBs are still valid because the fault
+ * is due to userspace losing part of the mapping or never
+ * having accessed it before (at this partials' range).
+ */
+ unsigned long base = vma->vm_start +
+ (view.params.partial.offset << PAGE_SHIFT);
+ unsigned int i;
- for (i = 0; i < size >> PAGE_SHIFT; i++) {
- ret = vm_insert_pfn(vma,
- (unsigned long)vma->vm_start + i * PAGE_SIZE,
- pfn + i);
+ for (i = 0; i < view.params.partial.size; i++) {
+ ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
if (ret)
break;
}
obj->fault_mappable = true;
- } else
- ret = vm_insert_pfn(vma,
- (unsigned long)vmf->virtual_address,
- pfn + page_offset);
+ } else {
+ if (!obj->fault_mappable) {
+ unsigned long size = min_t(unsigned long,
+ vma->vm_end - vma->vm_start,
+ obj->base.size);
+ int i;
+
+ for (i = 0; i < size >> PAGE_SHIFT; i++) {
+ ret = vm_insert_pfn(vma,
+ (unsigned long)vma->vm_start + i * PAGE_SIZE,
+ pfn + i);
+ if (ret)
+ break;
+ }
+
+ obj->fault_mappable = true;
+ } else
+ ret = vm_insert_pfn(vma,
+ (unsigned long)vmf->virtual_address,
+ pfn + page_offset);
+ }
unpin:
- i915_gem_object_ggtt_unpin(obj);
+ i915_gem_object_ggtt_unpin_view(obj, &view);
unlock:
mutex_unlock(&dev->struct_mutex);
out:
uint32_t handle,
uint64_t *offset)
{
- struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_object *obj;
int ret;
goto unlock;
}
- if (obj->base.size > dev_priv->gtt.mappable_end) {
- ret = -E2BIG;
- goto out;
- }
-
if (obj->madv != I915_MADV_WILLNEED) {
DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
ret = -EFAULT;
return 0;
}
-static void
-i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
- struct intel_engine_cs *ring)
+void i915_vma_move_to_active(struct i915_vma *vma,
+ struct intel_engine_cs *ring)
{
- struct drm_i915_gem_request *req;
- struct intel_engine_cs *old_ring;
-
- BUG_ON(ring == NULL);
-
- req = intel_ring_get_request(ring);
- old_ring = i915_gem_request_get_ring(obj->last_read_req);
-
- if (old_ring != ring && obj->last_write_req) {
- /* Keep the request relative to the current ring */
- i915_gem_request_assign(&obj->last_write_req, req);
- }
+ struct drm_i915_gem_object *obj = vma->obj;
/* Add a reference if we're newly entering the active list. */
- if (!obj->active) {
+ if (obj->active == 0)
drm_gem_object_reference(&obj->base);
- obj->active = 1;
- }
+ obj->active |= intel_ring_flag(ring);
- list_move_tail(&obj->ring_list, &ring->active_list);
+ list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
+ i915_gem_request_assign(&obj->last_read_req[ring->id],
+ intel_ring_get_request(ring));
- i915_gem_request_assign(&obj->last_read_req, req);
+ list_move_tail(&vma->mm_list, &vma->vm->active_list);
}
-void i915_vma_move_to_active(struct i915_vma *vma,
- struct intel_engine_cs *ring)
+static void
+i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
{
- list_move_tail(&vma->mm_list, &vma->vm->active_list);
- return i915_gem_object_move_to_active(vma->obj, ring);
+ RQ_BUG_ON(obj->last_write_req == NULL);
+ RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
+
+ i915_gem_request_assign(&obj->last_write_req, NULL);
+ intel_fb_obj_flush(obj, true);
}
static void
-i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
+i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
{
struct i915_vma *vma;
- BUG_ON(obj->base.write_domain & ~I915_GEM_GPU_DOMAINS);
- BUG_ON(!obj->active);
+ RQ_BUG_ON(obj->last_read_req[ring] == NULL);
+ RQ_BUG_ON(!(obj->active & (1 << ring)));
+
+ list_del_init(&obj->ring_list[ring]);
+ i915_gem_request_assign(&obj->last_read_req[ring], NULL);
+
+ if (obj->last_write_req && obj->last_write_req->ring->id == ring)
+ i915_gem_object_retire__write(obj);
+
+ obj->active &= ~(1 << ring);
+ if (obj->active)
+ return;
list_for_each_entry(vma, &obj->vma_list, vma_link) {
if (!list_empty(&vma->mm_list))
list_move_tail(&vma->mm_list, &vma->vm->inactive_list);
}
- intel_fb_obj_flush(obj, true);
-
- list_del_init(&obj->ring_list);
-
- i915_gem_request_assign(&obj->last_read_req, NULL);
- i915_gem_request_assign(&obj->last_write_req, NULL);
- obj->base.write_domain = 0;
-
i915_gem_request_assign(&obj->last_fenced_req, NULL);
-
- obj->active = 0;
drm_gem_object_unreference(&obj->base);
-
- WARN_ON(i915_verify_lists(dev));
-}
-
-static void
-i915_gem_object_retire(struct drm_i915_gem_object *obj)
-{
- if (obj->last_read_req == NULL)
- return;
-
- if (i915_gem_request_completed(obj->last_read_req, true))
- i915_gem_object_move_to_inactive(obj);
}
static int
return 0;
}
-static inline void
-i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
-{
- struct drm_i915_file_private *file_priv = request->file_priv;
-
- if (!file_priv)
- return;
-
- spin_lock(&file_priv->mm.lock);
- list_del(&request->client_list);
- request->file_priv = NULL;
- spin_unlock(&file_priv->mm.lock);
-}
-
static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
const struct intel_context *ctx)
{
}
}
-static void i915_gem_free_request(struct drm_i915_gem_request *request)
-{
- list_del(&request->list);
- i915_gem_request_remove_from_client(request);
-
- put_pid(request->pid);
-
- i915_gem_request_unreference(request);
-}
-
void i915_gem_request_free(struct kref *req_ref)
{
struct drm_i915_gem_request *req = container_of(req_ref,
struct intel_context *ctx)
{
struct drm_i915_private *dev_priv = to_i915(ring->dev);
- struct drm_i915_gem_request *rq;
+ struct drm_i915_gem_request *req;
int ret;
if (ring->outstanding_lazy_request)
return 0;
- rq = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
- if (rq == NULL)
+ req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
+ if (req == NULL)
return -ENOMEM;
- kref_init(&rq->ref);
- rq->i915 = dev_priv;
+ kref_init(&req->ref);
+ req->i915 = dev_priv;
- ret = i915_gem_get_seqno(ring->dev, &rq->seqno);
- if (ret) {
- kfree(rq);
- return ret;
- }
+ ret = i915_gem_get_seqno(ring->dev, &req->seqno);
+ if (ret)
+ goto err;
- rq->ring = ring;
+ req->ring = ring;
if (i915.enable_execlists)
- ret = intel_logical_ring_alloc_request_extras(rq, ctx);
+ ret = intel_logical_ring_alloc_request_extras(req, ctx);
else
- ret = intel_ring_alloc_request_extras(rq);
- if (ret) {
- kfree(rq);
- return ret;
- }
+ ret = intel_ring_alloc_request_extras(req);
+ if (ret)
+ goto err;
- ring->outstanding_lazy_request = rq;
+ ring->outstanding_lazy_request = req;
return 0;
+
+err:
+ kmem_cache_free(dev_priv->requests, req);
+ return ret;
}
struct drm_i915_gem_request *
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_retire__read(obj, ring->id);
}
/*
struct drm_i915_gem_request,
list);
- i915_gem_free_request(request);
+ i915_gem_request_retire(request);
}
/* This may not have been flushed before the reset, so clean it now */
i915_gem_context_reset(dev);
i915_gem_restore_fences(dev);
+
+ WARN_ON(i915_verify_lists(dev));
}
/**
void
i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
{
- if (list_empty(&ring->request_list))
- return;
-
WARN_ON(i915_verify_lists(ring->dev));
+ if (list_empty(&ring->active_list))
+ return;
+
/* Retire requests first as we use it above for the early return.
* If we retire requests last, we may use a later seqno and so clear
* the requests lists without clearing the active list, leading to
if (!i915_gem_request_completed(request, true))
break;
- trace_i915_gem_request_retire(request);
-
- /* We know the GPU must have read the request to have
- * sent us the seqno + interrupt, so use the position
- * of tail of the request to update the last known position
- * of the GPU head.
- */
- request->ringbuf->last_retired_head = request->postfix;
-
- i915_gem_free_request(request);
+ i915_gem_request_retire(request);
}
/* Move any buffers on the active list that are no longer referenced
obj = list_first_entry(&ring->active_list,
struct drm_i915_gem_object,
- ring_list);
+ ring_list[ring->id]);
- if (!i915_gem_request_completed(obj->last_read_req, true))
+ if (!list_empty(&obj->last_read_req[ring->id]->list))
break;
- i915_gem_object_move_to_inactive(obj);
+ i915_gem_object_retire__read(obj, ring->id);
}
if (unlikely(ring->trace_irq_req &&
static int
i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
{
- struct intel_engine_cs *ring;
- int ret;
+ int ret, i;
- if (obj->active) {
- ring = i915_gem_request_get_ring(obj->last_read_req);
+ if (!obj->active)
+ return 0;
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ struct drm_i915_gem_request *req;
+
+ req = obj->last_read_req[i];
+ if (req == NULL)
+ continue;
+
+ if (list_empty(&req->list))
+ goto retire;
- ret = i915_gem_check_olr(obj->last_read_req);
+ ret = i915_gem_check_olr(req);
if (ret)
return ret;
- i915_gem_retire_requests_ring(ring);
+ if (i915_gem_request_completed(req, true)) {
+ __i915_gem_request_retire__upto(req);
+retire:
+ i915_gem_object_retire__read(obj, i);
+ }
}
return 0;
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_gem_wait *args = data;
struct drm_i915_gem_object *obj;
- struct drm_i915_gem_request *req;
+ struct drm_i915_gem_request *req[I915_NUM_RINGS];
unsigned reset_counter;
- int ret = 0;
+ int i, n = 0;
+ int ret;
if (args->flags != 0)
return -EINVAL;
if (ret)
goto out;
- if (!obj->active || !obj->last_read_req)
+ if (!obj->active)
goto out;
- req = obj->last_read_req;
-
/* Do this after OLR check to make sure we make forward progress polling
* on this IOCTL with a timeout == 0 (like busy ioctl)
*/
drm_gem_object_unreference(&obj->base);
reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
- i915_gem_request_reference(req);
+
+ for (i = 0; i < I915_NUM_RINGS; i++) {
+ if (obj->last_read_req[i] == NULL)
+ continue;
+
+ req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
+ }
+
mutex_unlock(&dev->struct_mutex);
- ret = __i915_wait_request(req, reset_counter, true,
- args->timeout_ns > 0 ? &args->timeout_ns : NULL,
- file->driver_priv);
- i915_gem_request_unreference__unlocked(req);
+ for (i = 0; i < n; i++) {
+ if (ret == 0)
+ ret = __i915_wait_request(req[i], reset_counter, true,
+ args->timeout_ns > 0 ? &args->timeout_ns : NULL,
+ file->driver_priv);
+ i915_gem_request_unreference__unlocked(req[i]);
+ }
return ret;
out:
return ret;
}
+static int
+__i915_gem_object_sync(struct drm_i915_gem_object *obj,
+ struct intel_engine_cs *to,
+ struct drm_i915_gem_request *req)
+{
+ struct intel_engine_cs *from;
+ int ret;
+
+ from = i915_gem_request_get_ring(req);
+ if (to == from)
+ return 0;
+
+ if (i915_gem_request_completed(req, true))
+ return 0;
+
+ ret = i915_gem_check_olr(req);
+ if (ret)
+ return ret;
+
+ if (!i915_semaphore_is_enabled(obj->base.dev)) {
+ struct drm_i915_private *i915 = to_i915(obj->base.dev);
+ ret = __i915_wait_request(req,
+ atomic_read(&i915->gpu_error.reset_counter),
+ i915->mm.interruptible,
+ NULL,
+ &i915->rps.semaphores);
+ if (ret)
+ return ret;
+
+ i915_gem_object_retire_request(obj, req);
+ } else {
+ int idx = intel_ring_sync_index(from, to);
+ u32 seqno = i915_gem_request_get_seqno(req);
+
+ if (seqno <= from->semaphore.sync_seqno[idx])
+ return 0;
+
+ trace_i915_gem_ring_sync_to(from, to, req);
+ ret = to->semaphore.sync_to(to, from, seqno);
+ if (ret)
+ return ret;
+
+ /* We use last_read_req because sync_to()
+ * might have just caused seqno wrap under
+ * the radar.
+ */
+ from->semaphore.sync_seqno[idx] =
+ i915_gem_request_get_seqno(obj->last_read_req[from->id]);
+ }
+
+ return 0;
+}
+
/**
* i915_gem_object_sync - sync an object to a ring.
*
*
* This code is meant to abstract object synchronization with the GPU.
* Calling with NULL implies synchronizing the object with the CPU
- * rather than a particular GPU ring.
+ * rather than a particular GPU ring. Conceptually we serialise writes
+ * between engines inside the GPU. We only allow on engine to write
+ * into a buffer at any time, but multiple readers. To ensure each has
+ * a coherent view of memory, we must:
+ *
+ * - If there is an outstanding write request to the object, the new
+ * request must wait for it to complete (either CPU or in hw, requests
+ * on the same ring will be naturally ordered).
+ *
+ * - If we are a write request (pending_write_domain is set), the new
+ * request must wait for outstanding read requests to complete.
*
* Returns 0 if successful, else propagates up the lower layer error.
*/
i915_gem_object_sync(struct drm_i915_gem_object *obj,
struct intel_engine_cs *to)
{
- struct intel_engine_cs *from;
- u32 seqno;
- int ret, idx;
-
- from = i915_gem_request_get_ring(obj->last_read_req);
-
- if (from == NULL || to == from)
- return 0;
-
- if (to == NULL || !i915_semaphore_is_enabled(obj->base.dev))
- return i915_gem_object_wait_rendering(obj, false);
+ const bool readonly = obj->base.pending_write_domain == 0;
+ struct drm_i915_gem_request *req[I915_NUM_RINGS];
+ int ret, i, n;
- idx = intel_ring_sync_index(from, to);
-
- seqno = i915_gem_request_get_seqno(obj->last_read_req);
- /* Optimization: Avoid semaphore sync when we are sure we already
- * waited for an object with higher seqno */
- if (seqno <= from->semaphore.sync_seqno[idx])
+ if (!obj->active)
return 0;
- ret = i915_gem_check_olr(obj->last_read_req);
- if (ret)
- return ret;
+ if (to == NULL)
+ return i915_gem_object_wait_rendering(obj, readonly);
- trace_i915_gem_ring_sync_to(from, to, obj->last_read_req);
- ret = to->semaphore.sync_to(to, from, seqno);
- if (!ret)
- /* We use last_read_req because sync_to()
- * might have just caused seqno wrap under
- * the radar.
- */
- from->semaphore.sync_seqno[idx] =
- i915_gem_request_get_seqno(obj->last_read_req);
+ n = 0;
+ if (readonly) {
+ if (obj->last_write_req)
+ req[n++] = obj->last_write_req;
+ } else {
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ if (obj->last_read_req[i])
+ req[n++] = obj->last_read_req[i];
+ }
+ for (i = 0; i < n; i++) {
+ ret = __i915_gem_object_sync(obj, to, req[i]);
+ if (ret)
+ return ret;
+ }
- return ret;
+ return 0;
}
static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
BUG_ON(obj->pages == NULL);
- ret = i915_gem_object_finish_gpu(obj);
+ ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
/* Continue on if we fail due to EIO, the GPU is hung so we
/* Since the unbound list is global, only move to that list if
* no more VMAs exist. */
if (list_empty(&obj->vma_list)) {
- /* Throw away the active reference before
- * moving to the unbound list. */
- i915_gem_object_retire(obj);
-
i915_gem_gtt_finish_object(obj);
list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
}
return ret;
}
+ WARN_ON(i915_verify_lists(dev));
return 0;
}
}
/**
- * Finds free space in the GTT aperture and binds the object there.
+ * Finds free space in the GTT aperture and binds the object or a view of it
+ * there.
*/
static struct i915_vma *
i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
struct i915_vma *vma;
int ret;
- if(WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
- return ERR_PTR(-EINVAL);
+ if (i915_is_ggtt(vm)) {
+ u32 view_size;
+
+ if (WARN_ON(!ggtt_view))
+ return ERR_PTR(-EINVAL);
+
+ view_size = i915_ggtt_view_size(obj, ggtt_view);
- fence_size = i915_gem_get_gtt_size(dev,
- obj->base.size,
- obj->tiling_mode);
- fence_alignment = i915_gem_get_gtt_alignment(dev,
- obj->base.size,
- obj->tiling_mode, true);
- unfenced_alignment =
- i915_gem_get_gtt_alignment(dev,
- obj->base.size,
- obj->tiling_mode, false);
+ fence_size = i915_gem_get_gtt_size(dev,
+ view_size,
+ obj->tiling_mode);
+ fence_alignment = i915_gem_get_gtt_alignment(dev,
+ view_size,
+ obj->tiling_mode,
+ true);
+ unfenced_alignment = i915_gem_get_gtt_alignment(dev,
+ view_size,
+ obj->tiling_mode,
+ false);
+ size = flags & PIN_MAPPABLE ? fence_size : view_size;
+ } else {
+ fence_size = i915_gem_get_gtt_size(dev,
+ obj->base.size,
+ obj->tiling_mode);
+ fence_alignment = i915_gem_get_gtt_alignment(dev,
+ obj->base.size,
+ obj->tiling_mode,
+ true);
+ unfenced_alignment =
+ i915_gem_get_gtt_alignment(dev,
+ obj->base.size,
+ obj->tiling_mode,
+ false);
+ size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
+ }
if (alignment == 0)
alignment = flags & PIN_MAPPABLE ? fence_alignment :
unfenced_alignment;
if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
- DRM_DEBUG("Invalid object alignment requested %u\n", alignment);
+ DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
+ ggtt_view ? ggtt_view->type : 0,
+ alignment);
return ERR_PTR(-EINVAL);
}
- size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
-
- /* If the object is bigger than the entire aperture, reject it early
- * before evicting everything in a vain attempt to find space.
+ /* If binding the object/GGTT view requires more space than the entire
+ * aperture has, reject it early before evicting everything in a vain
+ * attempt to find space.
*/
- if (obj->base.size > end) {
- DRM_DEBUG("Attempting to bind an object larger than the aperture: object=%zd > %s aperture=%lu\n",
- obj->base.size,
+ if (size > end) {
+ DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%u > %s aperture=%lu\n",
+ ggtt_view ? ggtt_view->type : 0,
+ size,
flags & PIN_MAPPABLE ? "mappable" : "total",
end);
return ERR_PTR(-E2BIG);
if (ret)
return ret;
- i915_gem_object_retire(obj);
-
/* Flush and acquire obj->pages so that we are coherent through
* direct access in memory with previous cached writes through
* shmemfs and that our cache domain tracking remains valid.
}
if (i915_gem_obj_bound_any(obj)) {
- ret = i915_gem_object_finish_gpu(obj);
+ ret = i915_gem_object_wait_rendering(obj, false);
if (ret)
return ret;
u32 old_read_domains, old_write_domain;
int ret;
- if (pipelined != i915_gem_request_get_ring(obj->last_read_req)) {
- ret = i915_gem_object_sync(obj, pipelined);
- if (ret)
- return ret;
- }
+ ret = i915_gem_object_sync(obj, pipelined);
+ if (ret)
+ return ret;
/* Mark the pin_display early so that we account for the
* display coherency whilst setting up the cache domains.
obj->pin_display--;
}
-int
-i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
-{
- int ret;
-
- if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
- return 0;
-
- ret = i915_gem_object_wait_rendering(obj, false);
- if (ret)
- return ret;
-
- /* Ensure that we invalidate the GPU's caches and TLBs. */
- obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
- return 0;
-}
-
/**
* Moves a single object to the CPU read, and possibly write domain.
*
if (ret)
return ret;
- i915_gem_object_retire(obj);
i915_gem_object_flush_gtt_write_domain(obj);
old_write_domain = obj->base.write_domain;
{
struct drm_i915_private *dev_priv = dev->dev_private;
struct drm_i915_file_private *file_priv = file->driver_priv;
- unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
+ unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
struct drm_i915_gem_request *request, *target = NULL;
unsigned reset_counter;
int ret;
return ret;
}
- if ((bound ^ vma->bound) & GLOBAL_BIND) {
+ if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
+ (bound ^ vma->bound) & GLOBAL_BIND) {
bool mappable, fenceable;
u32 fence_size, fence_alignment;
dev_priv->gtt.mappable_end);
obj->map_and_fenceable = mappable && fenceable;
- }
- WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+ WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
+ }
vma->pin_count++;
return 0;
* necessary flushes here.
*/
ret = i915_gem_object_flush_active(obj);
+ if (ret)
+ goto unref;
- args->busy = obj->active;
- if (obj->last_read_req) {
- struct intel_engine_cs *ring;
- BUILD_BUG_ON(I915_NUM_RINGS > 16);
- ring = i915_gem_request_get_ring(obj->last_read_req);
- args->busy |= intel_ring_flag(ring) << 16;
- }
+ BUILD_BUG_ON(I915_NUM_RINGS > 16);
+ args->busy = obj->active << 16;
+ if (obj->last_write_req)
+ args->busy |= obj->last_write_req->ring->id;
+unref:
drm_gem_object_unreference(&obj->base);
unlock:
mutex_unlock(&dev->struct_mutex);
void i915_gem_object_init(struct drm_i915_gem_object *obj,
const struct drm_i915_gem_object_ops *ops)
{
+ int i;
+
INIT_LIST_HEAD(&obj->global_list);
- INIT_LIST_HEAD(&obj->ring_list);
+ for (i = 0; i < I915_NUM_RINGS; i++)
+ INIT_LIST_HEAD(&obj->ring_list[i]);
INIT_LIST_HEAD(&obj->obj_exec_link);
INIT_LIST_HEAD(&obj->vma_list);
INIT_LIST_HEAD(&obj->batch_pool_link);
}
spin_unlock(&file_priv->mm.lock);
- if (!list_empty(&file_priv->rps_boost)) {
+ if (!list_empty(&file_priv->rps.link)) {
mutex_lock(&to_i915(dev)->rps.hw_lock);
- list_del(&file_priv->rps_boost);
+ list_del(&file_priv->rps.link);
mutex_unlock(&to_i915(dev)->rps.hw_lock);
}
}
file->driver_priv = file_priv;
file_priv->dev_priv = dev->dev_private;
file_priv->file = file;
- INIT_LIST_HEAD(&file_priv->rps_boost);
+ INIT_LIST_HEAD(&file_priv->rps.link);
spin_lock_init(&file_priv->mm.lock);
INIT_LIST_HEAD(&file_priv->mm.request_list);
bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
{
struct i915_vma *vma;
- list_for_each_entry(vma, &obj->vma_list, vma_link) {
- if (i915_is_ggtt(vma->vm) &&
- vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
- continue;
+ list_for_each_entry(vma, &obj->vma_list, vma_link)
if (vma->pin_count > 0)
return true;
- }
+
return false;
}