1 // SPDX-License-Identifier: GPL-2.0-only
3 * Tegra host1x Command DMA
5 * Copyright (c) 2010-2013, NVIDIA Corporation.
9 #include <asm/cacheflush.h>
10 #include <linux/device.h>
11 #include <linux/dma-mapping.h>
12 #include <linux/host1x.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel.h>
15 #include <linux/kfifo.h>
16 #include <linux/slab.h>
17 #include <trace/events/host1x.h>
28 * The push buffer is a circular array of words to be fetched by command DMA.
29 * Note that it works slightly differently to the sync queue; fence == pos
30 * means that the push buffer is full, not empty.
34 * Typically the commands written into the push buffer are a pair of words. We
35 * use slots to represent each of these pairs and to simplify things. Note the
36 * strange number of slots allocated here. 512 slots will fit exactly within a
37 * single memory page. We also need one additional word at the end of the push
38 * buffer for the RESTART opcode that will instruct the CDMA to jump back to
39 * the beginning of the push buffer. With 512 slots, this means that we'll use
40 * 2 memory pages and waste 4092 bytes of the second page that will never be
43 #define HOST1X_PUSHBUFFER_SLOTS 511
46 * Clean up push buffer resources
48 static void host1x_pushbuffer_destroy(struct push_buffer *pb)
50 struct host1x_cdma *cdma = pb_to_cdma(pb);
51 struct host1x *host1x = cdma_to_host1x(cdma);
57 iommu_unmap(host1x->domain, pb->dma, pb->alloc_size);
58 free_iova(&host1x->iova, iova_pfn(&host1x->iova, pb->dma));
61 dma_free_wc(host1x->dev, pb->alloc_size, pb->mapped, pb->phys);
68 * Init push buffer resources
70 static int host1x_pushbuffer_init(struct push_buffer *pb)
72 struct host1x_cdma *cdma = pb_to_cdma(pb);
73 struct host1x *host1x = cdma_to_host1x(cdma);
80 pb->size = HOST1X_PUSHBUFFER_SLOTS * 8;
84 /* initialize buffer pointers */
85 pb->fence = pb->size - 8;
91 size = iova_align(&host1x->iova, size);
93 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
98 shift = iova_shift(&host1x->iova);
99 alloc = alloc_iova(&host1x->iova, size >> shift,
100 host1x->iova_end >> shift, true);
106 pb->dma = iova_dma_addr(&host1x->iova, alloc);
107 err = iommu_map(host1x->domain, pb->dma, pb->phys, size,
110 goto iommu_free_iova;
112 pb->mapped = dma_alloc_wc(host1x->dev, size, &pb->phys,
120 pb->alloc_size = size;
122 host1x_hw_pushbuffer_init(host1x, pb);
127 __free_iova(&host1x->iova, alloc);
129 dma_free_wc(host1x->dev, size, pb->mapped, pb->phys);
135 * Push two words to the push buffer
136 * Caller must ensure push buffer is not full
138 static void host1x_pushbuffer_push(struct push_buffer *pb, u32 op1, u32 op2)
140 u32 *p = (u32 *)((void *)pb->mapped + pb->pos);
142 WARN_ON(pb->pos == pb->fence);
147 if (pb->pos >= pb->size)
152 * Pop a number of two word slots from the push buffer
153 * Caller must ensure push buffer is not empty
155 static void host1x_pushbuffer_pop(struct push_buffer *pb, unsigned int slots)
157 /* Advance the next write position */
158 pb->fence += slots * 8;
160 if (pb->fence >= pb->size)
161 pb->fence -= pb->size;
165 * Return the number of two word slots free in the push buffer
167 static u32 host1x_pushbuffer_space(struct push_buffer *pb)
169 unsigned int fence = pb->fence;
171 if (pb->fence < pb->pos)
174 return (fence - pb->pos) / 8;
178 * Sleep (if necessary) until the requested event happens
179 * - CDMA_EVENT_SYNC_QUEUE_EMPTY : sync queue is completely empty.
181 * - CDMA_EVENT_PUSH_BUFFER_SPACE : there is space in the push buffer
182 * - Return the amount of space (> 0)
183 * Must be called with the cdma lock held.
185 unsigned int host1x_cdma_wait_locked(struct host1x_cdma *cdma,
186 enum cdma_event event)
189 struct push_buffer *pb = &cdma->push_buffer;
193 case CDMA_EVENT_SYNC_QUEUE_EMPTY:
194 space = list_empty(&cdma->sync_queue) ? 1 : 0;
197 case CDMA_EVENT_PUSH_BUFFER_SPACE:
198 space = host1x_pushbuffer_space(pb);
209 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
212 /* If somebody has managed to already start waiting, yield */
213 if (cdma->event != CDMA_EVENT_NONE) {
214 mutex_unlock(&cdma->lock);
216 mutex_lock(&cdma->lock);
222 mutex_unlock(&cdma->lock);
223 wait_for_completion(&cdma->complete);
224 mutex_lock(&cdma->lock);
231 * Sleep (if necessary) until the push buffer has enough free space.
233 * Must be called with the cdma lock held.
235 static int host1x_cdma_wait_pushbuffer_space(struct host1x *host1x,
236 struct host1x_cdma *cdma,
240 struct push_buffer *pb = &cdma->push_buffer;
243 space = host1x_pushbuffer_space(pb);
247 trace_host1x_wait_cdma(dev_name(cdma_to_channel(cdma)->dev),
248 CDMA_EVENT_PUSH_BUFFER_SPACE);
250 host1x_hw_cdma_flush(host1x, cdma);
252 /* If somebody has managed to already start waiting, yield */
253 if (cdma->event != CDMA_EVENT_NONE) {
254 mutex_unlock(&cdma->lock);
256 mutex_lock(&cdma->lock);
260 cdma->event = CDMA_EVENT_PUSH_BUFFER_SPACE;
262 mutex_unlock(&cdma->lock);
263 wait_for_completion(&cdma->complete);
264 mutex_lock(&cdma->lock);
270 * Start timer that tracks the time spent by the job.
271 * Must be called with the cdma lock held.
273 static void cdma_start_timer_locked(struct host1x_cdma *cdma,
274 struct host1x_job *job)
276 if (cdma->timeout.client) {
277 /* timer already started */
281 cdma->timeout.client = job->client;
282 cdma->timeout.syncpt = job->syncpt;
283 cdma->timeout.syncpt_val = job->syncpt_end;
284 cdma->timeout.start_ktime = ktime_get();
286 schedule_delayed_work(&cdma->timeout.wq,
287 msecs_to_jiffies(job->timeout));
291 * Stop timer when a buffer submission completes.
292 * Must be called with the cdma lock held.
294 static void stop_cdma_timer_locked(struct host1x_cdma *cdma)
296 cancel_delayed_work(&cdma->timeout.wq);
297 cdma->timeout.client = NULL;
301 * For all sync queue entries that have already finished according to the
302 * current sync point registers:
303 * - unpin & unref their mems
304 * - pop their push buffer slots
305 * - remove them from the sync queue
306 * This is normally called from the host code's worker thread, but can be
307 * called manually if necessary.
308 * Must be called with the cdma lock held.
310 static void update_cdma_locked(struct host1x_cdma *cdma)
313 struct host1x_job *job, *n;
315 /* If CDMA is stopped, queue is cleared and we can return */
320 * Walk the sync queue, reading the sync point registers as necessary,
321 * to consume as many sync queue entries as possible without blocking
323 list_for_each_entry_safe(job, n, &cdma->sync_queue, list) {
324 struct host1x_syncpt *sp = job->syncpt;
326 /* Check whether this syncpt has completed, and bail if not */
327 if (!host1x_syncpt_is_expired(sp, job->syncpt_end)) {
328 /* Start timer on next pending syncpt */
330 cdma_start_timer_locked(cdma, job);
335 /* Cancel timeout, when a buffer completes */
336 if (cdma->timeout.client)
337 stop_cdma_timer_locked(cdma);
339 /* Unpin the memory */
340 host1x_job_unpin(job);
342 /* Pop push buffer slots */
343 if (job->num_slots) {
344 struct push_buffer *pb = &cdma->push_buffer;
346 host1x_pushbuffer_pop(pb, job->num_slots);
348 if (cdma->event == CDMA_EVENT_PUSH_BUFFER_SPACE)
352 list_del(&job->list);
356 if (cdma->event == CDMA_EVENT_SYNC_QUEUE_EMPTY &&
357 list_empty(&cdma->sync_queue))
361 cdma->event = CDMA_EVENT_NONE;
362 complete(&cdma->complete);
366 void host1x_cdma_update_sync_queue(struct host1x_cdma *cdma,
369 struct host1x *host1x = cdma_to_host1x(cdma);
370 u32 restart_addr, syncpt_incrs, syncpt_val;
371 struct host1x_job *job, *next_job = NULL;
373 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
375 dev_dbg(dev, "%s: starting cleanup (thresh %d)\n",
376 __func__, syncpt_val);
379 * Move the sync_queue read pointer to the first entry that hasn't
380 * completed based on the current HW syncpt value. It's likely there
381 * won't be any (i.e. we're still at the head), but covers the case
382 * where a syncpt incr happens just prior/during the teardown.
385 dev_dbg(dev, "%s: skip completed buffers still in sync_queue\n",
388 list_for_each_entry(job, &cdma->sync_queue, list) {
389 if (syncpt_val < job->syncpt_end) {
391 if (!list_is_last(&job->list, &cdma->sync_queue))
392 next_job = list_next_entry(job, list);
397 host1x_job_dump(dev, job);
400 /* all jobs have been completed */
406 * Increment with CPU the remaining syncpts of a partially executed job.
408 * CDMA will continue execution starting with the next job or will get
412 restart_addr = next_job->first_get;
414 restart_addr = cdma->last_pos;
416 /* do CPU increments for the remaining syncpts */
418 dev_dbg(dev, "%s: perform CPU incr on pending buffers\n",
421 /* won't need a timeout when replayed */
424 syncpt_incrs = job->syncpt_end - syncpt_val;
425 dev_dbg(dev, "%s: CPU incr (%d)\n", __func__, syncpt_incrs);
427 host1x_job_dump(dev, job);
429 /* safe to use CPU to incr syncpts */
430 host1x_hw_cdma_timeout_cpu_incr(host1x, cdma, job->first_get,
431 syncpt_incrs, job->syncpt_end,
434 dev_dbg(dev, "%s: finished sync_queue modification\n",
438 /* roll back DMAGET and start up channel again */
439 host1x_hw_cdma_resume(host1x, cdma, restart_addr);
445 int host1x_cdma_init(struct host1x_cdma *cdma)
449 mutex_init(&cdma->lock);
450 init_completion(&cdma->complete);
452 INIT_LIST_HEAD(&cdma->sync_queue);
454 cdma->event = CDMA_EVENT_NONE;
455 cdma->running = false;
456 cdma->torndown = false;
458 err = host1x_pushbuffer_init(&cdma->push_buffer);
468 int host1x_cdma_deinit(struct host1x_cdma *cdma)
470 struct push_buffer *pb = &cdma->push_buffer;
471 struct host1x *host1x = cdma_to_host1x(cdma);
474 pr_warn("%s: CDMA still running\n", __func__);
478 host1x_pushbuffer_destroy(pb);
479 host1x_hw_cdma_timeout_destroy(host1x, cdma);
485 * Begin a cdma submit
487 int host1x_cdma_begin(struct host1x_cdma *cdma, struct host1x_job *job)
489 struct host1x *host1x = cdma_to_host1x(cdma);
491 mutex_lock(&cdma->lock);
494 /* init state on first submit with timeout value */
495 if (!cdma->timeout.initialized) {
498 err = host1x_hw_cdma_timeout_init(host1x, cdma);
500 mutex_unlock(&cdma->lock);
507 host1x_hw_cdma_start(host1x, cdma);
509 cdma->slots_free = 0;
510 cdma->slots_used = 0;
511 cdma->first_get = cdma->push_buffer.pos;
513 trace_host1x_cdma_begin(dev_name(job->channel->dev));
518 * Push two words into a push buffer slot
519 * Blocks as necessary if the push buffer is full.
521 void host1x_cdma_push(struct host1x_cdma *cdma, u32 op1, u32 op2)
523 struct host1x *host1x = cdma_to_host1x(cdma);
524 struct push_buffer *pb = &cdma->push_buffer;
525 u32 slots_free = cdma->slots_free;
527 if (host1x_debug_trace_cmdbuf)
528 trace_host1x_cdma_push(dev_name(cdma_to_channel(cdma)->dev),
531 if (slots_free == 0) {
532 host1x_hw_cdma_flush(host1x, cdma);
533 slots_free = host1x_cdma_wait_locked(cdma,
534 CDMA_EVENT_PUSH_BUFFER_SPACE);
537 cdma->slots_free = slots_free - 1;
539 host1x_pushbuffer_push(pb, op1, op2);
543 * Push four words into two consecutive push buffer slots. Note that extra
544 * care needs to be taken not to split the two slots across the end of the
545 * push buffer. Otherwise the RESTART opcode at the end of the push buffer
546 * that ensures processing will restart at the beginning will break up the
549 * Blocks as necessary if the push buffer is full.
551 void host1x_cdma_push_wide(struct host1x_cdma *cdma, u32 op1, u32 op2,
554 struct host1x_channel *channel = cdma_to_channel(cdma);
555 struct host1x *host1x = cdma_to_host1x(cdma);
556 struct push_buffer *pb = &cdma->push_buffer;
557 unsigned int needed = 2, extra = 0, i;
558 unsigned int space = cdma->slots_free;
560 if (host1x_debug_trace_cmdbuf)
561 trace_host1x_cdma_push_wide(dev_name(channel->dev), op1, op2,
564 /* compute number of extra slots needed for padding */
565 if (pb->pos + 16 > pb->size) {
566 extra = (pb->size - pb->pos) / 8;
570 host1x_cdma_wait_pushbuffer_space(host1x, cdma, needed);
571 space = host1x_pushbuffer_space(pb);
573 cdma->slots_free = space - needed;
574 cdma->slots_used += needed;
577 * Note that we rely on the fact that this is only used to submit wide
578 * gather opcodes, which consist of 3 words, and they are padded with
579 * a NOP to avoid having to deal with fractional slots (a slot always
580 * represents 2 words). The fourth opcode passed to this function will
581 * therefore always be a NOP.
583 * This works around a slight ambiguity when it comes to opcodes. For
584 * all current host1x incarnations the NOP opcode uses the exact same
585 * encoding (0x20000000), so we could hard-code the value here, but a
586 * new incarnation may change it and break that assumption.
588 for (i = 0; i < extra; i++)
589 host1x_pushbuffer_push(pb, op4, op4);
591 host1x_pushbuffer_push(pb, op1, op2);
592 host1x_pushbuffer_push(pb, op3, op4);
597 * Kick off DMA, add job to the sync queue, and a number of slots to be freed
598 * from the pushbuffer. The handles for a submit must all be pinned at the same
599 * time, but they can be unpinned in smaller chunks.
601 void host1x_cdma_end(struct host1x_cdma *cdma,
602 struct host1x_job *job)
604 struct host1x *host1x = cdma_to_host1x(cdma);
605 bool idle = list_empty(&cdma->sync_queue);
607 host1x_hw_cdma_flush(host1x, cdma);
609 job->first_get = cdma->first_get;
610 job->num_slots = cdma->slots_used;
612 list_add_tail(&job->list, &cdma->sync_queue);
614 /* start timer on idle -> active transitions */
615 if (job->timeout && idle)
616 cdma_start_timer_locked(cdma, job);
618 trace_host1x_cdma_end(dev_name(job->channel->dev));
619 mutex_unlock(&cdma->lock);
623 * Update cdma state according to current sync point values
625 void host1x_cdma_update(struct host1x_cdma *cdma)
627 mutex_lock(&cdma->lock);
628 update_cdma_locked(cdma);
629 mutex_unlock(&cdma->lock);