1 // SPDX-License-Identifier: MIT
4 * Copyright © 2019 Intel Corporation
7 #include <linux/delay.h>
8 #include <linux/dma-fence.h>
9 #include <linux/dma-fence-chain.h>
10 #include <linux/kernel.h>
11 #include <linux/kthread.h>
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 #include <linux/random.h>
20 #define CHAIN_SZ (4 << 10)
22 static struct kmem_cache *slab_fences;
24 static inline struct mock_fence {
25 struct dma_fence base;
27 } *to_mock_fence(struct dma_fence *f) {
28 return container_of(f, struct mock_fence, base);
31 static const char *mock_name(struct dma_fence *f)
36 static void mock_fence_release(struct dma_fence *f)
38 kmem_cache_free(slab_fences, to_mock_fence(f));
41 static const struct dma_fence_ops mock_ops = {
42 .get_driver_name = mock_name,
43 .get_timeline_name = mock_name,
44 .release = mock_fence_release,
47 static struct dma_fence *mock_fence(void)
51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
55 spin_lock_init(&f->lock);
56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
61 static struct dma_fence *mock_chain(struct dma_fence *prev,
62 struct dma_fence *fence,
65 struct dma_fence_chain *f;
67 f = dma_fence_chain_alloc();
71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence),
77 static int sanitycheck(void *arg)
79 struct dma_fence *f, *chain;
86 chain = mock_chain(NULL, f, 1);
99 unsigned int chain_length;
100 struct dma_fence **fences;
101 struct dma_fence **chains;
103 struct dma_fence *tail;
106 static uint64_t seqno_inc(unsigned int i)
111 static int fence_chains_init(struct fence_chains *fc, unsigned int count,
112 uint64_t (*seqno_fn)(unsigned int))
117 fc->chains = kvmalloc_array(count, sizeof(*fc->chains),
118 GFP_KERNEL | __GFP_ZERO);
122 fc->fences = kvmalloc_array(count, sizeof(*fc->fences),
123 GFP_KERNEL | __GFP_ZERO);
130 for (i = 0; i < count; i++) {
131 fc->fences[i] = mock_fence();
132 if (!fc->fences[i]) {
137 fc->chains[i] = mock_chain(fc->tail,
140 if (!fc->chains[i]) {
145 fc->tail = fc->chains[i];
148 fc->chain_length = i;
152 for (i = 0; i < count; i++) {
153 dma_fence_put(fc->fences[i]);
154 dma_fence_put(fc->chains[i]);
162 static void fence_chains_fini(struct fence_chains *fc)
166 for (i = 0; i < fc->chain_length; i++) {
167 dma_fence_signal(fc->fences[i]);
168 dma_fence_put(fc->fences[i]);
172 for (i = 0; i < fc->chain_length; i++)
173 dma_fence_put(fc->chains[i]);
177 static int find_seqno(void *arg)
179 struct fence_chains fc;
180 struct dma_fence *fence;
184 err = fence_chains_init(&fc, 64, seqno_inc);
188 fence = dma_fence_get(fc.tail);
189 err = dma_fence_chain_find_seqno(&fence, 0);
190 dma_fence_put(fence);
192 pr_err("Reported %d for find_seqno(0)!\n", err);
196 for (i = 0; i < fc.chain_length; i++) {
197 fence = dma_fence_get(fc.tail);
198 err = dma_fence_chain_find_seqno(&fence, i + 1);
199 dma_fence_put(fence);
201 pr_err("Reported %d for find_seqno(%d:%d)!\n",
202 err, fc.chain_length + 1, i + 1);
205 if (fence != fc.chains[i]) {
206 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
207 fc.chain_length + 1, i + 1);
212 dma_fence_get(fence);
213 err = dma_fence_chain_find_seqno(&fence, i + 1);
214 dma_fence_put(fence);
216 pr_err("Error reported for finding self\n");
219 if (fence != fc.chains[i]) {
220 pr_err("Incorrect fence reported by find self\n");
225 dma_fence_get(fence);
226 err = dma_fence_chain_find_seqno(&fence, i + 2);
227 dma_fence_put(fence);
229 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
235 dma_fence_get(fence);
236 err = dma_fence_chain_find_seqno(&fence, i);
237 dma_fence_put(fence);
239 pr_err("Error reported for previous fence!\n");
242 if (i > 0 && fence != fc.chains[i - 1]) {
243 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
251 fence_chains_fini(&fc);
255 static int find_signaled(void *arg)
257 struct fence_chains fc;
258 struct dma_fence *fence;
261 err = fence_chains_init(&fc, 2, seqno_inc);
265 dma_fence_signal(fc.fences[0]);
267 fence = dma_fence_get(fc.tail);
268 err = dma_fence_chain_find_seqno(&fence, 1);
269 dma_fence_put(fence);
271 pr_err("Reported %d for find_seqno()!\n", err);
275 if (fence && fence != fc.chains[0]) {
276 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
279 dma_fence_get(fence);
280 err = dma_fence_chain_find_seqno(&fence, 1);
281 dma_fence_put(fence);
283 pr_err("Reported %d for finding self!\n", err);
289 fence_chains_fini(&fc);
293 static int find_out_of_order(void *arg)
295 struct fence_chains fc;
296 struct dma_fence *fence;
299 err = fence_chains_init(&fc, 3, seqno_inc);
303 dma_fence_signal(fc.fences[1]);
305 fence = dma_fence_get(fc.tail);
306 err = dma_fence_chain_find_seqno(&fence, 2);
307 dma_fence_put(fence);
309 pr_err("Reported %d for find_seqno()!\n", err);
314 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior
315 * of the dma-fence-chain is to make us wait for all the fences up to
316 * the point we want. Since fence 1 is still not signaled, this what
317 * we should get as fence to wait upon (fence 2 being garbage
318 * collected during the traversal of the chain).
320 if (fence != fc.chains[0]) {
321 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
322 fence ? fence->seqno : 0);
328 fence_chains_fini(&fc);
332 static uint64_t seqno_inc2(unsigned int i)
337 static int find_gap(void *arg)
339 struct fence_chains fc;
340 struct dma_fence *fence;
344 err = fence_chains_init(&fc, 64, seqno_inc2);
348 for (i = 0; i < fc.chain_length; i++) {
349 fence = dma_fence_get(fc.tail);
350 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
351 dma_fence_put(fence);
353 pr_err("Reported %d for find_seqno(%d:%d)!\n",
354 err, fc.chain_length + 1, 2 * i + 1);
357 if (fence != fc.chains[i]) {
358 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
366 dma_fence_get(fence);
367 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
368 dma_fence_put(fence);
370 pr_err("Error reported for finding self\n");
373 if (fence != fc.chains[i]) {
374 pr_err("Incorrect fence reported by find self\n");
381 fence_chains_fini(&fc);
386 struct fence_chains fc;
390 static int __find_race(void *arg)
392 struct find_race *data = arg;
395 while (!kthread_should_stop()) {
396 struct dma_fence *fence = dma_fence_get(data->fc.tail);
399 seqno = prandom_u32_max(data->fc.chain_length) + 1;
401 err = dma_fence_chain_find_seqno(&fence, seqno);
403 pr_err("Failed to find fence seqno:%d\n",
405 dma_fence_put(fence);
412 * We can only find ourselves if we are on fence we were
415 if (fence->seqno == seqno) {
416 err = dma_fence_chain_find_seqno(&fence, seqno);
418 pr_err("Reported an invalid fence for find-self:%d\n",
420 dma_fence_put(fence);
425 dma_fence_put(fence);
428 seqno = prandom_u32_max(data->fc.chain_length - 1);
429 dma_fence_signal(data->fc.fences[seqno]);
433 if (atomic_dec_and_test(&data->children))
434 wake_up_var(&data->children);
438 static int find_race(void *arg)
440 struct find_race data;
441 int ncpus = num_online_cpus();
442 struct task_struct **threads;
447 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
451 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);
457 atomic_set(&data.children, 0);
458 for (i = 0; i < ncpus; i++) {
459 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
460 if (IS_ERR(threads[i])) {
464 atomic_inc(&data.children);
465 get_task_struct(threads[i]);
468 wait_var_event_timeout(&data.children,
469 !atomic_read(&data.children),
472 for (i = 0; i < ncpus; i++) {
475 ret = kthread_stop(threads[i]);
478 put_task_struct(threads[i]);
483 for (i = 0; i < data.fc.chain_length; i++)
484 if (dma_fence_is_signaled(data.fc.fences[i]))
486 pr_info("Completed %lu cycles\n", count);
489 fence_chains_fini(&data.fc);
493 static int signal_forward(void *arg)
495 struct fence_chains fc;
499 err = fence_chains_init(&fc, 64, seqno_inc);
503 for (i = 0; i < fc.chain_length; i++) {
504 dma_fence_signal(fc.fences[i]);
506 if (!dma_fence_is_signaled(fc.chains[i])) {
507 pr_err("chain[%d] not signaled!\n", i);
512 if (i + 1 < fc.chain_length &&
513 dma_fence_is_signaled(fc.chains[i + 1])) {
514 pr_err("chain[%d] is signaled!\n", i);
521 fence_chains_fini(&fc);
525 static int signal_backward(void *arg)
527 struct fence_chains fc;
531 err = fence_chains_init(&fc, 64, seqno_inc);
535 for (i = fc.chain_length; i--; ) {
536 dma_fence_signal(fc.fences[i]);
538 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
539 pr_err("chain[%d] is signaled!\n", i);
545 for (i = 0; i < fc.chain_length; i++) {
546 if (!dma_fence_is_signaled(fc.chains[i])) {
547 pr_err("chain[%d] was not signaled!\n", i);
554 fence_chains_fini(&fc);
558 static int __wait_fence_chains(void *arg)
560 struct fence_chains *fc = arg;
562 if (dma_fence_wait(fc->tail, false))
568 static int wait_forward(void *arg)
570 struct fence_chains fc;
571 struct task_struct *tsk;
575 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
579 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
584 get_task_struct(tsk);
587 for (i = 0; i < fc.chain_length; i++)
588 dma_fence_signal(fc.fences[i]);
590 err = kthread_stop(tsk);
591 put_task_struct(tsk);
594 fence_chains_fini(&fc);
598 static int wait_backward(void *arg)
600 struct fence_chains fc;
601 struct task_struct *tsk;
605 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
609 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
614 get_task_struct(tsk);
617 for (i = fc.chain_length; i--; )
618 dma_fence_signal(fc.fences[i]);
620 err = kthread_stop(tsk);
621 put_task_struct(tsk);
624 fence_chains_fini(&fc);
628 static void randomise_fences(struct fence_chains *fc)
630 unsigned int count = fc->chain_length;
632 /* Fisher-Yates shuffle courtesy of Knuth */
636 swp = prandom_u32_max(count + 1);
640 swap(fc->fences[count], fc->fences[swp]);
644 static int wait_random(void *arg)
646 struct fence_chains fc;
647 struct task_struct *tsk;
651 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
655 randomise_fences(&fc);
657 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
662 get_task_struct(tsk);
665 for (i = 0; i < fc.chain_length; i++)
666 dma_fence_signal(fc.fences[i]);
668 err = kthread_stop(tsk);
669 put_task_struct(tsk);
672 fence_chains_fini(&fc);
676 int dma_fence_chain(void)
678 static const struct subtest tests[] = {
679 SUBTEST(sanitycheck),
681 SUBTEST(find_signaled),
682 SUBTEST(find_out_of_order),
685 SUBTEST(signal_forward),
686 SUBTEST(signal_backward),
687 SUBTEST(wait_forward),
688 SUBTEST(wait_backward),
689 SUBTEST(wait_random),
693 pr_info("sizeof(dma_fence_chain)=%zu\n",
694 sizeof(struct dma_fence_chain));
696 slab_fences = KMEM_CACHE(mock_fence,
697 SLAB_TYPESAFE_BY_RCU |
702 ret = subtests(tests, NULL);
704 kmem_cache_destroy(slab_fences);