Merge series "regulator/mfd/clock: dt-bindings: Samsung S2M and S5M to dtschema"...
[linux-2.6-microblaze.git] / drivers / dma-buf / dma-fence-chain.c
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * fence-chain: chain fences together in a timeline
4  *
5  * Copyright (C) 2018 Advanced Micro Devices, Inc.
6  * Authors:
7  *      Christian König <christian.koenig@amd.com>
8  */
9
10 #include <linux/dma-fence-chain.h>
11
12 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence);
13
14 /**
15  * dma_fence_chain_get_prev - use RCU to get a reference to the previous fence
16  * @chain: chain node to get the previous node from
17  *
18  * Use dma_fence_get_rcu_safe to get a reference to the previous fence of the
19  * chain node.
20  */
21 static struct dma_fence *dma_fence_chain_get_prev(struct dma_fence_chain *chain)
22 {
23         struct dma_fence *prev;
24
25         rcu_read_lock();
26         prev = dma_fence_get_rcu_safe(&chain->prev);
27         rcu_read_unlock();
28         return prev;
29 }
30
31 /**
32  * dma_fence_chain_walk - chain walking function
33  * @fence: current chain node
34  *
35  * Walk the chain to the next node. Returns the next fence or NULL if we are at
36  * the end of the chain. Garbage collects chain nodes which are already
37  * signaled.
38  */
39 struct dma_fence *dma_fence_chain_walk(struct dma_fence *fence)
40 {
41         struct dma_fence_chain *chain, *prev_chain;
42         struct dma_fence *prev, *replacement, *tmp;
43
44         chain = to_dma_fence_chain(fence);
45         if (!chain) {
46                 dma_fence_put(fence);
47                 return NULL;
48         }
49
50         while ((prev = dma_fence_chain_get_prev(chain))) {
51
52                 prev_chain = to_dma_fence_chain(prev);
53                 if (prev_chain) {
54                         if (!dma_fence_is_signaled(prev_chain->fence))
55                                 break;
56
57                         replacement = dma_fence_chain_get_prev(prev_chain);
58                 } else {
59                         if (!dma_fence_is_signaled(prev))
60                                 break;
61
62                         replacement = NULL;
63                 }
64
65                 tmp = cmpxchg((struct dma_fence __force **)&chain->prev,
66                               prev, replacement);
67                 if (tmp == prev)
68                         dma_fence_put(tmp);
69                 else
70                         dma_fence_put(replacement);
71                 dma_fence_put(prev);
72         }
73
74         dma_fence_put(fence);
75         return prev;
76 }
77 EXPORT_SYMBOL(dma_fence_chain_walk);
78
79 /**
80  * dma_fence_chain_find_seqno - find fence chain node by seqno
81  * @pfence: pointer to the chain node where to start
82  * @seqno: the sequence number to search for
83  *
84  * Advance the fence pointer to the chain node which will signal this sequence
85  * number. If no sequence number is provided then this is a no-op.
86  *
87  * Returns EINVAL if the fence is not a chain node or the sequence number has
88  * not yet advanced far enough.
89  */
90 int dma_fence_chain_find_seqno(struct dma_fence **pfence, uint64_t seqno)
91 {
92         struct dma_fence_chain *chain;
93
94         if (!seqno)
95                 return 0;
96
97         chain = to_dma_fence_chain(*pfence);
98         if (!chain || chain->base.seqno < seqno)
99                 return -EINVAL;
100
101         dma_fence_chain_for_each(*pfence, &chain->base) {
102                 if ((*pfence)->context != chain->base.context ||
103                     to_dma_fence_chain(*pfence)->prev_seqno < seqno)
104                         break;
105         }
106         dma_fence_put(&chain->base);
107
108         return 0;
109 }
110 EXPORT_SYMBOL(dma_fence_chain_find_seqno);
111
112 static const char *dma_fence_chain_get_driver_name(struct dma_fence *fence)
113 {
114         return "dma_fence_chain";
115 }
116
117 static const char *dma_fence_chain_get_timeline_name(struct dma_fence *fence)
118 {
119         return "unbound";
120 }
121
122 static void dma_fence_chain_irq_work(struct irq_work *work)
123 {
124         struct dma_fence_chain *chain;
125
126         chain = container_of(work, typeof(*chain), work);
127
128         /* Try to rearm the callback */
129         if (!dma_fence_chain_enable_signaling(&chain->base))
130                 /* Ok, we are done. No more unsignaled fences left */
131                 dma_fence_signal(&chain->base);
132         dma_fence_put(&chain->base);
133 }
134
135 static void dma_fence_chain_cb(struct dma_fence *f, struct dma_fence_cb *cb)
136 {
137         struct dma_fence_chain *chain;
138
139         chain = container_of(cb, typeof(*chain), cb);
140         init_irq_work(&chain->work, dma_fence_chain_irq_work);
141         irq_work_queue(&chain->work);
142         dma_fence_put(f);
143 }
144
145 static bool dma_fence_chain_enable_signaling(struct dma_fence *fence)
146 {
147         struct dma_fence_chain *head = to_dma_fence_chain(fence);
148
149         dma_fence_get(&head->base);
150         dma_fence_chain_for_each(fence, &head->base) {
151                 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
152                 struct dma_fence *f = chain ? chain->fence : fence;
153
154                 dma_fence_get(f);
155                 if (!dma_fence_add_callback(f, &head->cb, dma_fence_chain_cb)) {
156                         dma_fence_put(fence);
157                         return true;
158                 }
159                 dma_fence_put(f);
160         }
161         dma_fence_put(&head->base);
162         return false;
163 }
164
165 static bool dma_fence_chain_signaled(struct dma_fence *fence)
166 {
167         dma_fence_chain_for_each(fence, fence) {
168                 struct dma_fence_chain *chain = to_dma_fence_chain(fence);
169                 struct dma_fence *f = chain ? chain->fence : fence;
170
171                 if (!dma_fence_is_signaled(f)) {
172                         dma_fence_put(fence);
173                         return false;
174                 }
175         }
176
177         return true;
178 }
179
180 static void dma_fence_chain_release(struct dma_fence *fence)
181 {
182         struct dma_fence_chain *chain = to_dma_fence_chain(fence);
183         struct dma_fence *prev;
184
185         /* Manually unlink the chain as much as possible to avoid recursion
186          * and potential stack overflow.
187          */
188         while ((prev = rcu_dereference_protected(chain->prev, true))) {
189                 struct dma_fence_chain *prev_chain;
190
191                 if (kref_read(&prev->refcount) > 1)
192                        break;
193
194                 prev_chain = to_dma_fence_chain(prev);
195                 if (!prev_chain)
196                         break;
197
198                 /* No need for atomic operations since we hold the last
199                  * reference to prev_chain.
200                  */
201                 chain->prev = prev_chain->prev;
202                 RCU_INIT_POINTER(prev_chain->prev, NULL);
203                 dma_fence_put(prev);
204         }
205         dma_fence_put(prev);
206
207         dma_fence_put(chain->fence);
208         dma_fence_free(fence);
209 }
210
211 const struct dma_fence_ops dma_fence_chain_ops = {
212         .use_64bit_seqno = true,
213         .get_driver_name = dma_fence_chain_get_driver_name,
214         .get_timeline_name = dma_fence_chain_get_timeline_name,
215         .enable_signaling = dma_fence_chain_enable_signaling,
216         .signaled = dma_fence_chain_signaled,
217         .release = dma_fence_chain_release,
218 };
219 EXPORT_SYMBOL(dma_fence_chain_ops);
220
221 /**
222  * dma_fence_chain_init - initialize a fence chain
223  * @chain: the chain node to initialize
224  * @prev: the previous fence
225  * @fence: the current fence
226  * @seqno: the sequence number to use for the fence chain
227  *
228  * Initialize a new chain node and either start a new chain or add the node to
229  * the existing chain of the previous fence.
230  */
231 void dma_fence_chain_init(struct dma_fence_chain *chain,
232                           struct dma_fence *prev,
233                           struct dma_fence *fence,
234                           uint64_t seqno)
235 {
236         struct dma_fence_chain *prev_chain = to_dma_fence_chain(prev);
237         uint64_t context;
238
239         spin_lock_init(&chain->lock);
240         rcu_assign_pointer(chain->prev, prev);
241         chain->fence = fence;
242         chain->prev_seqno = 0;
243
244         /* Try to reuse the context of the previous chain node. */
245         if (prev_chain && __dma_fence_is_later(seqno, prev->seqno, prev->ops)) {
246                 context = prev->context;
247                 chain->prev_seqno = prev->seqno;
248         } else {
249                 context = dma_fence_context_alloc(1);
250                 /* Make sure that we always have a valid sequence number. */
251                 if (prev_chain)
252                         seqno = max(prev->seqno, seqno);
253         }
254
255         dma_fence_init(&chain->base, &dma_fence_chain_ops,
256                        &chain->lock, context, seqno);
257 }
258 EXPORT_SYMBOL(dma_fence_chain_init);