interconnect: qcom: icc-rpm: Fix peak rate calculation
[linux-2.6-microblaze.git] / lib / closure.c
1 // SPDX-License-Identifier: GPL-2.0
2 /*
3  * Asynchronous refcounty things
4  *
5  * Copyright 2010, 2011 Kent Overstreet <kent.overstreet@gmail.com>
6  * Copyright 2012 Google, Inc.
7  */
8
9 #include <linux/closure.h>
10 #include <linux/debugfs.h>
11 #include <linux/export.h>
12 #include <linux/rcupdate.h>
13 #include <linux/seq_file.h>
14 #include <linux/sched/debug.h>
15
16 static inline void closure_put_after_sub(struct closure *cl, int flags)
17 {
18         int r = flags & CLOSURE_REMAINING_MASK;
19
20         BUG_ON(flags & CLOSURE_GUARD_MASK);
21         BUG_ON(!r && (flags & ~CLOSURE_DESTRUCTOR));
22
23         if (!r) {
24                 smp_acquire__after_ctrl_dep();
25
26                 cl->closure_get_happened = false;
27
28                 if (cl->fn && !(flags & CLOSURE_DESTRUCTOR)) {
29                         atomic_set(&cl->remaining,
30                                    CLOSURE_REMAINING_INITIALIZER);
31                         closure_queue(cl);
32                 } else {
33                         struct closure *parent = cl->parent;
34                         closure_fn *destructor = cl->fn;
35
36                         closure_debug_destroy(cl);
37
38                         if (destructor)
39                                 destructor(cl);
40
41                         if (parent)
42                                 closure_put(parent);
43                 }
44         }
45 }
46
47 /* For clearing flags with the same atomic op as a put */
48 void closure_sub(struct closure *cl, int v)
49 {
50         closure_put_after_sub(cl, atomic_sub_return_release(v, &cl->remaining));
51 }
52 EXPORT_SYMBOL(closure_sub);
53
54 /*
55  * closure_put - decrement a closure's refcount
56  */
57 void closure_put(struct closure *cl)
58 {
59         closure_put_after_sub(cl, atomic_dec_return_release(&cl->remaining));
60 }
61 EXPORT_SYMBOL(closure_put);
62
63 /*
64  * closure_wake_up - wake up all closures on a wait list, without memory barrier
65  */
66 void __closure_wake_up(struct closure_waitlist *wait_list)
67 {
68         struct llist_node *list;
69         struct closure *cl, *t;
70         struct llist_node *reverse = NULL;
71
72         list = llist_del_all(&wait_list->list);
73
74         /* We first reverse the list to preserve FIFO ordering and fairness */
75         reverse = llist_reverse_order(list);
76
77         /* Then do the wakeups */
78         llist_for_each_entry_safe(cl, t, reverse, list) {
79                 closure_set_waiting(cl, 0);
80                 closure_sub(cl, CLOSURE_WAITING + 1);
81         }
82 }
83 EXPORT_SYMBOL(__closure_wake_up);
84
85 /**
86  * closure_wait - add a closure to a waitlist
87  * @waitlist: will own a ref on @cl, which will be released when
88  * closure_wake_up() is called on @waitlist.
89  * @cl: closure pointer.
90  *
91  */
92 bool closure_wait(struct closure_waitlist *waitlist, struct closure *cl)
93 {
94         if (atomic_read(&cl->remaining) & CLOSURE_WAITING)
95                 return false;
96
97         cl->closure_get_happened = true;
98         closure_set_waiting(cl, _RET_IP_);
99         atomic_add(CLOSURE_WAITING + 1, &cl->remaining);
100         llist_add(&cl->list, &waitlist->list);
101
102         return true;
103 }
104 EXPORT_SYMBOL(closure_wait);
105
106 struct closure_syncer {
107         struct task_struct      *task;
108         int                     done;
109 };
110
111 static void closure_sync_fn(struct closure *cl)
112 {
113         struct closure_syncer *s = cl->s;
114         struct task_struct *p;
115
116         rcu_read_lock();
117         p = READ_ONCE(s->task);
118         s->done = 1;
119         wake_up_process(p);
120         rcu_read_unlock();
121 }
122
123 void __sched __closure_sync(struct closure *cl)
124 {
125         struct closure_syncer s = { .task = current };
126
127         cl->s = &s;
128         continue_at(cl, closure_sync_fn, NULL);
129
130         while (1) {
131                 set_current_state(TASK_UNINTERRUPTIBLE);
132                 if (s.done)
133                         break;
134                 schedule();
135         }
136
137         __set_current_state(TASK_RUNNING);
138 }
139 EXPORT_SYMBOL(__closure_sync);
140
141 #ifdef CONFIG_DEBUG_CLOSURES
142
143 static LIST_HEAD(closure_list);
144 static DEFINE_SPINLOCK(closure_list_lock);
145
146 void closure_debug_create(struct closure *cl)
147 {
148         unsigned long flags;
149
150         BUG_ON(cl->magic == CLOSURE_MAGIC_ALIVE);
151         cl->magic = CLOSURE_MAGIC_ALIVE;
152
153         spin_lock_irqsave(&closure_list_lock, flags);
154         list_add(&cl->all, &closure_list);
155         spin_unlock_irqrestore(&closure_list_lock, flags);
156 }
157 EXPORT_SYMBOL(closure_debug_create);
158
159 void closure_debug_destroy(struct closure *cl)
160 {
161         unsigned long flags;
162
163         BUG_ON(cl->magic != CLOSURE_MAGIC_ALIVE);
164         cl->magic = CLOSURE_MAGIC_DEAD;
165
166         spin_lock_irqsave(&closure_list_lock, flags);
167         list_del(&cl->all);
168         spin_unlock_irqrestore(&closure_list_lock, flags);
169 }
170 EXPORT_SYMBOL(closure_debug_destroy);
171
172 static int debug_show(struct seq_file *f, void *data)
173 {
174         struct closure *cl;
175
176         spin_lock_irq(&closure_list_lock);
177
178         list_for_each_entry(cl, &closure_list, all) {
179                 int r = atomic_read(&cl->remaining);
180
181                 seq_printf(f, "%p: %pS -> %pS p %p r %i ",
182                            cl, (void *) cl->ip, cl->fn, cl->parent,
183                            r & CLOSURE_REMAINING_MASK);
184
185                 seq_printf(f, "%s%s\n",
186                            test_bit(WORK_STRUCT_PENDING_BIT,
187                                     work_data_bits(&cl->work)) ? "Q" : "",
188                            r & CLOSURE_RUNNING  ? "R" : "");
189
190                 if (r & CLOSURE_WAITING)
191                         seq_printf(f, " W %pS\n",
192                                    (void *) cl->waiting_on);
193
194                 seq_puts(f, "\n");
195         }
196
197         spin_unlock_irq(&closure_list_lock);
198         return 0;
199 }
200
201 DEFINE_SHOW_ATTRIBUTE(debug);
202
203 static int __init closure_debug_init(void)
204 {
205         debugfs_create_file("closures", 0400, NULL, NULL, &debug_fops);
206         return 0;
207 }
208 late_initcall(closure_debug_init)
209
210 #endif