x86/ioapic: Cleanup the timer_works() irqflags mess
[linux-2.6-microblaze.git] / kernel / futex.c
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *  Fast Userspace Mutexes (which I call "Futexes!").
4  *  (C) Rusty Russell, IBM 2002
5  *
6  *  Generalized futexes, futex requeueing, misc fixes by Ingo Molnar
7  *  (C) Copyright 2003 Red Hat Inc, All Rights Reserved
8  *
9  *  Removed page pinning, fix privately mapped COW pages and other cleanups
10  *  (C) Copyright 2003, 2004 Jamie Lokier
11  *
12  *  Robust futex support started by Ingo Molnar
13  *  (C) Copyright 2006 Red Hat Inc, All Rights Reserved
14  *  Thanks to Thomas Gleixner for suggestions, analysis and fixes.
15  *
16  *  PI-futex support started by Ingo Molnar and Thomas Gleixner
17  *  Copyright (C) 2006 Red Hat, Inc., Ingo Molnar <mingo@redhat.com>
18  *  Copyright (C) 2006 Timesys Corp., Thomas Gleixner <tglx@timesys.com>
19  *
20  *  PRIVATE futexes by Eric Dumazet
21  *  Copyright (C) 2007 Eric Dumazet <dada1@cosmosbay.com>
22  *
23  *  Requeue-PI support by Darren Hart <dvhltc@us.ibm.com>
24  *  Copyright (C) IBM Corporation, 2009
25  *  Thanks to Thomas Gleixner for conceptual design and careful reviews.
26  *
27  *  Thanks to Ben LaHaise for yelling "hashed waitqueues" loudly
28  *  enough at me, Linus for the original (flawed) idea, Matthew
29  *  Kirkwood for proof-of-concept implementation.
30  *
31  *  "The futexes are also cursed."
32  *  "But they come in a choice of three flavours!"
33  */
34 #include <linux/compat.h>
35 #include <linux/jhash.h>
36 #include <linux/pagemap.h>
37 #include <linux/syscalls.h>
38 #include <linux/hugetlb.h>
39 #include <linux/freezer.h>
40 #include <linux/memblock.h>
41 #include <linux/fault-inject.h>
42 #include <linux/time_namespace.h>
43
44 #include <asm/futex.h>
45
46 #include "locking/rtmutex_common.h"
47
48 /*
49  * READ this before attempting to hack on futexes!
50  *
51  * Basic futex operation and ordering guarantees
52  * =============================================
53  *
54  * The waiter reads the futex value in user space and calls
55  * futex_wait(). This function computes the hash bucket and acquires
56  * the hash bucket lock. After that it reads the futex user space value
57  * again and verifies that the data has not changed. If it has not changed
58  * it enqueues itself into the hash bucket, releases the hash bucket lock
59  * and schedules.
60  *
61  * The waker side modifies the user space value of the futex and calls
62  * futex_wake(). This function computes the hash bucket and acquires the
63  * hash bucket lock. Then it looks for waiters on that futex in the hash
64  * bucket and wakes them.
65  *
66  * In futex wake up scenarios where no tasks are blocked on a futex, taking
67  * the hb spinlock can be avoided and simply return. In order for this
68  * optimization to work, ordering guarantees must exist so that the waiter
69  * being added to the list is acknowledged when the list is concurrently being
70  * checked by the waker, avoiding scenarios like the following:
71  *
72  * CPU 0                               CPU 1
73  * val = *futex;
74  * sys_futex(WAIT, futex, val);
75  *   futex_wait(futex, val);
76  *   uval = *futex;
77  *                                     *futex = newval;
78  *                                     sys_futex(WAKE, futex);
79  *                                       futex_wake(futex);
80  *                                       if (queue_empty())
81  *                                         return;
82  *   if (uval == val)
83  *      lock(hash_bucket(futex));
84  *      queue();
85  *     unlock(hash_bucket(futex));
86  *     schedule();
87  *
88  * This would cause the waiter on CPU 0 to wait forever because it
89  * missed the transition of the user space value from val to newval
90  * and the waker did not find the waiter in the hash bucket queue.
91  *
92  * The correct serialization ensures that a waiter either observes
93  * the changed user space value before blocking or is woken by a
94  * concurrent waker:
95  *
96  * CPU 0                                 CPU 1
97  * val = *futex;
98  * sys_futex(WAIT, futex, val);
99  *   futex_wait(futex, val);
100  *
101  *   waiters++; (a)
102  *   smp_mb(); (A) <-- paired with -.
103  *                                  |
104  *   lock(hash_bucket(futex));      |
105  *                                  |
106  *   uval = *futex;                 |
107  *                                  |        *futex = newval;
108  *                                  |        sys_futex(WAKE, futex);
109  *                                  |          futex_wake(futex);
110  *                                  |
111  *                                  `--------> smp_mb(); (B)
112  *   if (uval == val)
113  *     queue();
114  *     unlock(hash_bucket(futex));
115  *     schedule();                         if (waiters)
116  *                                           lock(hash_bucket(futex));
117  *   else                                    wake_waiters(futex);
118  *     waiters--; (b)                        unlock(hash_bucket(futex));
119  *
120  * Where (A) orders the waiters increment and the futex value read through
121  * atomic operations (see hb_waiters_inc) and where (B) orders the write
122  * to futex and the waiters read (see hb_waiters_pending()).
123  *
124  * This yields the following case (where X:=waiters, Y:=futex):
125  *
126  *      X = Y = 0
127  *
128  *      w[X]=1          w[Y]=1
129  *      MB              MB
130  *      r[Y]=y          r[X]=x
131  *
132  * Which guarantees that x==0 && y==0 is impossible; which translates back into
133  * the guarantee that we cannot both miss the futex variable change and the
134  * enqueue.
135  *
136  * Note that a new waiter is accounted for in (a) even when it is possible that
137  * the wait call can return error, in which case we backtrack from it in (b).
138  * Refer to the comment in queue_lock().
139  *
140  * Similarly, in order to account for waiters being requeued on another
141  * address we always increment the waiters for the destination bucket before
142  * acquiring the lock. It then decrements them again  after releasing it -
143  * the code that actually moves the futex(es) between hash buckets (requeue_futex)
144  * will do the additional required waiter count housekeeping. This is done for
145  * double_lock_hb() and double_unlock_hb(), respectively.
146  */
147
148 #ifdef CONFIG_HAVE_FUTEX_CMPXCHG
149 #define futex_cmpxchg_enabled 1
150 #else
151 static int  __read_mostly futex_cmpxchg_enabled;
152 #endif
153
154 /*
155  * Futex flags used to encode options to functions and preserve them across
156  * restarts.
157  */
158 #ifdef CONFIG_MMU
159 # define FLAGS_SHARED           0x01
160 #else
161 /*
162  * NOMMU does not have per process address space. Let the compiler optimize
163  * code away.
164  */
165 # define FLAGS_SHARED           0x00
166 #endif
167 #define FLAGS_CLOCKRT           0x02
168 #define FLAGS_HAS_TIMEOUT       0x04
169
170 /*
171  * Priority Inheritance state:
172  */
173 struct futex_pi_state {
174         /*
175          * list of 'owned' pi_state instances - these have to be
176          * cleaned up in do_exit() if the task exits prematurely:
177          */
178         struct list_head list;
179
180         /*
181          * The PI object:
182          */
183         struct rt_mutex pi_mutex;
184
185         struct task_struct *owner;
186         refcount_t refcount;
187
188         union futex_key key;
189 } __randomize_layout;
190
191 /**
192  * struct futex_q - The hashed futex queue entry, one per waiting task
193  * @list:               priority-sorted list of tasks waiting on this futex
194  * @task:               the task waiting on the futex
195  * @lock_ptr:           the hash bucket lock
196  * @key:                the key the futex is hashed on
197  * @pi_state:           optional priority inheritance state
198  * @rt_waiter:          rt_waiter storage for use with requeue_pi
199  * @requeue_pi_key:     the requeue_pi target futex key
200  * @bitset:             bitset for the optional bitmasked wakeup
201  *
202  * We use this hashed waitqueue, instead of a normal wait_queue_entry_t, so
203  * we can wake only the relevant ones (hashed queues may be shared).
204  *
205  * A futex_q has a woken state, just like tasks have TASK_RUNNING.
206  * It is considered woken when plist_node_empty(&q->list) || q->lock_ptr == 0.
207  * The order of wakeup is always to make the first condition true, then
208  * the second.
209  *
210  * PI futexes are typically woken before they are removed from the hash list via
211  * the rt_mutex code. See unqueue_me_pi().
212  */
213 struct futex_q {
214         struct plist_node list;
215
216         struct task_struct *task;
217         spinlock_t *lock_ptr;
218         union futex_key key;
219         struct futex_pi_state *pi_state;
220         struct rt_mutex_waiter *rt_waiter;
221         union futex_key *requeue_pi_key;
222         u32 bitset;
223 } __randomize_layout;
224
225 static const struct futex_q futex_q_init = {
226         /* list gets initialized in queue_me()*/
227         .key = FUTEX_KEY_INIT,
228         .bitset = FUTEX_BITSET_MATCH_ANY
229 };
230
231 /*
232  * Hash buckets are shared by all the futex_keys that hash to the same
233  * location.  Each key may have multiple futex_q structures, one for each task
234  * waiting on a futex.
235  */
236 struct futex_hash_bucket {
237         atomic_t waiters;
238         spinlock_t lock;
239         struct plist_head chain;
240 } ____cacheline_aligned_in_smp;
241
242 /*
243  * The base of the bucket array and its size are always used together
244  * (after initialization only in hash_futex()), so ensure that they
245  * reside in the same cacheline.
246  */
247 static struct {
248         struct futex_hash_bucket *queues;
249         unsigned long            hashsize;
250 } __futex_data __read_mostly __aligned(2*sizeof(long));
251 #define futex_queues   (__futex_data.queues)
252 #define futex_hashsize (__futex_data.hashsize)
253
254
255 /*
256  * Fault injections for futexes.
257  */
258 #ifdef CONFIG_FAIL_FUTEX
259
260 static struct {
261         struct fault_attr attr;
262
263         bool ignore_private;
264 } fail_futex = {
265         .attr = FAULT_ATTR_INITIALIZER,
266         .ignore_private = false,
267 };
268
269 static int __init setup_fail_futex(char *str)
270 {
271         return setup_fault_attr(&fail_futex.attr, str);
272 }
273 __setup("fail_futex=", setup_fail_futex);
274
275 static bool should_fail_futex(bool fshared)
276 {
277         if (fail_futex.ignore_private && !fshared)
278                 return false;
279
280         return should_fail(&fail_futex.attr, 1);
281 }
282
283 #ifdef CONFIG_FAULT_INJECTION_DEBUG_FS
284
285 static int __init fail_futex_debugfs(void)
286 {
287         umode_t mode = S_IFREG | S_IRUSR | S_IWUSR;
288         struct dentry *dir;
289
290         dir = fault_create_debugfs_attr("fail_futex", NULL,
291                                         &fail_futex.attr);
292         if (IS_ERR(dir))
293                 return PTR_ERR(dir);
294
295         debugfs_create_bool("ignore-private", mode, dir,
296                             &fail_futex.ignore_private);
297         return 0;
298 }
299
300 late_initcall(fail_futex_debugfs);
301
302 #endif /* CONFIG_FAULT_INJECTION_DEBUG_FS */
303
304 #else
305 static inline bool should_fail_futex(bool fshared)
306 {
307         return false;
308 }
309 #endif /* CONFIG_FAIL_FUTEX */
310
311 #ifdef CONFIG_COMPAT
312 static void compat_exit_robust_list(struct task_struct *curr);
313 #else
314 static inline void compat_exit_robust_list(struct task_struct *curr) { }
315 #endif
316
317 /*
318  * Reflects a new waiter being added to the waitqueue.
319  */
320 static inline void hb_waiters_inc(struct futex_hash_bucket *hb)
321 {
322 #ifdef CONFIG_SMP
323         atomic_inc(&hb->waiters);
324         /*
325          * Full barrier (A), see the ordering comment above.
326          */
327         smp_mb__after_atomic();
328 #endif
329 }
330
331 /*
332  * Reflects a waiter being removed from the waitqueue by wakeup
333  * paths.
334  */
335 static inline void hb_waiters_dec(struct futex_hash_bucket *hb)
336 {
337 #ifdef CONFIG_SMP
338         atomic_dec(&hb->waiters);
339 #endif
340 }
341
342 static inline int hb_waiters_pending(struct futex_hash_bucket *hb)
343 {
344 #ifdef CONFIG_SMP
345         /*
346          * Full barrier (B), see the ordering comment above.
347          */
348         smp_mb();
349         return atomic_read(&hb->waiters);
350 #else
351         return 1;
352 #endif
353 }
354
355 /**
356  * hash_futex - Return the hash bucket in the global hash
357  * @key:        Pointer to the futex key for which the hash is calculated
358  *
359  * We hash on the keys returned from get_futex_key (see below) and return the
360  * corresponding hash bucket in the global hash.
361  */
362 static struct futex_hash_bucket *hash_futex(union futex_key *key)
363 {
364         u32 hash = jhash2((u32 *)key, offsetof(typeof(*key), both.offset) / 4,
365                           key->both.offset);
366
367         return &futex_queues[hash & (futex_hashsize - 1)];
368 }
369
370
371 /**
372  * match_futex - Check whether two futex keys are equal
373  * @key1:       Pointer to key1
374  * @key2:       Pointer to key2
375  *
376  * Return 1 if two futex_keys are equal, 0 otherwise.
377  */
378 static inline int match_futex(union futex_key *key1, union futex_key *key2)
379 {
380         return (key1 && key2
381                 && key1->both.word == key2->both.word
382                 && key1->both.ptr == key2->both.ptr
383                 && key1->both.offset == key2->both.offset);
384 }
385
386 enum futex_access {
387         FUTEX_READ,
388         FUTEX_WRITE
389 };
390
391 /**
392  * futex_setup_timer - set up the sleeping hrtimer.
393  * @time:       ptr to the given timeout value
394  * @timeout:    the hrtimer_sleeper structure to be set up
395  * @flags:      futex flags
396  * @range_ns:   optional range in ns
397  *
398  * Return: Initialized hrtimer_sleeper structure or NULL if no timeout
399  *         value given
400  */
401 static inline struct hrtimer_sleeper *
402 futex_setup_timer(ktime_t *time, struct hrtimer_sleeper *timeout,
403                   int flags, u64 range_ns)
404 {
405         if (!time)
406                 return NULL;
407
408         hrtimer_init_sleeper_on_stack(timeout, (flags & FLAGS_CLOCKRT) ?
409                                       CLOCK_REALTIME : CLOCK_MONOTONIC,
410                                       HRTIMER_MODE_ABS);
411         /*
412          * If range_ns is 0, calling hrtimer_set_expires_range_ns() is
413          * effectively the same as calling hrtimer_set_expires().
414          */
415         hrtimer_set_expires_range_ns(&timeout->timer, *time, range_ns);
416
417         return timeout;
418 }
419
420 /*
421  * Generate a machine wide unique identifier for this inode.
422  *
423  * This relies on u64 not wrapping in the life-time of the machine; which with
424  * 1ns resolution means almost 585 years.
425  *
426  * This further relies on the fact that a well formed program will not unmap
427  * the file while it has a (shared) futex waiting on it. This mapping will have
428  * a file reference which pins the mount and inode.
429  *
430  * If for some reason an inode gets evicted and read back in again, it will get
431  * a new sequence number and will _NOT_ match, even though it is the exact same
432  * file.
433  *
434  * It is important that match_futex() will never have a false-positive, esp.
435  * for PI futexes that can mess up the state. The above argues that false-negatives
436  * are only possible for malformed programs.
437  */
438 static u64 get_inode_sequence_number(struct inode *inode)
439 {
440         static atomic64_t i_seq;
441         u64 old;
442
443         /* Does the inode already have a sequence number? */
444         old = atomic64_read(&inode->i_sequence);
445         if (likely(old))
446                 return old;
447
448         for (;;) {
449                 u64 new = atomic64_add_return(1, &i_seq);
450                 if (WARN_ON_ONCE(!new))
451                         continue;
452
453                 old = atomic64_cmpxchg_relaxed(&inode->i_sequence, 0, new);
454                 if (old)
455                         return old;
456                 return new;
457         }
458 }
459
460 /**
461  * get_futex_key() - Get parameters which are the keys for a futex
462  * @uaddr:      virtual address of the futex
463  * @fshared:    false for a PROCESS_PRIVATE futex, true for PROCESS_SHARED
464  * @key:        address where result is stored.
465  * @rw:         mapping needs to be read/write (values: FUTEX_READ,
466  *              FUTEX_WRITE)
467  *
468  * Return: a negative error code or 0
469  *
470  * The key words are stored in @key on success.
471  *
472  * For shared mappings (when @fshared), the key is:
473  *
474  *   ( inode->i_sequence, page->index, offset_within_page )
475  *
476  * [ also see get_inode_sequence_number() ]
477  *
478  * For private mappings (or when !@fshared), the key is:
479  *
480  *   ( current->mm, address, 0 )
481  *
482  * This allows (cross process, where applicable) identification of the futex
483  * without keeping the page pinned for the duration of the FUTEX_WAIT.
484  *
485  * lock_page() might sleep, the caller should not hold a spinlock.
486  */
487 static int get_futex_key(u32 __user *uaddr, bool fshared, union futex_key *key,
488                          enum futex_access rw)
489 {
490         unsigned long address = (unsigned long)uaddr;
491         struct mm_struct *mm = current->mm;
492         struct page *page, *tail;
493         struct address_space *mapping;
494         int err, ro = 0;
495
496         /*
497          * The futex address must be "naturally" aligned.
498          */
499         key->both.offset = address % PAGE_SIZE;
500         if (unlikely((address % sizeof(u32)) != 0))
501                 return -EINVAL;
502         address -= key->both.offset;
503
504         if (unlikely(!access_ok(uaddr, sizeof(u32))))
505                 return -EFAULT;
506
507         if (unlikely(should_fail_futex(fshared)))
508                 return -EFAULT;
509
510         /*
511          * PROCESS_PRIVATE futexes are fast.
512          * As the mm cannot disappear under us and the 'key' only needs
513          * virtual address, we dont even have to find the underlying vma.
514          * Note : We do have to check 'uaddr' is a valid user address,
515          *        but access_ok() should be faster than find_vma()
516          */
517         if (!fshared) {
518                 key->private.mm = mm;
519                 key->private.address = address;
520                 return 0;
521         }
522
523 again:
524         /* Ignore any VERIFY_READ mapping (futex common case) */
525         if (unlikely(should_fail_futex(true)))
526                 return -EFAULT;
527
528         err = get_user_pages_fast(address, 1, FOLL_WRITE, &page);
529         /*
530          * If write access is not required (eg. FUTEX_WAIT), try
531          * and get read-only access.
532          */
533         if (err == -EFAULT && rw == FUTEX_READ) {
534                 err = get_user_pages_fast(address, 1, 0, &page);
535                 ro = 1;
536         }
537         if (err < 0)
538                 return err;
539         else
540                 err = 0;
541
542         /*
543          * The treatment of mapping from this point on is critical. The page
544          * lock protects many things but in this context the page lock
545          * stabilizes mapping, prevents inode freeing in the shared
546          * file-backed region case and guards against movement to swap cache.
547          *
548          * Strictly speaking the page lock is not needed in all cases being
549          * considered here and page lock forces unnecessarily serialization
550          * From this point on, mapping will be re-verified if necessary and
551          * page lock will be acquired only if it is unavoidable
552          *
553          * Mapping checks require the head page for any compound page so the
554          * head page and mapping is looked up now. For anonymous pages, it
555          * does not matter if the page splits in the future as the key is
556          * based on the address. For filesystem-backed pages, the tail is
557          * required as the index of the page determines the key. For
558          * base pages, there is no tail page and tail == page.
559          */
560         tail = page;
561         page = compound_head(page);
562         mapping = READ_ONCE(page->mapping);
563
564         /*
565          * If page->mapping is NULL, then it cannot be a PageAnon
566          * page; but it might be the ZERO_PAGE or in the gate area or
567          * in a special mapping (all cases which we are happy to fail);
568          * or it may have been a good file page when get_user_pages_fast
569          * found it, but truncated or holepunched or subjected to
570          * invalidate_complete_page2 before we got the page lock (also
571          * cases which we are happy to fail).  And we hold a reference,
572          * so refcount care in invalidate_complete_page's remove_mapping
573          * prevents drop_caches from setting mapping to NULL beneath us.
574          *
575          * The case we do have to guard against is when memory pressure made
576          * shmem_writepage move it from filecache to swapcache beneath us:
577          * an unlikely race, but we do need to retry for page->mapping.
578          */
579         if (unlikely(!mapping)) {
580                 int shmem_swizzled;
581
582                 /*
583                  * Page lock is required to identify which special case above
584                  * applies. If this is really a shmem page then the page lock
585                  * will prevent unexpected transitions.
586                  */
587                 lock_page(page);
588                 shmem_swizzled = PageSwapCache(page) || page->mapping;
589                 unlock_page(page);
590                 put_page(page);
591
592                 if (shmem_swizzled)
593                         goto again;
594
595                 return -EFAULT;
596         }
597
598         /*
599          * Private mappings are handled in a simple way.
600          *
601          * If the futex key is stored on an anonymous page, then the associated
602          * object is the mm which is implicitly pinned by the calling process.
603          *
604          * NOTE: When userspace waits on a MAP_SHARED mapping, even if
605          * it's a read-only handle, it's expected that futexes attach to
606          * the object not the particular process.
607          */
608         if (PageAnon(page)) {
609                 /*
610                  * A RO anonymous page will never change and thus doesn't make
611                  * sense for futex operations.
612                  */
613                 if (unlikely(should_fail_futex(true)) || ro) {
614                         err = -EFAULT;
615                         goto out;
616                 }
617
618                 key->both.offset |= FUT_OFF_MMSHARED; /* ref taken on mm */
619                 key->private.mm = mm;
620                 key->private.address = address;
621
622         } else {
623                 struct inode *inode;
624
625                 /*
626                  * The associated futex object in this case is the inode and
627                  * the page->mapping must be traversed. Ordinarily this should
628                  * be stabilised under page lock but it's not strictly
629                  * necessary in this case as we just want to pin the inode, not
630                  * update the radix tree or anything like that.
631                  *
632                  * The RCU read lock is taken as the inode is finally freed
633                  * under RCU. If the mapping still matches expectations then the
634                  * mapping->host can be safely accessed as being a valid inode.
635                  */
636                 rcu_read_lock();
637
638                 if (READ_ONCE(page->mapping) != mapping) {
639                         rcu_read_unlock();
640                         put_page(page);
641
642                         goto again;
643                 }
644
645                 inode = READ_ONCE(mapping->host);
646                 if (!inode) {
647                         rcu_read_unlock();
648                         put_page(page);
649
650                         goto again;
651                 }
652
653                 key->both.offset |= FUT_OFF_INODE; /* inode-based key */
654                 key->shared.i_seq = get_inode_sequence_number(inode);
655                 key->shared.pgoff = basepage_index(tail);
656                 rcu_read_unlock();
657         }
658
659 out:
660         put_page(page);
661         return err;
662 }
663
664 /**
665  * fault_in_user_writeable() - Fault in user address and verify RW access
666  * @uaddr:      pointer to faulting user space address
667  *
668  * Slow path to fixup the fault we just took in the atomic write
669  * access to @uaddr.
670  *
671  * We have no generic implementation of a non-destructive write to the
672  * user address. We know that we faulted in the atomic pagefault
673  * disabled section so we can as well avoid the #PF overhead by
674  * calling get_user_pages() right away.
675  */
676 static int fault_in_user_writeable(u32 __user *uaddr)
677 {
678         struct mm_struct *mm = current->mm;
679         int ret;
680
681         mmap_read_lock(mm);
682         ret = fixup_user_fault(mm, (unsigned long)uaddr,
683                                FAULT_FLAG_WRITE, NULL);
684         mmap_read_unlock(mm);
685
686         return ret < 0 ? ret : 0;
687 }
688
689 /**
690  * futex_top_waiter() - Return the highest priority waiter on a futex
691  * @hb:         the hash bucket the futex_q's reside in
692  * @key:        the futex key (to distinguish it from other futex futex_q's)
693  *
694  * Must be called with the hb lock held.
695  */
696 static struct futex_q *futex_top_waiter(struct futex_hash_bucket *hb,
697                                         union futex_key *key)
698 {
699         struct futex_q *this;
700
701         plist_for_each_entry(this, &hb->chain, list) {
702                 if (match_futex(&this->key, key))
703                         return this;
704         }
705         return NULL;
706 }
707
708 static int cmpxchg_futex_value_locked(u32 *curval, u32 __user *uaddr,
709                                       u32 uval, u32 newval)
710 {
711         int ret;
712
713         pagefault_disable();
714         ret = futex_atomic_cmpxchg_inatomic(curval, uaddr, uval, newval);
715         pagefault_enable();
716
717         return ret;
718 }
719
720 static int get_futex_value_locked(u32 *dest, u32 __user *from)
721 {
722         int ret;
723
724         pagefault_disable();
725         ret = __get_user(*dest, from);
726         pagefault_enable();
727
728         return ret ? -EFAULT : 0;
729 }
730
731
732 /*
733  * PI code:
734  */
735 static int refill_pi_state_cache(void)
736 {
737         struct futex_pi_state *pi_state;
738
739         if (likely(current->pi_state_cache))
740                 return 0;
741
742         pi_state = kzalloc(sizeof(*pi_state), GFP_KERNEL);
743
744         if (!pi_state)
745                 return -ENOMEM;
746
747         INIT_LIST_HEAD(&pi_state->list);
748         /* pi_mutex gets initialized later */
749         pi_state->owner = NULL;
750         refcount_set(&pi_state->refcount, 1);
751         pi_state->key = FUTEX_KEY_INIT;
752
753         current->pi_state_cache = pi_state;
754
755         return 0;
756 }
757
758 static struct futex_pi_state *alloc_pi_state(void)
759 {
760         struct futex_pi_state *pi_state = current->pi_state_cache;
761
762         WARN_ON(!pi_state);
763         current->pi_state_cache = NULL;
764
765         return pi_state;
766 }
767
768 static void get_pi_state(struct futex_pi_state *pi_state)
769 {
770         WARN_ON_ONCE(!refcount_inc_not_zero(&pi_state->refcount));
771 }
772
773 /*
774  * Drops a reference to the pi_state object and frees or caches it
775  * when the last reference is gone.
776  */
777 static void put_pi_state(struct futex_pi_state *pi_state)
778 {
779         if (!pi_state)
780                 return;
781
782         if (!refcount_dec_and_test(&pi_state->refcount))
783                 return;
784
785         /*
786          * If pi_state->owner is NULL, the owner is most probably dying
787          * and has cleaned up the pi_state already
788          */
789         if (pi_state->owner) {
790                 struct task_struct *owner;
791
792                 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
793                 owner = pi_state->owner;
794                 if (owner) {
795                         raw_spin_lock(&owner->pi_lock);
796                         list_del_init(&pi_state->list);
797                         raw_spin_unlock(&owner->pi_lock);
798                 }
799                 rt_mutex_proxy_unlock(&pi_state->pi_mutex, owner);
800                 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
801         }
802
803         if (current->pi_state_cache) {
804                 kfree(pi_state);
805         } else {
806                 /*
807                  * pi_state->list is already empty.
808                  * clear pi_state->owner.
809                  * refcount is at 0 - put it back to 1.
810                  */
811                 pi_state->owner = NULL;
812                 refcount_set(&pi_state->refcount, 1);
813                 current->pi_state_cache = pi_state;
814         }
815 }
816
817 #ifdef CONFIG_FUTEX_PI
818
819 /*
820  * This task is holding PI mutexes at exit time => bad.
821  * Kernel cleans up PI-state, but userspace is likely hosed.
822  * (Robust-futex cleanup is separate and might save the day for userspace.)
823  */
824 static void exit_pi_state_list(struct task_struct *curr)
825 {
826         struct list_head *next, *head = &curr->pi_state_list;
827         struct futex_pi_state *pi_state;
828         struct futex_hash_bucket *hb;
829         union futex_key key = FUTEX_KEY_INIT;
830
831         if (!futex_cmpxchg_enabled)
832                 return;
833         /*
834          * We are a ZOMBIE and nobody can enqueue itself on
835          * pi_state_list anymore, but we have to be careful
836          * versus waiters unqueueing themselves:
837          */
838         raw_spin_lock_irq(&curr->pi_lock);
839         while (!list_empty(head)) {
840                 next = head->next;
841                 pi_state = list_entry(next, struct futex_pi_state, list);
842                 key = pi_state->key;
843                 hb = hash_futex(&key);
844
845                 /*
846                  * We can race against put_pi_state() removing itself from the
847                  * list (a waiter going away). put_pi_state() will first
848                  * decrement the reference count and then modify the list, so
849                  * its possible to see the list entry but fail this reference
850                  * acquire.
851                  *
852                  * In that case; drop the locks to let put_pi_state() make
853                  * progress and retry the loop.
854                  */
855                 if (!refcount_inc_not_zero(&pi_state->refcount)) {
856                         raw_spin_unlock_irq(&curr->pi_lock);
857                         cpu_relax();
858                         raw_spin_lock_irq(&curr->pi_lock);
859                         continue;
860                 }
861                 raw_spin_unlock_irq(&curr->pi_lock);
862
863                 spin_lock(&hb->lock);
864                 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
865                 raw_spin_lock(&curr->pi_lock);
866                 /*
867                  * We dropped the pi-lock, so re-check whether this
868                  * task still owns the PI-state:
869                  */
870                 if (head->next != next) {
871                         /* retain curr->pi_lock for the loop invariant */
872                         raw_spin_unlock(&pi_state->pi_mutex.wait_lock);
873                         spin_unlock(&hb->lock);
874                         put_pi_state(pi_state);
875                         continue;
876                 }
877
878                 WARN_ON(pi_state->owner != curr);
879                 WARN_ON(list_empty(&pi_state->list));
880                 list_del_init(&pi_state->list);
881                 pi_state->owner = NULL;
882
883                 raw_spin_unlock(&curr->pi_lock);
884                 raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
885                 spin_unlock(&hb->lock);
886
887                 rt_mutex_futex_unlock(&pi_state->pi_mutex);
888                 put_pi_state(pi_state);
889
890                 raw_spin_lock_irq(&curr->pi_lock);
891         }
892         raw_spin_unlock_irq(&curr->pi_lock);
893 }
894 #else
895 static inline void exit_pi_state_list(struct task_struct *curr) { }
896 #endif
897
898 /*
899  * We need to check the following states:
900  *
901  *      Waiter | pi_state | pi->owner | uTID      | uODIED | ?
902  *
903  * [1]  NULL   | ---      | ---       | 0         | 0/1    | Valid
904  * [2]  NULL   | ---      | ---       | >0        | 0/1    | Valid
905  *
906  * [3]  Found  | NULL     | --        | Any       | 0/1    | Invalid
907  *
908  * [4]  Found  | Found    | NULL      | 0         | 1      | Valid
909  * [5]  Found  | Found    | NULL      | >0        | 1      | Invalid
910  *
911  * [6]  Found  | Found    | task      | 0         | 1      | Valid
912  *
913  * [7]  Found  | Found    | NULL      | Any       | 0      | Invalid
914  *
915  * [8]  Found  | Found    | task      | ==taskTID | 0/1    | Valid
916  * [9]  Found  | Found    | task      | 0         | 0      | Invalid
917  * [10] Found  | Found    | task      | !=taskTID | 0/1    | Invalid
918  *
919  * [1]  Indicates that the kernel can acquire the futex atomically. We
920  *      came here due to a stale FUTEX_WAITERS/FUTEX_OWNER_DIED bit.
921  *
922  * [2]  Valid, if TID does not belong to a kernel thread. If no matching
923  *      thread is found then it indicates that the owner TID has died.
924  *
925  * [3]  Invalid. The waiter is queued on a non PI futex
926  *
927  * [4]  Valid state after exit_robust_list(), which sets the user space
928  *      value to FUTEX_WAITERS | FUTEX_OWNER_DIED.
929  *
930  * [5]  The user space value got manipulated between exit_robust_list()
931  *      and exit_pi_state_list()
932  *
933  * [6]  Valid state after exit_pi_state_list() which sets the new owner in
934  *      the pi_state but cannot access the user space value.
935  *
936  * [7]  pi_state->owner can only be NULL when the OWNER_DIED bit is set.
937  *
938  * [8]  Owner and user space value match
939  *
940  * [9]  There is no transient state which sets the user space TID to 0
941  *      except exit_robust_list(), but this is indicated by the
942  *      FUTEX_OWNER_DIED bit. See [4]
943  *
944  * [10] There is no transient state which leaves owner and user space
945  *      TID out of sync.
946  *
947  *
948  * Serialization and lifetime rules:
949  *
950  * hb->lock:
951  *
952  *      hb -> futex_q, relation
953  *      futex_q -> pi_state, relation
954  *
955  *      (cannot be raw because hb can contain arbitrary amount
956  *       of futex_q's)
957  *
958  * pi_mutex->wait_lock:
959  *
960  *      {uval, pi_state}
961  *
962  *      (and pi_mutex 'obviously')
963  *
964  * p->pi_lock:
965  *
966  *      p->pi_state_list -> pi_state->list, relation
967  *
968  * pi_state->refcount:
969  *
970  *      pi_state lifetime
971  *
972  *
973  * Lock order:
974  *
975  *   hb->lock
976  *     pi_mutex->wait_lock
977  *       p->pi_lock
978  *
979  */
980
981 /*
982  * Validate that the existing waiter has a pi_state and sanity check
983  * the pi_state against the user space value. If correct, attach to
984  * it.
985  */
986 static int attach_to_pi_state(u32 __user *uaddr, u32 uval,
987                               struct futex_pi_state *pi_state,
988                               struct futex_pi_state **ps)
989 {
990         pid_t pid = uval & FUTEX_TID_MASK;
991         u32 uval2;
992         int ret;
993
994         /*
995          * Userspace might have messed up non-PI and PI futexes [3]
996          */
997         if (unlikely(!pi_state))
998                 return -EINVAL;
999
1000         /*
1001          * We get here with hb->lock held, and having found a
1002          * futex_top_waiter(). This means that futex_lock_pi() of said futex_q
1003          * has dropped the hb->lock in between queue_me() and unqueue_me_pi(),
1004          * which in turn means that futex_lock_pi() still has a reference on
1005          * our pi_state.
1006          *
1007          * The waiter holding a reference on @pi_state also protects against
1008          * the unlocked put_pi_state() in futex_unlock_pi(), futex_lock_pi()
1009          * and futex_wait_requeue_pi() as it cannot go to 0 and consequently
1010          * free pi_state before we can take a reference ourselves.
1011          */
1012         WARN_ON(!refcount_read(&pi_state->refcount));
1013
1014         /*
1015          * Now that we have a pi_state, we can acquire wait_lock
1016          * and do the state validation.
1017          */
1018         raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
1019
1020         /*
1021          * Since {uval, pi_state} is serialized by wait_lock, and our current
1022          * uval was read without holding it, it can have changed. Verify it
1023          * still is what we expect it to be, otherwise retry the entire
1024          * operation.
1025          */
1026         if (get_futex_value_locked(&uval2, uaddr))
1027                 goto out_efault;
1028
1029         if (uval != uval2)
1030                 goto out_eagain;
1031
1032         /*
1033          * Handle the owner died case:
1034          */
1035         if (uval & FUTEX_OWNER_DIED) {
1036                 /*
1037                  * exit_pi_state_list sets owner to NULL and wakes the
1038                  * topmost waiter. The task which acquires the
1039                  * pi_state->rt_mutex will fixup owner.
1040                  */
1041                 if (!pi_state->owner) {
1042                         /*
1043                          * No pi state owner, but the user space TID
1044                          * is not 0. Inconsistent state. [5]
1045                          */
1046                         if (pid)
1047                                 goto out_einval;
1048                         /*
1049                          * Take a ref on the state and return success. [4]
1050                          */
1051                         goto out_attach;
1052                 }
1053
1054                 /*
1055                  * If TID is 0, then either the dying owner has not
1056                  * yet executed exit_pi_state_list() or some waiter
1057                  * acquired the rtmutex in the pi state, but did not
1058                  * yet fixup the TID in user space.
1059                  *
1060                  * Take a ref on the state and return success. [6]
1061                  */
1062                 if (!pid)
1063                         goto out_attach;
1064         } else {
1065                 /*
1066                  * If the owner died bit is not set, then the pi_state
1067                  * must have an owner. [7]
1068                  */
1069                 if (!pi_state->owner)
1070                         goto out_einval;
1071         }
1072
1073         /*
1074          * Bail out if user space manipulated the futex value. If pi
1075          * state exists then the owner TID must be the same as the
1076          * user space TID. [9/10]
1077          */
1078         if (pid != task_pid_vnr(pi_state->owner))
1079                 goto out_einval;
1080
1081 out_attach:
1082         get_pi_state(pi_state);
1083         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1084         *ps = pi_state;
1085         return 0;
1086
1087 out_einval:
1088         ret = -EINVAL;
1089         goto out_error;
1090
1091 out_eagain:
1092         ret = -EAGAIN;
1093         goto out_error;
1094
1095 out_efault:
1096         ret = -EFAULT;
1097         goto out_error;
1098
1099 out_error:
1100         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1101         return ret;
1102 }
1103
1104 /**
1105  * wait_for_owner_exiting - Block until the owner has exited
1106  * @ret: owner's current futex lock status
1107  * @exiting:    Pointer to the exiting task
1108  *
1109  * Caller must hold a refcount on @exiting.
1110  */
1111 static void wait_for_owner_exiting(int ret, struct task_struct *exiting)
1112 {
1113         if (ret != -EBUSY) {
1114                 WARN_ON_ONCE(exiting);
1115                 return;
1116         }
1117
1118         if (WARN_ON_ONCE(ret == -EBUSY && !exiting))
1119                 return;
1120
1121         mutex_lock(&exiting->futex_exit_mutex);
1122         /*
1123          * No point in doing state checking here. If the waiter got here
1124          * while the task was in exec()->exec_futex_release() then it can
1125          * have any FUTEX_STATE_* value when the waiter has acquired the
1126          * mutex. OK, if running, EXITING or DEAD if it reached exit()
1127          * already. Highly unlikely and not a problem. Just one more round
1128          * through the futex maze.
1129          */
1130         mutex_unlock(&exiting->futex_exit_mutex);
1131
1132         put_task_struct(exiting);
1133 }
1134
1135 static int handle_exit_race(u32 __user *uaddr, u32 uval,
1136                             struct task_struct *tsk)
1137 {
1138         u32 uval2;
1139
1140         /*
1141          * If the futex exit state is not yet FUTEX_STATE_DEAD, tell the
1142          * caller that the alleged owner is busy.
1143          */
1144         if (tsk && tsk->futex_state != FUTEX_STATE_DEAD)
1145                 return -EBUSY;
1146
1147         /*
1148          * Reread the user space value to handle the following situation:
1149          *
1150          * CPU0                         CPU1
1151          *
1152          * sys_exit()                   sys_futex()
1153          *  do_exit()                    futex_lock_pi()
1154          *                                futex_lock_pi_atomic()
1155          *   exit_signals(tsk)              No waiters:
1156          *    tsk->flags |= PF_EXITING;     *uaddr == 0x00000PID
1157          *  mm_release(tsk)                 Set waiter bit
1158          *   exit_robust_list(tsk) {        *uaddr = 0x80000PID;
1159          *      Set owner died              attach_to_pi_owner() {
1160          *    *uaddr = 0xC0000000;           tsk = get_task(PID);
1161          *   }                               if (!tsk->flags & PF_EXITING) {
1162          *  ...                                attach();
1163          *  tsk->futex_state =               } else {
1164          *      FUTEX_STATE_DEAD;              if (tsk->futex_state !=
1165          *                                        FUTEX_STATE_DEAD)
1166          *                                       return -EAGAIN;
1167          *                                     return -ESRCH; <--- FAIL
1168          *                                   }
1169          *
1170          * Returning ESRCH unconditionally is wrong here because the
1171          * user space value has been changed by the exiting task.
1172          *
1173          * The same logic applies to the case where the exiting task is
1174          * already gone.
1175          */
1176         if (get_futex_value_locked(&uval2, uaddr))
1177                 return -EFAULT;
1178
1179         /* If the user space value has changed, try again. */
1180         if (uval2 != uval)
1181                 return -EAGAIN;
1182
1183         /*
1184          * The exiting task did not have a robust list, the robust list was
1185          * corrupted or the user space value in *uaddr is simply bogus.
1186          * Give up and tell user space.
1187          */
1188         return -ESRCH;
1189 }
1190
1191 /*
1192  * Lookup the task for the TID provided from user space and attach to
1193  * it after doing proper sanity checks.
1194  */
1195 static int attach_to_pi_owner(u32 __user *uaddr, u32 uval, union futex_key *key,
1196                               struct futex_pi_state **ps,
1197                               struct task_struct **exiting)
1198 {
1199         pid_t pid = uval & FUTEX_TID_MASK;
1200         struct futex_pi_state *pi_state;
1201         struct task_struct *p;
1202
1203         /*
1204          * We are the first waiter - try to look up the real owner and attach
1205          * the new pi_state to it, but bail out when TID = 0 [1]
1206          *
1207          * The !pid check is paranoid. None of the call sites should end up
1208          * with pid == 0, but better safe than sorry. Let the caller retry
1209          */
1210         if (!pid)
1211                 return -EAGAIN;
1212         p = find_get_task_by_vpid(pid);
1213         if (!p)
1214                 return handle_exit_race(uaddr, uval, NULL);
1215
1216         if (unlikely(p->flags & PF_KTHREAD)) {
1217                 put_task_struct(p);
1218                 return -EPERM;
1219         }
1220
1221         /*
1222          * We need to look at the task state to figure out, whether the
1223          * task is exiting. To protect against the change of the task state
1224          * in futex_exit_release(), we do this protected by p->pi_lock:
1225          */
1226         raw_spin_lock_irq(&p->pi_lock);
1227         if (unlikely(p->futex_state != FUTEX_STATE_OK)) {
1228                 /*
1229                  * The task is on the way out. When the futex state is
1230                  * FUTEX_STATE_DEAD, we know that the task has finished
1231                  * the cleanup:
1232                  */
1233                 int ret = handle_exit_race(uaddr, uval, p);
1234
1235                 raw_spin_unlock_irq(&p->pi_lock);
1236                 /*
1237                  * If the owner task is between FUTEX_STATE_EXITING and
1238                  * FUTEX_STATE_DEAD then store the task pointer and keep
1239                  * the reference on the task struct. The calling code will
1240                  * drop all locks, wait for the task to reach
1241                  * FUTEX_STATE_DEAD and then drop the refcount. This is
1242                  * required to prevent a live lock when the current task
1243                  * preempted the exiting task between the two states.
1244                  */
1245                 if (ret == -EBUSY)
1246                         *exiting = p;
1247                 else
1248                         put_task_struct(p);
1249                 return ret;
1250         }
1251
1252         /*
1253          * No existing pi state. First waiter. [2]
1254          *
1255          * This creates pi_state, we have hb->lock held, this means nothing can
1256          * observe this state, wait_lock is irrelevant.
1257          */
1258         pi_state = alloc_pi_state();
1259
1260         /*
1261          * Initialize the pi_mutex in locked state and make @p
1262          * the owner of it:
1263          */
1264         rt_mutex_init_proxy_locked(&pi_state->pi_mutex, p);
1265
1266         /* Store the key for possible exit cleanups: */
1267         pi_state->key = *key;
1268
1269         WARN_ON(!list_empty(&pi_state->list));
1270         list_add(&pi_state->list, &p->pi_state_list);
1271         /*
1272          * Assignment without holding pi_state->pi_mutex.wait_lock is safe
1273          * because there is no concurrency as the object is not published yet.
1274          */
1275         pi_state->owner = p;
1276         raw_spin_unlock_irq(&p->pi_lock);
1277
1278         put_task_struct(p);
1279
1280         *ps = pi_state;
1281
1282         return 0;
1283 }
1284
1285 static int lookup_pi_state(u32 __user *uaddr, u32 uval,
1286                            struct futex_hash_bucket *hb,
1287                            union futex_key *key, struct futex_pi_state **ps,
1288                            struct task_struct **exiting)
1289 {
1290         struct futex_q *top_waiter = futex_top_waiter(hb, key);
1291
1292         /*
1293          * If there is a waiter on that futex, validate it and
1294          * attach to the pi_state when the validation succeeds.
1295          */
1296         if (top_waiter)
1297                 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
1298
1299         /*
1300          * We are the first waiter - try to look up the owner based on
1301          * @uval and attach to it.
1302          */
1303         return attach_to_pi_owner(uaddr, uval, key, ps, exiting);
1304 }
1305
1306 static int lock_pi_update_atomic(u32 __user *uaddr, u32 uval, u32 newval)
1307 {
1308         int err;
1309         u32 curval;
1310
1311         if (unlikely(should_fail_futex(true)))
1312                 return -EFAULT;
1313
1314         err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
1315         if (unlikely(err))
1316                 return err;
1317
1318         /* If user space value changed, let the caller retry */
1319         return curval != uval ? -EAGAIN : 0;
1320 }
1321
1322 /**
1323  * futex_lock_pi_atomic() - Atomic work required to acquire a pi aware futex
1324  * @uaddr:              the pi futex user address
1325  * @hb:                 the pi futex hash bucket
1326  * @key:                the futex key associated with uaddr and hb
1327  * @ps:                 the pi_state pointer where we store the result of the
1328  *                      lookup
1329  * @task:               the task to perform the atomic lock work for.  This will
1330  *                      be "current" except in the case of requeue pi.
1331  * @exiting:            Pointer to store the task pointer of the owner task
1332  *                      which is in the middle of exiting
1333  * @set_waiters:        force setting the FUTEX_WAITERS bit (1) or not (0)
1334  *
1335  * Return:
1336  *  -  0 - ready to wait;
1337  *  -  1 - acquired the lock;
1338  *  - <0 - error
1339  *
1340  * The hb->lock and futex_key refs shall be held by the caller.
1341  *
1342  * @exiting is only set when the return value is -EBUSY. If so, this holds
1343  * a refcount on the exiting task on return and the caller needs to drop it
1344  * after waiting for the exit to complete.
1345  */
1346 static int futex_lock_pi_atomic(u32 __user *uaddr, struct futex_hash_bucket *hb,
1347                                 union futex_key *key,
1348                                 struct futex_pi_state **ps,
1349                                 struct task_struct *task,
1350                                 struct task_struct **exiting,
1351                                 int set_waiters)
1352 {
1353         u32 uval, newval, vpid = task_pid_vnr(task);
1354         struct futex_q *top_waiter;
1355         int ret;
1356
1357         /*
1358          * Read the user space value first so we can validate a few
1359          * things before proceeding further.
1360          */
1361         if (get_futex_value_locked(&uval, uaddr))
1362                 return -EFAULT;
1363
1364         if (unlikely(should_fail_futex(true)))
1365                 return -EFAULT;
1366
1367         /*
1368          * Detect deadlocks.
1369          */
1370         if ((unlikely((uval & FUTEX_TID_MASK) == vpid)))
1371                 return -EDEADLK;
1372
1373         if ((unlikely(should_fail_futex(true))))
1374                 return -EDEADLK;
1375
1376         /*
1377          * Lookup existing state first. If it exists, try to attach to
1378          * its pi_state.
1379          */
1380         top_waiter = futex_top_waiter(hb, key);
1381         if (top_waiter)
1382                 return attach_to_pi_state(uaddr, uval, top_waiter->pi_state, ps);
1383
1384         /*
1385          * No waiter and user TID is 0. We are here because the
1386          * waiters or the owner died bit is set or called from
1387          * requeue_cmp_pi or for whatever reason something took the
1388          * syscall.
1389          */
1390         if (!(uval & FUTEX_TID_MASK)) {
1391                 /*
1392                  * We take over the futex. No other waiters and the user space
1393                  * TID is 0. We preserve the owner died bit.
1394                  */
1395                 newval = uval & FUTEX_OWNER_DIED;
1396                 newval |= vpid;
1397
1398                 /* The futex requeue_pi code can enforce the waiters bit */
1399                 if (set_waiters)
1400                         newval |= FUTEX_WAITERS;
1401
1402                 ret = lock_pi_update_atomic(uaddr, uval, newval);
1403                 /* If the take over worked, return 1 */
1404                 return ret < 0 ? ret : 1;
1405         }
1406
1407         /*
1408          * First waiter. Set the waiters bit before attaching ourself to
1409          * the owner. If owner tries to unlock, it will be forced into
1410          * the kernel and blocked on hb->lock.
1411          */
1412         newval = uval | FUTEX_WAITERS;
1413         ret = lock_pi_update_atomic(uaddr, uval, newval);
1414         if (ret)
1415                 return ret;
1416         /*
1417          * If the update of the user space value succeeded, we try to
1418          * attach to the owner. If that fails, no harm done, we only
1419          * set the FUTEX_WAITERS bit in the user space variable.
1420          */
1421         return attach_to_pi_owner(uaddr, newval, key, ps, exiting);
1422 }
1423
1424 /**
1425  * __unqueue_futex() - Remove the futex_q from its futex_hash_bucket
1426  * @q:  The futex_q to unqueue
1427  *
1428  * The q->lock_ptr must not be NULL and must be held by the caller.
1429  */
1430 static void __unqueue_futex(struct futex_q *q)
1431 {
1432         struct futex_hash_bucket *hb;
1433
1434         if (WARN_ON_SMP(!q->lock_ptr) || WARN_ON(plist_node_empty(&q->list)))
1435                 return;
1436         lockdep_assert_held(q->lock_ptr);
1437
1438         hb = container_of(q->lock_ptr, struct futex_hash_bucket, lock);
1439         plist_del(&q->list, &hb->chain);
1440         hb_waiters_dec(hb);
1441 }
1442
1443 /*
1444  * The hash bucket lock must be held when this is called.
1445  * Afterwards, the futex_q must not be accessed. Callers
1446  * must ensure to later call wake_up_q() for the actual
1447  * wakeups to occur.
1448  */
1449 static void mark_wake_futex(struct wake_q_head *wake_q, struct futex_q *q)
1450 {
1451         struct task_struct *p = q->task;
1452
1453         if (WARN(q->pi_state || q->rt_waiter, "refusing to wake PI futex\n"))
1454                 return;
1455
1456         get_task_struct(p);
1457         __unqueue_futex(q);
1458         /*
1459          * The waiting task can free the futex_q as soon as q->lock_ptr = NULL
1460          * is written, without taking any locks. This is possible in the event
1461          * of a spurious wakeup, for example. A memory barrier is required here
1462          * to prevent the following store to lock_ptr from getting ahead of the
1463          * plist_del in __unqueue_futex().
1464          */
1465         smp_store_release(&q->lock_ptr, NULL);
1466
1467         /*
1468          * Queue the task for later wakeup for after we've released
1469          * the hb->lock.
1470          */
1471         wake_q_add_safe(wake_q, p);
1472 }
1473
1474 /*
1475  * Caller must hold a reference on @pi_state.
1476  */
1477 static int wake_futex_pi(u32 __user *uaddr, u32 uval, struct futex_pi_state *pi_state)
1478 {
1479         u32 curval, newval;
1480         struct task_struct *new_owner;
1481         bool postunlock = false;
1482         DEFINE_WAKE_Q(wake_q);
1483         int ret = 0;
1484
1485         new_owner = rt_mutex_next_owner(&pi_state->pi_mutex);
1486         if (WARN_ON_ONCE(!new_owner)) {
1487                 /*
1488                  * As per the comment in futex_unlock_pi() this should not happen.
1489                  *
1490                  * When this happens, give up our locks and try again, giving
1491                  * the futex_lock_pi() instance time to complete, either by
1492                  * waiting on the rtmutex or removing itself from the futex
1493                  * queue.
1494                  */
1495                 ret = -EAGAIN;
1496                 goto out_unlock;
1497         }
1498
1499         /*
1500          * We pass it to the next owner. The WAITERS bit is always kept
1501          * enabled while there is PI state around. We cleanup the owner
1502          * died bit, because we are the owner.
1503          */
1504         newval = FUTEX_WAITERS | task_pid_vnr(new_owner);
1505
1506         if (unlikely(should_fail_futex(true)))
1507                 ret = -EFAULT;
1508
1509         ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
1510         if (!ret && (curval != uval)) {
1511                 /*
1512                  * If a unconditional UNLOCK_PI operation (user space did not
1513                  * try the TID->0 transition) raced with a waiter setting the
1514                  * FUTEX_WAITERS flag between get_user() and locking the hash
1515                  * bucket lock, retry the operation.
1516                  */
1517                 if ((FUTEX_TID_MASK & curval) == uval)
1518                         ret = -EAGAIN;
1519                 else
1520                         ret = -EINVAL;
1521         }
1522
1523         if (ret)
1524                 goto out_unlock;
1525
1526         /*
1527          * This is a point of no return; once we modify the uval there is no
1528          * going back and subsequent operations must not fail.
1529          */
1530
1531         raw_spin_lock(&pi_state->owner->pi_lock);
1532         WARN_ON(list_empty(&pi_state->list));
1533         list_del_init(&pi_state->list);
1534         raw_spin_unlock(&pi_state->owner->pi_lock);
1535
1536         raw_spin_lock(&new_owner->pi_lock);
1537         WARN_ON(!list_empty(&pi_state->list));
1538         list_add(&pi_state->list, &new_owner->pi_state_list);
1539         pi_state->owner = new_owner;
1540         raw_spin_unlock(&new_owner->pi_lock);
1541
1542         postunlock = __rt_mutex_futex_unlock(&pi_state->pi_mutex, &wake_q);
1543
1544 out_unlock:
1545         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
1546
1547         if (postunlock)
1548                 rt_mutex_postunlock(&wake_q);
1549
1550         return ret;
1551 }
1552
1553 /*
1554  * Express the locking dependencies for lockdep:
1555  */
1556 static inline void
1557 double_lock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
1558 {
1559         if (hb1 <= hb2) {
1560                 spin_lock(&hb1->lock);
1561                 if (hb1 < hb2)
1562                         spin_lock_nested(&hb2->lock, SINGLE_DEPTH_NESTING);
1563         } else { /* hb1 > hb2 */
1564                 spin_lock(&hb2->lock);
1565                 spin_lock_nested(&hb1->lock, SINGLE_DEPTH_NESTING);
1566         }
1567 }
1568
1569 static inline void
1570 double_unlock_hb(struct futex_hash_bucket *hb1, struct futex_hash_bucket *hb2)
1571 {
1572         spin_unlock(&hb1->lock);
1573         if (hb1 != hb2)
1574                 spin_unlock(&hb2->lock);
1575 }
1576
1577 /*
1578  * Wake up waiters matching bitset queued on this futex (uaddr).
1579  */
1580 static int
1581 futex_wake(u32 __user *uaddr, unsigned int flags, int nr_wake, u32 bitset)
1582 {
1583         struct futex_hash_bucket *hb;
1584         struct futex_q *this, *next;
1585         union futex_key key = FUTEX_KEY_INIT;
1586         int ret;
1587         DEFINE_WAKE_Q(wake_q);
1588
1589         if (!bitset)
1590                 return -EINVAL;
1591
1592         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_READ);
1593         if (unlikely(ret != 0))
1594                 return ret;
1595
1596         hb = hash_futex(&key);
1597
1598         /* Make sure we really have tasks to wakeup */
1599         if (!hb_waiters_pending(hb))
1600                 return ret;
1601
1602         spin_lock(&hb->lock);
1603
1604         plist_for_each_entry_safe(this, next, &hb->chain, list) {
1605                 if (match_futex (&this->key, &key)) {
1606                         if (this->pi_state || this->rt_waiter) {
1607                                 ret = -EINVAL;
1608                                 break;
1609                         }
1610
1611                         /* Check if one of the bits is set in both bitsets */
1612                         if (!(this->bitset & bitset))
1613                                 continue;
1614
1615                         mark_wake_futex(&wake_q, this);
1616                         if (++ret >= nr_wake)
1617                                 break;
1618                 }
1619         }
1620
1621         spin_unlock(&hb->lock);
1622         wake_up_q(&wake_q);
1623         return ret;
1624 }
1625
1626 static int futex_atomic_op_inuser(unsigned int encoded_op, u32 __user *uaddr)
1627 {
1628         unsigned int op =         (encoded_op & 0x70000000) >> 28;
1629         unsigned int cmp =        (encoded_op & 0x0f000000) >> 24;
1630         int oparg = sign_extend32((encoded_op & 0x00fff000) >> 12, 11);
1631         int cmparg = sign_extend32(encoded_op & 0x00000fff, 11);
1632         int oldval, ret;
1633
1634         if (encoded_op & (FUTEX_OP_OPARG_SHIFT << 28)) {
1635                 if (oparg < 0 || oparg > 31) {
1636                         char comm[sizeof(current->comm)];
1637                         /*
1638                          * kill this print and return -EINVAL when userspace
1639                          * is sane again
1640                          */
1641                         pr_info_ratelimited("futex_wake_op: %s tries to shift op by %d; fix this program\n",
1642                                         get_task_comm(comm, current), oparg);
1643                         oparg &= 31;
1644                 }
1645                 oparg = 1 << oparg;
1646         }
1647
1648         pagefault_disable();
1649         ret = arch_futex_atomic_op_inuser(op, oparg, &oldval, uaddr);
1650         pagefault_enable();
1651         if (ret)
1652                 return ret;
1653
1654         switch (cmp) {
1655         case FUTEX_OP_CMP_EQ:
1656                 return oldval == cmparg;
1657         case FUTEX_OP_CMP_NE:
1658                 return oldval != cmparg;
1659         case FUTEX_OP_CMP_LT:
1660                 return oldval < cmparg;
1661         case FUTEX_OP_CMP_GE:
1662                 return oldval >= cmparg;
1663         case FUTEX_OP_CMP_LE:
1664                 return oldval <= cmparg;
1665         case FUTEX_OP_CMP_GT:
1666                 return oldval > cmparg;
1667         default:
1668                 return -ENOSYS;
1669         }
1670 }
1671
1672 /*
1673  * Wake up all waiters hashed on the physical page that is mapped
1674  * to this virtual address:
1675  */
1676 static int
1677 futex_wake_op(u32 __user *uaddr1, unsigned int flags, u32 __user *uaddr2,
1678               int nr_wake, int nr_wake2, int op)
1679 {
1680         union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
1681         struct futex_hash_bucket *hb1, *hb2;
1682         struct futex_q *this, *next;
1683         int ret, op_ret;
1684         DEFINE_WAKE_Q(wake_q);
1685
1686 retry:
1687         ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
1688         if (unlikely(ret != 0))
1689                 return ret;
1690         ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
1691         if (unlikely(ret != 0))
1692                 return ret;
1693
1694         hb1 = hash_futex(&key1);
1695         hb2 = hash_futex(&key2);
1696
1697 retry_private:
1698         double_lock_hb(hb1, hb2);
1699         op_ret = futex_atomic_op_inuser(op, uaddr2);
1700         if (unlikely(op_ret < 0)) {
1701                 double_unlock_hb(hb1, hb2);
1702
1703                 if (!IS_ENABLED(CONFIG_MMU) ||
1704                     unlikely(op_ret != -EFAULT && op_ret != -EAGAIN)) {
1705                         /*
1706                          * we don't get EFAULT from MMU faults if we don't have
1707                          * an MMU, but we might get them from range checking
1708                          */
1709                         ret = op_ret;
1710                         return ret;
1711                 }
1712
1713                 if (op_ret == -EFAULT) {
1714                         ret = fault_in_user_writeable(uaddr2);
1715                         if (ret)
1716                                 return ret;
1717                 }
1718
1719                 if (!(flags & FLAGS_SHARED)) {
1720                         cond_resched();
1721                         goto retry_private;
1722                 }
1723
1724                 cond_resched();
1725                 goto retry;
1726         }
1727
1728         plist_for_each_entry_safe(this, next, &hb1->chain, list) {
1729                 if (match_futex (&this->key, &key1)) {
1730                         if (this->pi_state || this->rt_waiter) {
1731                                 ret = -EINVAL;
1732                                 goto out_unlock;
1733                         }
1734                         mark_wake_futex(&wake_q, this);
1735                         if (++ret >= nr_wake)
1736                                 break;
1737                 }
1738         }
1739
1740         if (op_ret > 0) {
1741                 op_ret = 0;
1742                 plist_for_each_entry_safe(this, next, &hb2->chain, list) {
1743                         if (match_futex (&this->key, &key2)) {
1744                                 if (this->pi_state || this->rt_waiter) {
1745                                         ret = -EINVAL;
1746                                         goto out_unlock;
1747                                 }
1748                                 mark_wake_futex(&wake_q, this);
1749                                 if (++op_ret >= nr_wake2)
1750                                         break;
1751                         }
1752                 }
1753                 ret += op_ret;
1754         }
1755
1756 out_unlock:
1757         double_unlock_hb(hb1, hb2);
1758         wake_up_q(&wake_q);
1759         return ret;
1760 }
1761
1762 /**
1763  * requeue_futex() - Requeue a futex_q from one hb to another
1764  * @q:          the futex_q to requeue
1765  * @hb1:        the source hash_bucket
1766  * @hb2:        the target hash_bucket
1767  * @key2:       the new key for the requeued futex_q
1768  */
1769 static inline
1770 void requeue_futex(struct futex_q *q, struct futex_hash_bucket *hb1,
1771                    struct futex_hash_bucket *hb2, union futex_key *key2)
1772 {
1773
1774         /*
1775          * If key1 and key2 hash to the same bucket, no need to
1776          * requeue.
1777          */
1778         if (likely(&hb1->chain != &hb2->chain)) {
1779                 plist_del(&q->list, &hb1->chain);
1780                 hb_waiters_dec(hb1);
1781                 hb_waiters_inc(hb2);
1782                 plist_add(&q->list, &hb2->chain);
1783                 q->lock_ptr = &hb2->lock;
1784         }
1785         q->key = *key2;
1786 }
1787
1788 /**
1789  * requeue_pi_wake_futex() - Wake a task that acquired the lock during requeue
1790  * @q:          the futex_q
1791  * @key:        the key of the requeue target futex
1792  * @hb:         the hash_bucket of the requeue target futex
1793  *
1794  * During futex_requeue, with requeue_pi=1, it is possible to acquire the
1795  * target futex if it is uncontended or via a lock steal.  Set the futex_q key
1796  * to the requeue target futex so the waiter can detect the wakeup on the right
1797  * futex, but remove it from the hb and NULL the rt_waiter so it can detect
1798  * atomic lock acquisition.  Set the q->lock_ptr to the requeue target hb->lock
1799  * to protect access to the pi_state to fixup the owner later.  Must be called
1800  * with both q->lock_ptr and hb->lock held.
1801  */
1802 static inline
1803 void requeue_pi_wake_futex(struct futex_q *q, union futex_key *key,
1804                            struct futex_hash_bucket *hb)
1805 {
1806         q->key = *key;
1807
1808         __unqueue_futex(q);
1809
1810         WARN_ON(!q->rt_waiter);
1811         q->rt_waiter = NULL;
1812
1813         q->lock_ptr = &hb->lock;
1814
1815         wake_up_state(q->task, TASK_NORMAL);
1816 }
1817
1818 /**
1819  * futex_proxy_trylock_atomic() - Attempt an atomic lock for the top waiter
1820  * @pifutex:            the user address of the to futex
1821  * @hb1:                the from futex hash bucket, must be locked by the caller
1822  * @hb2:                the to futex hash bucket, must be locked by the caller
1823  * @key1:               the from futex key
1824  * @key2:               the to futex key
1825  * @ps:                 address to store the pi_state pointer
1826  * @exiting:            Pointer to store the task pointer of the owner task
1827  *                      which is in the middle of exiting
1828  * @set_waiters:        force setting the FUTEX_WAITERS bit (1) or not (0)
1829  *
1830  * Try and get the lock on behalf of the top waiter if we can do it atomically.
1831  * Wake the top waiter if we succeed.  If the caller specified set_waiters,
1832  * then direct futex_lock_pi_atomic() to force setting the FUTEX_WAITERS bit.
1833  * hb1 and hb2 must be held by the caller.
1834  *
1835  * @exiting is only set when the return value is -EBUSY. If so, this holds
1836  * a refcount on the exiting task on return and the caller needs to drop it
1837  * after waiting for the exit to complete.
1838  *
1839  * Return:
1840  *  -  0 - failed to acquire the lock atomically;
1841  *  - >0 - acquired the lock, return value is vpid of the top_waiter
1842  *  - <0 - error
1843  */
1844 static int
1845 futex_proxy_trylock_atomic(u32 __user *pifutex, struct futex_hash_bucket *hb1,
1846                            struct futex_hash_bucket *hb2, union futex_key *key1,
1847                            union futex_key *key2, struct futex_pi_state **ps,
1848                            struct task_struct **exiting, int set_waiters)
1849 {
1850         struct futex_q *top_waiter = NULL;
1851         u32 curval;
1852         int ret, vpid;
1853
1854         if (get_futex_value_locked(&curval, pifutex))
1855                 return -EFAULT;
1856
1857         if (unlikely(should_fail_futex(true)))
1858                 return -EFAULT;
1859
1860         /*
1861          * Find the top_waiter and determine if there are additional waiters.
1862          * If the caller intends to requeue more than 1 waiter to pifutex,
1863          * force futex_lock_pi_atomic() to set the FUTEX_WAITERS bit now,
1864          * as we have means to handle the possible fault.  If not, don't set
1865          * the bit unecessarily as it will force the subsequent unlock to enter
1866          * the kernel.
1867          */
1868         top_waiter = futex_top_waiter(hb1, key1);
1869
1870         /* There are no waiters, nothing for us to do. */
1871         if (!top_waiter)
1872                 return 0;
1873
1874         /* Ensure we requeue to the expected futex. */
1875         if (!match_futex(top_waiter->requeue_pi_key, key2))
1876                 return -EINVAL;
1877
1878         /*
1879          * Try to take the lock for top_waiter.  Set the FUTEX_WAITERS bit in
1880          * the contended case or if set_waiters is 1.  The pi_state is returned
1881          * in ps in contended cases.
1882          */
1883         vpid = task_pid_vnr(top_waiter->task);
1884         ret = futex_lock_pi_atomic(pifutex, hb2, key2, ps, top_waiter->task,
1885                                    exiting, set_waiters);
1886         if (ret == 1) {
1887                 requeue_pi_wake_futex(top_waiter, key2, hb2);
1888                 return vpid;
1889         }
1890         return ret;
1891 }
1892
1893 /**
1894  * futex_requeue() - Requeue waiters from uaddr1 to uaddr2
1895  * @uaddr1:     source futex user address
1896  * @flags:      futex flags (FLAGS_SHARED, etc.)
1897  * @uaddr2:     target futex user address
1898  * @nr_wake:    number of waiters to wake (must be 1 for requeue_pi)
1899  * @nr_requeue: number of waiters to requeue (0-INT_MAX)
1900  * @cmpval:     @uaddr1 expected value (or %NULL)
1901  * @requeue_pi: if we are attempting to requeue from a non-pi futex to a
1902  *              pi futex (pi to pi requeue is not supported)
1903  *
1904  * Requeue waiters on uaddr1 to uaddr2. In the requeue_pi case, try to acquire
1905  * uaddr2 atomically on behalf of the top waiter.
1906  *
1907  * Return:
1908  *  - >=0 - on success, the number of tasks requeued or woken;
1909  *  -  <0 - on error
1910  */
1911 static int futex_requeue(u32 __user *uaddr1, unsigned int flags,
1912                          u32 __user *uaddr2, int nr_wake, int nr_requeue,
1913                          u32 *cmpval, int requeue_pi)
1914 {
1915         union futex_key key1 = FUTEX_KEY_INIT, key2 = FUTEX_KEY_INIT;
1916         int task_count = 0, ret;
1917         struct futex_pi_state *pi_state = NULL;
1918         struct futex_hash_bucket *hb1, *hb2;
1919         struct futex_q *this, *next;
1920         DEFINE_WAKE_Q(wake_q);
1921
1922         if (nr_wake < 0 || nr_requeue < 0)
1923                 return -EINVAL;
1924
1925         /*
1926          * When PI not supported: return -ENOSYS if requeue_pi is true,
1927          * consequently the compiler knows requeue_pi is always false past
1928          * this point which will optimize away all the conditional code
1929          * further down.
1930          */
1931         if (!IS_ENABLED(CONFIG_FUTEX_PI) && requeue_pi)
1932                 return -ENOSYS;
1933
1934         if (requeue_pi) {
1935                 /*
1936                  * Requeue PI only works on two distinct uaddrs. This
1937                  * check is only valid for private futexes. See below.
1938                  */
1939                 if (uaddr1 == uaddr2)
1940                         return -EINVAL;
1941
1942                 /*
1943                  * requeue_pi requires a pi_state, try to allocate it now
1944                  * without any locks in case it fails.
1945                  */
1946                 if (refill_pi_state_cache())
1947                         return -ENOMEM;
1948                 /*
1949                  * requeue_pi must wake as many tasks as it can, up to nr_wake
1950                  * + nr_requeue, since it acquires the rt_mutex prior to
1951                  * returning to userspace, so as to not leave the rt_mutex with
1952                  * waiters and no owner.  However, second and third wake-ups
1953                  * cannot be predicted as they involve race conditions with the
1954                  * first wake and a fault while looking up the pi_state.  Both
1955                  * pthread_cond_signal() and pthread_cond_broadcast() should
1956                  * use nr_wake=1.
1957                  */
1958                 if (nr_wake != 1)
1959                         return -EINVAL;
1960         }
1961
1962 retry:
1963         ret = get_futex_key(uaddr1, flags & FLAGS_SHARED, &key1, FUTEX_READ);
1964         if (unlikely(ret != 0))
1965                 return ret;
1966         ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2,
1967                             requeue_pi ? FUTEX_WRITE : FUTEX_READ);
1968         if (unlikely(ret != 0))
1969                 return ret;
1970
1971         /*
1972          * The check above which compares uaddrs is not sufficient for
1973          * shared futexes. We need to compare the keys:
1974          */
1975         if (requeue_pi && match_futex(&key1, &key2))
1976                 return -EINVAL;
1977
1978         hb1 = hash_futex(&key1);
1979         hb2 = hash_futex(&key2);
1980
1981 retry_private:
1982         hb_waiters_inc(hb2);
1983         double_lock_hb(hb1, hb2);
1984
1985         if (likely(cmpval != NULL)) {
1986                 u32 curval;
1987
1988                 ret = get_futex_value_locked(&curval, uaddr1);
1989
1990                 if (unlikely(ret)) {
1991                         double_unlock_hb(hb1, hb2);
1992                         hb_waiters_dec(hb2);
1993
1994                         ret = get_user(curval, uaddr1);
1995                         if (ret)
1996                                 return ret;
1997
1998                         if (!(flags & FLAGS_SHARED))
1999                                 goto retry_private;
2000
2001                         goto retry;
2002                 }
2003                 if (curval != *cmpval) {
2004                         ret = -EAGAIN;
2005                         goto out_unlock;
2006                 }
2007         }
2008
2009         if (requeue_pi && (task_count - nr_wake < nr_requeue)) {
2010                 struct task_struct *exiting = NULL;
2011
2012                 /*
2013                  * Attempt to acquire uaddr2 and wake the top waiter. If we
2014                  * intend to requeue waiters, force setting the FUTEX_WAITERS
2015                  * bit.  We force this here where we are able to easily handle
2016                  * faults rather in the requeue loop below.
2017                  */
2018                 ret = futex_proxy_trylock_atomic(uaddr2, hb1, hb2, &key1,
2019                                                  &key2, &pi_state,
2020                                                  &exiting, nr_requeue);
2021
2022                 /*
2023                  * At this point the top_waiter has either taken uaddr2 or is
2024                  * waiting on it.  If the former, then the pi_state will not
2025                  * exist yet, look it up one more time to ensure we have a
2026                  * reference to it. If the lock was taken, ret contains the
2027                  * vpid of the top waiter task.
2028                  * If the lock was not taken, we have pi_state and an initial
2029                  * refcount on it. In case of an error we have nothing.
2030                  */
2031                 if (ret > 0) {
2032                         WARN_ON(pi_state);
2033                         task_count++;
2034                         /*
2035                          * If we acquired the lock, then the user space value
2036                          * of uaddr2 should be vpid. It cannot be changed by
2037                          * the top waiter as it is blocked on hb2 lock if it
2038                          * tries to do so. If something fiddled with it behind
2039                          * our back the pi state lookup might unearth it. So
2040                          * we rather use the known value than rereading and
2041                          * handing potential crap to lookup_pi_state.
2042                          *
2043                          * If that call succeeds then we have pi_state and an
2044                          * initial refcount on it.
2045                          */
2046                         ret = lookup_pi_state(uaddr2, ret, hb2, &key2,
2047                                               &pi_state, &exiting);
2048                 }
2049
2050                 switch (ret) {
2051                 case 0:
2052                         /* We hold a reference on the pi state. */
2053                         break;
2054
2055                         /* If the above failed, then pi_state is NULL */
2056                 case -EFAULT:
2057                         double_unlock_hb(hb1, hb2);
2058                         hb_waiters_dec(hb2);
2059                         ret = fault_in_user_writeable(uaddr2);
2060                         if (!ret)
2061                                 goto retry;
2062                         return ret;
2063                 case -EBUSY:
2064                 case -EAGAIN:
2065                         /*
2066                          * Two reasons for this:
2067                          * - EBUSY: Owner is exiting and we just wait for the
2068                          *   exit to complete.
2069                          * - EAGAIN: The user space value changed.
2070                          */
2071                         double_unlock_hb(hb1, hb2);
2072                         hb_waiters_dec(hb2);
2073                         /*
2074                          * Handle the case where the owner is in the middle of
2075                          * exiting. Wait for the exit to complete otherwise
2076                          * this task might loop forever, aka. live lock.
2077                          */
2078                         wait_for_owner_exiting(ret, exiting);
2079                         cond_resched();
2080                         goto retry;
2081                 default:
2082                         goto out_unlock;
2083                 }
2084         }
2085
2086         plist_for_each_entry_safe(this, next, &hb1->chain, list) {
2087                 if (task_count - nr_wake >= nr_requeue)
2088                         break;
2089
2090                 if (!match_futex(&this->key, &key1))
2091                         continue;
2092
2093                 /*
2094                  * FUTEX_WAIT_REQEUE_PI and FUTEX_CMP_REQUEUE_PI should always
2095                  * be paired with each other and no other futex ops.
2096                  *
2097                  * We should never be requeueing a futex_q with a pi_state,
2098                  * which is awaiting a futex_unlock_pi().
2099                  */
2100                 if ((requeue_pi && !this->rt_waiter) ||
2101                     (!requeue_pi && this->rt_waiter) ||
2102                     this->pi_state) {
2103                         ret = -EINVAL;
2104                         break;
2105                 }
2106
2107                 /*
2108                  * Wake nr_wake waiters.  For requeue_pi, if we acquired the
2109                  * lock, we already woke the top_waiter.  If not, it will be
2110                  * woken by futex_unlock_pi().
2111                  */
2112                 if (++task_count <= nr_wake && !requeue_pi) {
2113                         mark_wake_futex(&wake_q, this);
2114                         continue;
2115                 }
2116
2117                 /* Ensure we requeue to the expected futex for requeue_pi. */
2118                 if (requeue_pi && !match_futex(this->requeue_pi_key, &key2)) {
2119                         ret = -EINVAL;
2120                         break;
2121                 }
2122
2123                 /*
2124                  * Requeue nr_requeue waiters and possibly one more in the case
2125                  * of requeue_pi if we couldn't acquire the lock atomically.
2126                  */
2127                 if (requeue_pi) {
2128                         /*
2129                          * Prepare the waiter to take the rt_mutex. Take a
2130                          * refcount on the pi_state and store the pointer in
2131                          * the futex_q object of the waiter.
2132                          */
2133                         get_pi_state(pi_state);
2134                         this->pi_state = pi_state;
2135                         ret = rt_mutex_start_proxy_lock(&pi_state->pi_mutex,
2136                                                         this->rt_waiter,
2137                                                         this->task);
2138                         if (ret == 1) {
2139                                 /*
2140                                  * We got the lock. We do neither drop the
2141                                  * refcount on pi_state nor clear
2142                                  * this->pi_state because the waiter needs the
2143                                  * pi_state for cleaning up the user space
2144                                  * value. It will drop the refcount after
2145                                  * doing so.
2146                                  */
2147                                 requeue_pi_wake_futex(this, &key2, hb2);
2148                                 continue;
2149                         } else if (ret) {
2150                                 /*
2151                                  * rt_mutex_start_proxy_lock() detected a
2152                                  * potential deadlock when we tried to queue
2153                                  * that waiter. Drop the pi_state reference
2154                                  * which we took above and remove the pointer
2155                                  * to the state from the waiters futex_q
2156                                  * object.
2157                                  */
2158                                 this->pi_state = NULL;
2159                                 put_pi_state(pi_state);
2160                                 /*
2161                                  * We stop queueing more waiters and let user
2162                                  * space deal with the mess.
2163                                  */
2164                                 break;
2165                         }
2166                 }
2167                 requeue_futex(this, hb1, hb2, &key2);
2168         }
2169
2170         /*
2171          * We took an extra initial reference to the pi_state either
2172          * in futex_proxy_trylock_atomic() or in lookup_pi_state(). We
2173          * need to drop it here again.
2174          */
2175         put_pi_state(pi_state);
2176
2177 out_unlock:
2178         double_unlock_hb(hb1, hb2);
2179         wake_up_q(&wake_q);
2180         hb_waiters_dec(hb2);
2181         return ret ? ret : task_count;
2182 }
2183
2184 /* The key must be already stored in q->key. */
2185 static inline struct futex_hash_bucket *queue_lock(struct futex_q *q)
2186         __acquires(&hb->lock)
2187 {
2188         struct futex_hash_bucket *hb;
2189
2190         hb = hash_futex(&q->key);
2191
2192         /*
2193          * Increment the counter before taking the lock so that
2194          * a potential waker won't miss a to-be-slept task that is
2195          * waiting for the spinlock. This is safe as all queue_lock()
2196          * users end up calling queue_me(). Similarly, for housekeeping,
2197          * decrement the counter at queue_unlock() when some error has
2198          * occurred and we don't end up adding the task to the list.
2199          */
2200         hb_waiters_inc(hb); /* implies smp_mb(); (A) */
2201
2202         q->lock_ptr = &hb->lock;
2203
2204         spin_lock(&hb->lock);
2205         return hb;
2206 }
2207
2208 static inline void
2209 queue_unlock(struct futex_hash_bucket *hb)
2210         __releases(&hb->lock)
2211 {
2212         spin_unlock(&hb->lock);
2213         hb_waiters_dec(hb);
2214 }
2215
2216 static inline void __queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
2217 {
2218         int prio;
2219
2220         /*
2221          * The priority used to register this element is
2222          * - either the real thread-priority for the real-time threads
2223          * (i.e. threads with a priority lower than MAX_RT_PRIO)
2224          * - or MAX_RT_PRIO for non-RT threads.
2225          * Thus, all RT-threads are woken first in priority order, and
2226          * the others are woken last, in FIFO order.
2227          */
2228         prio = min(current->normal_prio, MAX_RT_PRIO);
2229
2230         plist_node_init(&q->list, prio);
2231         plist_add(&q->list, &hb->chain);
2232         q->task = current;
2233 }
2234
2235 /**
2236  * queue_me() - Enqueue the futex_q on the futex_hash_bucket
2237  * @q:  The futex_q to enqueue
2238  * @hb: The destination hash bucket
2239  *
2240  * The hb->lock must be held by the caller, and is released here. A call to
2241  * queue_me() is typically paired with exactly one call to unqueue_me().  The
2242  * exceptions involve the PI related operations, which may use unqueue_me_pi()
2243  * or nothing if the unqueue is done as part of the wake process and the unqueue
2244  * state is implicit in the state of woken task (see futex_wait_requeue_pi() for
2245  * an example).
2246  */
2247 static inline void queue_me(struct futex_q *q, struct futex_hash_bucket *hb)
2248         __releases(&hb->lock)
2249 {
2250         __queue_me(q, hb);
2251         spin_unlock(&hb->lock);
2252 }
2253
2254 /**
2255  * unqueue_me() - Remove the futex_q from its futex_hash_bucket
2256  * @q:  The futex_q to unqueue
2257  *
2258  * The q->lock_ptr must not be held by the caller. A call to unqueue_me() must
2259  * be paired with exactly one earlier call to queue_me().
2260  *
2261  * Return:
2262  *  - 1 - if the futex_q was still queued (and we removed unqueued it);
2263  *  - 0 - if the futex_q was already removed by the waking thread
2264  */
2265 static int unqueue_me(struct futex_q *q)
2266 {
2267         spinlock_t *lock_ptr;
2268         int ret = 0;
2269
2270         /* In the common case we don't take the spinlock, which is nice. */
2271 retry:
2272         /*
2273          * q->lock_ptr can change between this read and the following spin_lock.
2274          * Use READ_ONCE to forbid the compiler from reloading q->lock_ptr and
2275          * optimizing lock_ptr out of the logic below.
2276          */
2277         lock_ptr = READ_ONCE(q->lock_ptr);
2278         if (lock_ptr != NULL) {
2279                 spin_lock(lock_ptr);
2280                 /*
2281                  * q->lock_ptr can change between reading it and
2282                  * spin_lock(), causing us to take the wrong lock.  This
2283                  * corrects the race condition.
2284                  *
2285                  * Reasoning goes like this: if we have the wrong lock,
2286                  * q->lock_ptr must have changed (maybe several times)
2287                  * between reading it and the spin_lock().  It can
2288                  * change again after the spin_lock() but only if it was
2289                  * already changed before the spin_lock().  It cannot,
2290                  * however, change back to the original value.  Therefore
2291                  * we can detect whether we acquired the correct lock.
2292                  */
2293                 if (unlikely(lock_ptr != q->lock_ptr)) {
2294                         spin_unlock(lock_ptr);
2295                         goto retry;
2296                 }
2297                 __unqueue_futex(q);
2298
2299                 BUG_ON(q->pi_state);
2300
2301                 spin_unlock(lock_ptr);
2302                 ret = 1;
2303         }
2304
2305         return ret;
2306 }
2307
2308 /*
2309  * PI futexes can not be requeued and must remove themself from the
2310  * hash bucket. The hash bucket lock (i.e. lock_ptr) is held on entry
2311  * and dropped here.
2312  */
2313 static void unqueue_me_pi(struct futex_q *q)
2314         __releases(q->lock_ptr)
2315 {
2316         __unqueue_futex(q);
2317
2318         BUG_ON(!q->pi_state);
2319         put_pi_state(q->pi_state);
2320         q->pi_state = NULL;
2321
2322         spin_unlock(q->lock_ptr);
2323 }
2324
2325 static int fixup_pi_state_owner(u32 __user *uaddr, struct futex_q *q,
2326                                 struct task_struct *argowner)
2327 {
2328         struct futex_pi_state *pi_state = q->pi_state;
2329         u32 uval, curval, newval;
2330         struct task_struct *oldowner, *newowner;
2331         u32 newtid;
2332         int ret, err = 0;
2333
2334         lockdep_assert_held(q->lock_ptr);
2335
2336         raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
2337
2338         oldowner = pi_state->owner;
2339
2340         /*
2341          * We are here because either:
2342          *
2343          *  - we stole the lock and pi_state->owner needs updating to reflect
2344          *    that (@argowner == current),
2345          *
2346          * or:
2347          *
2348          *  - someone stole our lock and we need to fix things to point to the
2349          *    new owner (@argowner == NULL).
2350          *
2351          * Either way, we have to replace the TID in the user space variable.
2352          * This must be atomic as we have to preserve the owner died bit here.
2353          *
2354          * Note: We write the user space value _before_ changing the pi_state
2355          * because we can fault here. Imagine swapped out pages or a fork
2356          * that marked all the anonymous memory readonly for cow.
2357          *
2358          * Modifying pi_state _before_ the user space value would leave the
2359          * pi_state in an inconsistent state when we fault here, because we
2360          * need to drop the locks to handle the fault. This might be observed
2361          * in the PID check in lookup_pi_state.
2362          */
2363 retry:
2364         if (!argowner) {
2365                 if (oldowner != current) {
2366                         /*
2367                          * We raced against a concurrent self; things are
2368                          * already fixed up. Nothing to do.
2369                          */
2370                         ret = 0;
2371                         goto out_unlock;
2372                 }
2373
2374                 if (__rt_mutex_futex_trylock(&pi_state->pi_mutex)) {
2375                         /* We got the lock after all, nothing to fix. */
2376                         ret = 0;
2377                         goto out_unlock;
2378                 }
2379
2380                 /*
2381                  * Since we just failed the trylock; there must be an owner.
2382                  */
2383                 newowner = rt_mutex_owner(&pi_state->pi_mutex);
2384                 BUG_ON(!newowner);
2385         } else {
2386                 WARN_ON_ONCE(argowner != current);
2387                 if (oldowner == current) {
2388                         /*
2389                          * We raced against a concurrent self; things are
2390                          * already fixed up. Nothing to do.
2391                          */
2392                         ret = 0;
2393                         goto out_unlock;
2394                 }
2395                 newowner = argowner;
2396         }
2397
2398         newtid = task_pid_vnr(newowner) | FUTEX_WAITERS;
2399         /* Owner died? */
2400         if (!pi_state->owner)
2401                 newtid |= FUTEX_OWNER_DIED;
2402
2403         err = get_futex_value_locked(&uval, uaddr);
2404         if (err)
2405                 goto handle_err;
2406
2407         for (;;) {
2408                 newval = (uval & FUTEX_OWNER_DIED) | newtid;
2409
2410                 err = cmpxchg_futex_value_locked(&curval, uaddr, uval, newval);
2411                 if (err)
2412                         goto handle_err;
2413
2414                 if (curval == uval)
2415                         break;
2416                 uval = curval;
2417         }
2418
2419         /*
2420          * We fixed up user space. Now we need to fix the pi_state
2421          * itself.
2422          */
2423         if (pi_state->owner != NULL) {
2424                 raw_spin_lock(&pi_state->owner->pi_lock);
2425                 WARN_ON(list_empty(&pi_state->list));
2426                 list_del_init(&pi_state->list);
2427                 raw_spin_unlock(&pi_state->owner->pi_lock);
2428         }
2429
2430         pi_state->owner = newowner;
2431
2432         raw_spin_lock(&newowner->pi_lock);
2433         WARN_ON(!list_empty(&pi_state->list));
2434         list_add(&pi_state->list, &newowner->pi_state_list);
2435         raw_spin_unlock(&newowner->pi_lock);
2436         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2437
2438         return 0;
2439
2440         /*
2441          * In order to reschedule or handle a page fault, we need to drop the
2442          * locks here. In the case of a fault, this gives the other task
2443          * (either the highest priority waiter itself or the task which stole
2444          * the rtmutex) the chance to try the fixup of the pi_state. So once we
2445          * are back from handling the fault we need to check the pi_state after
2446          * reacquiring the locks and before trying to do another fixup. When
2447          * the fixup has been done already we simply return.
2448          *
2449          * Note: we hold both hb->lock and pi_mutex->wait_lock. We can safely
2450          * drop hb->lock since the caller owns the hb -> futex_q relation.
2451          * Dropping the pi_mutex->wait_lock requires the state revalidate.
2452          */
2453 handle_err:
2454         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2455         spin_unlock(q->lock_ptr);
2456
2457         switch (err) {
2458         case -EFAULT:
2459                 ret = fault_in_user_writeable(uaddr);
2460                 break;
2461
2462         case -EAGAIN:
2463                 cond_resched();
2464                 ret = 0;
2465                 break;
2466
2467         default:
2468                 WARN_ON_ONCE(1);
2469                 ret = err;
2470                 break;
2471         }
2472
2473         spin_lock(q->lock_ptr);
2474         raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
2475
2476         /*
2477          * Check if someone else fixed it for us:
2478          */
2479         if (pi_state->owner != oldowner) {
2480                 ret = 0;
2481                 goto out_unlock;
2482         }
2483
2484         if (ret)
2485                 goto out_unlock;
2486
2487         goto retry;
2488
2489 out_unlock:
2490         raw_spin_unlock_irq(&pi_state->pi_mutex.wait_lock);
2491         return ret;
2492 }
2493
2494 static long futex_wait_restart(struct restart_block *restart);
2495
2496 /**
2497  * fixup_owner() - Post lock pi_state and corner case management
2498  * @uaddr:      user address of the futex
2499  * @q:          futex_q (contains pi_state and access to the rt_mutex)
2500  * @locked:     if the attempt to take the rt_mutex succeeded (1) or not (0)
2501  *
2502  * After attempting to lock an rt_mutex, this function is called to cleanup
2503  * the pi_state owner as well as handle race conditions that may allow us to
2504  * acquire the lock. Must be called with the hb lock held.
2505  *
2506  * Return:
2507  *  -  1 - success, lock taken;
2508  *  -  0 - success, lock not taken;
2509  *  - <0 - on error (-EFAULT)
2510  */
2511 static int fixup_owner(u32 __user *uaddr, struct futex_q *q, int locked)
2512 {
2513         int ret = 0;
2514
2515         if (locked) {
2516                 /*
2517                  * Got the lock. We might not be the anticipated owner if we
2518                  * did a lock-steal - fix up the PI-state in that case:
2519                  *
2520                  * Speculative pi_state->owner read (we don't hold wait_lock);
2521                  * since we own the lock pi_state->owner == current is the
2522                  * stable state, anything else needs more attention.
2523                  */
2524                 if (q->pi_state->owner != current)
2525                         ret = fixup_pi_state_owner(uaddr, q, current);
2526                 return ret ? ret : locked;
2527         }
2528
2529         /*
2530          * If we didn't get the lock; check if anybody stole it from us. In
2531          * that case, we need to fix up the uval to point to them instead of
2532          * us, otherwise bad things happen. [10]
2533          *
2534          * Another speculative read; pi_state->owner == current is unstable
2535          * but needs our attention.
2536          */
2537         if (q->pi_state->owner == current) {
2538                 ret = fixup_pi_state_owner(uaddr, q, NULL);
2539                 return ret;
2540         }
2541
2542         /*
2543          * Paranoia check. If we did not take the lock, then we should not be
2544          * the owner of the rt_mutex.
2545          */
2546         if (rt_mutex_owner(&q->pi_state->pi_mutex) == current) {
2547                 printk(KERN_ERR "fixup_owner: ret = %d pi-mutex: %p "
2548                                 "pi-state %p\n", ret,
2549                                 q->pi_state->pi_mutex.owner,
2550                                 q->pi_state->owner);
2551         }
2552
2553         return ret;
2554 }
2555
2556 /**
2557  * futex_wait_queue_me() - queue_me() and wait for wakeup, timeout, or signal
2558  * @hb:         the futex hash bucket, must be locked by the caller
2559  * @q:          the futex_q to queue up on
2560  * @timeout:    the prepared hrtimer_sleeper, or null for no timeout
2561  */
2562 static void futex_wait_queue_me(struct futex_hash_bucket *hb, struct futex_q *q,
2563                                 struct hrtimer_sleeper *timeout)
2564 {
2565         /*
2566          * The task state is guaranteed to be set before another task can
2567          * wake it. set_current_state() is implemented using smp_store_mb() and
2568          * queue_me() calls spin_unlock() upon completion, both serializing
2569          * access to the hash list and forcing another memory barrier.
2570          */
2571         set_current_state(TASK_INTERRUPTIBLE);
2572         queue_me(q, hb);
2573
2574         /* Arm the timer */
2575         if (timeout)
2576                 hrtimer_sleeper_start_expires(timeout, HRTIMER_MODE_ABS);
2577
2578         /*
2579          * If we have been removed from the hash list, then another task
2580          * has tried to wake us, and we can skip the call to schedule().
2581          */
2582         if (likely(!plist_node_empty(&q->list))) {
2583                 /*
2584                  * If the timer has already expired, current will already be
2585                  * flagged for rescheduling. Only call schedule if there
2586                  * is no timeout, or if it has yet to expire.
2587                  */
2588                 if (!timeout || timeout->task)
2589                         freezable_schedule();
2590         }
2591         __set_current_state(TASK_RUNNING);
2592 }
2593
2594 /**
2595  * futex_wait_setup() - Prepare to wait on a futex
2596  * @uaddr:      the futex userspace address
2597  * @val:        the expected value
2598  * @flags:      futex flags (FLAGS_SHARED, etc.)
2599  * @q:          the associated futex_q
2600  * @hb:         storage for hash_bucket pointer to be returned to caller
2601  *
2602  * Setup the futex_q and locate the hash_bucket.  Get the futex value and
2603  * compare it with the expected value.  Handle atomic faults internally.
2604  * Return with the hb lock held and a q.key reference on success, and unlocked
2605  * with no q.key reference on failure.
2606  *
2607  * Return:
2608  *  -  0 - uaddr contains val and hb has been locked;
2609  *  - <1 - -EFAULT or -EWOULDBLOCK (uaddr does not contain val) and hb is unlocked
2610  */
2611 static int futex_wait_setup(u32 __user *uaddr, u32 val, unsigned int flags,
2612                            struct futex_q *q, struct futex_hash_bucket **hb)
2613 {
2614         u32 uval;
2615         int ret;
2616
2617         /*
2618          * Access the page AFTER the hash-bucket is locked.
2619          * Order is important:
2620          *
2621          *   Userspace waiter: val = var; if (cond(val)) futex_wait(&var, val);
2622          *   Userspace waker:  if (cond(var)) { var = new; futex_wake(&var); }
2623          *
2624          * The basic logical guarantee of a futex is that it blocks ONLY
2625          * if cond(var) is known to be true at the time of blocking, for
2626          * any cond.  If we locked the hash-bucket after testing *uaddr, that
2627          * would open a race condition where we could block indefinitely with
2628          * cond(var) false, which would violate the guarantee.
2629          *
2630          * On the other hand, we insert q and release the hash-bucket only
2631          * after testing *uaddr.  This guarantees that futex_wait() will NOT
2632          * absorb a wakeup if *uaddr does not match the desired values
2633          * while the syscall executes.
2634          */
2635 retry:
2636         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q->key, FUTEX_READ);
2637         if (unlikely(ret != 0))
2638                 return ret;
2639
2640 retry_private:
2641         *hb = queue_lock(q);
2642
2643         ret = get_futex_value_locked(&uval, uaddr);
2644
2645         if (ret) {
2646                 queue_unlock(*hb);
2647
2648                 ret = get_user(uval, uaddr);
2649                 if (ret)
2650                         return ret;
2651
2652                 if (!(flags & FLAGS_SHARED))
2653                         goto retry_private;
2654
2655                 goto retry;
2656         }
2657
2658         if (uval != val) {
2659                 queue_unlock(*hb);
2660                 ret = -EWOULDBLOCK;
2661         }
2662
2663         return ret;
2664 }
2665
2666 static int futex_wait(u32 __user *uaddr, unsigned int flags, u32 val,
2667                       ktime_t *abs_time, u32 bitset)
2668 {
2669         struct hrtimer_sleeper timeout, *to;
2670         struct restart_block *restart;
2671         struct futex_hash_bucket *hb;
2672         struct futex_q q = futex_q_init;
2673         int ret;
2674
2675         if (!bitset)
2676                 return -EINVAL;
2677         q.bitset = bitset;
2678
2679         to = futex_setup_timer(abs_time, &timeout, flags,
2680                                current->timer_slack_ns);
2681 retry:
2682         /*
2683          * Prepare to wait on uaddr. On success, holds hb lock and increments
2684          * q.key refs.
2685          */
2686         ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
2687         if (ret)
2688                 goto out;
2689
2690         /* queue_me and wait for wakeup, timeout, or a signal. */
2691         futex_wait_queue_me(hb, &q, to);
2692
2693         /* If we were woken (and unqueued), we succeeded, whatever. */
2694         ret = 0;
2695         /* unqueue_me() drops q.key ref */
2696         if (!unqueue_me(&q))
2697                 goto out;
2698         ret = -ETIMEDOUT;
2699         if (to && !to->task)
2700                 goto out;
2701
2702         /*
2703          * We expect signal_pending(current), but we might be the
2704          * victim of a spurious wakeup as well.
2705          */
2706         if (!signal_pending(current))
2707                 goto retry;
2708
2709         ret = -ERESTARTSYS;
2710         if (!abs_time)
2711                 goto out;
2712
2713         restart = &current->restart_block;
2714         restart->fn = futex_wait_restart;
2715         restart->futex.uaddr = uaddr;
2716         restart->futex.val = val;
2717         restart->futex.time = *abs_time;
2718         restart->futex.bitset = bitset;
2719         restart->futex.flags = flags | FLAGS_HAS_TIMEOUT;
2720
2721         ret = -ERESTART_RESTARTBLOCK;
2722
2723 out:
2724         if (to) {
2725                 hrtimer_cancel(&to->timer);
2726                 destroy_hrtimer_on_stack(&to->timer);
2727         }
2728         return ret;
2729 }
2730
2731
2732 static long futex_wait_restart(struct restart_block *restart)
2733 {
2734         u32 __user *uaddr = restart->futex.uaddr;
2735         ktime_t t, *tp = NULL;
2736
2737         if (restart->futex.flags & FLAGS_HAS_TIMEOUT) {
2738                 t = restart->futex.time;
2739                 tp = &t;
2740         }
2741         restart->fn = do_no_restart_syscall;
2742
2743         return (long)futex_wait(uaddr, restart->futex.flags,
2744                                 restart->futex.val, tp, restart->futex.bitset);
2745 }
2746
2747
2748 /*
2749  * Userspace tried a 0 -> TID atomic transition of the futex value
2750  * and failed. The kernel side here does the whole locking operation:
2751  * if there are waiters then it will block as a consequence of relying
2752  * on rt-mutexes, it does PI, etc. (Due to races the kernel might see
2753  * a 0 value of the futex too.).
2754  *
2755  * Also serves as futex trylock_pi()'ing, and due semantics.
2756  */
2757 static int futex_lock_pi(u32 __user *uaddr, unsigned int flags,
2758                          ktime_t *time, int trylock)
2759 {
2760         struct hrtimer_sleeper timeout, *to;
2761         struct futex_pi_state *pi_state = NULL;
2762         struct task_struct *exiting = NULL;
2763         struct rt_mutex_waiter rt_waiter;
2764         struct futex_hash_bucket *hb;
2765         struct futex_q q = futex_q_init;
2766         int res, ret;
2767
2768         if (!IS_ENABLED(CONFIG_FUTEX_PI))
2769                 return -ENOSYS;
2770
2771         if (refill_pi_state_cache())
2772                 return -ENOMEM;
2773
2774         to = futex_setup_timer(time, &timeout, FLAGS_CLOCKRT, 0);
2775
2776 retry:
2777         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &q.key, FUTEX_WRITE);
2778         if (unlikely(ret != 0))
2779                 goto out;
2780
2781 retry_private:
2782         hb = queue_lock(&q);
2783
2784         ret = futex_lock_pi_atomic(uaddr, hb, &q.key, &q.pi_state, current,
2785                                    &exiting, 0);
2786         if (unlikely(ret)) {
2787                 /*
2788                  * Atomic work succeeded and we got the lock,
2789                  * or failed. Either way, we do _not_ block.
2790                  */
2791                 switch (ret) {
2792                 case 1:
2793                         /* We got the lock. */
2794                         ret = 0;
2795                         goto out_unlock_put_key;
2796                 case -EFAULT:
2797                         goto uaddr_faulted;
2798                 case -EBUSY:
2799                 case -EAGAIN:
2800                         /*
2801                          * Two reasons for this:
2802                          * - EBUSY: Task is exiting and we just wait for the
2803                          *   exit to complete.
2804                          * - EAGAIN: The user space value changed.
2805                          */
2806                         queue_unlock(hb);
2807                         /*
2808                          * Handle the case where the owner is in the middle of
2809                          * exiting. Wait for the exit to complete otherwise
2810                          * this task might loop forever, aka. live lock.
2811                          */
2812                         wait_for_owner_exiting(ret, exiting);
2813                         cond_resched();
2814                         goto retry;
2815                 default:
2816                         goto out_unlock_put_key;
2817                 }
2818         }
2819
2820         WARN_ON(!q.pi_state);
2821
2822         /*
2823          * Only actually queue now that the atomic ops are done:
2824          */
2825         __queue_me(&q, hb);
2826
2827         if (trylock) {
2828                 ret = rt_mutex_futex_trylock(&q.pi_state->pi_mutex);
2829                 /* Fixup the trylock return value: */
2830                 ret = ret ? 0 : -EWOULDBLOCK;
2831                 goto no_block;
2832         }
2833
2834         rt_mutex_init_waiter(&rt_waiter);
2835
2836         /*
2837          * On PREEMPT_RT_FULL, when hb->lock becomes an rt_mutex, we must not
2838          * hold it while doing rt_mutex_start_proxy(), because then it will
2839          * include hb->lock in the blocking chain, even through we'll not in
2840          * fact hold it while blocking. This will lead it to report -EDEADLK
2841          * and BUG when futex_unlock_pi() interleaves with this.
2842          *
2843          * Therefore acquire wait_lock while holding hb->lock, but drop the
2844          * latter before calling __rt_mutex_start_proxy_lock(). This
2845          * interleaves with futex_unlock_pi() -- which does a similar lock
2846          * handoff -- such that the latter can observe the futex_q::pi_state
2847          * before __rt_mutex_start_proxy_lock() is done.
2848          */
2849         raw_spin_lock_irq(&q.pi_state->pi_mutex.wait_lock);
2850         spin_unlock(q.lock_ptr);
2851         /*
2852          * __rt_mutex_start_proxy_lock() unconditionally enqueues the @rt_waiter
2853          * such that futex_unlock_pi() is guaranteed to observe the waiter when
2854          * it sees the futex_q::pi_state.
2855          */
2856         ret = __rt_mutex_start_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter, current);
2857         raw_spin_unlock_irq(&q.pi_state->pi_mutex.wait_lock);
2858
2859         if (ret) {
2860                 if (ret == 1)
2861                         ret = 0;
2862                 goto cleanup;
2863         }
2864
2865         if (unlikely(to))
2866                 hrtimer_sleeper_start_expires(to, HRTIMER_MODE_ABS);
2867
2868         ret = rt_mutex_wait_proxy_lock(&q.pi_state->pi_mutex, to, &rt_waiter);
2869
2870 cleanup:
2871         spin_lock(q.lock_ptr);
2872         /*
2873          * If we failed to acquire the lock (deadlock/signal/timeout), we must
2874          * first acquire the hb->lock before removing the lock from the
2875          * rt_mutex waitqueue, such that we can keep the hb and rt_mutex wait
2876          * lists consistent.
2877          *
2878          * In particular; it is important that futex_unlock_pi() can not
2879          * observe this inconsistency.
2880          */
2881         if (ret && !rt_mutex_cleanup_proxy_lock(&q.pi_state->pi_mutex, &rt_waiter))
2882                 ret = 0;
2883
2884 no_block:
2885         /*
2886          * Fixup the pi_state owner and possibly acquire the lock if we
2887          * haven't already.
2888          */
2889         res = fixup_owner(uaddr, &q, !ret);
2890         /*
2891          * If fixup_owner() returned an error, proprogate that.  If it acquired
2892          * the lock, clear our -ETIMEDOUT or -EINTR.
2893          */
2894         if (res)
2895                 ret = (res < 0) ? res : 0;
2896
2897         /*
2898          * If fixup_owner() faulted and was unable to handle the fault, unlock
2899          * it and return the fault to userspace.
2900          */
2901         if (ret && (rt_mutex_owner(&q.pi_state->pi_mutex) == current)) {
2902                 pi_state = q.pi_state;
2903                 get_pi_state(pi_state);
2904         }
2905
2906         /* Unqueue and drop the lock */
2907         unqueue_me_pi(&q);
2908
2909         if (pi_state) {
2910                 rt_mutex_futex_unlock(&pi_state->pi_mutex);
2911                 put_pi_state(pi_state);
2912         }
2913
2914         goto out;
2915
2916 out_unlock_put_key:
2917         queue_unlock(hb);
2918
2919 out:
2920         if (to) {
2921                 hrtimer_cancel(&to->timer);
2922                 destroy_hrtimer_on_stack(&to->timer);
2923         }
2924         return ret != -EINTR ? ret : -ERESTARTNOINTR;
2925
2926 uaddr_faulted:
2927         queue_unlock(hb);
2928
2929         ret = fault_in_user_writeable(uaddr);
2930         if (ret)
2931                 goto out;
2932
2933         if (!(flags & FLAGS_SHARED))
2934                 goto retry_private;
2935
2936         goto retry;
2937 }
2938
2939 /*
2940  * Userspace attempted a TID -> 0 atomic transition, and failed.
2941  * This is the in-kernel slowpath: we look up the PI state (if any),
2942  * and do the rt-mutex unlock.
2943  */
2944 static int futex_unlock_pi(u32 __user *uaddr, unsigned int flags)
2945 {
2946         u32 curval, uval, vpid = task_pid_vnr(current);
2947         union futex_key key = FUTEX_KEY_INIT;
2948         struct futex_hash_bucket *hb;
2949         struct futex_q *top_waiter;
2950         int ret;
2951
2952         if (!IS_ENABLED(CONFIG_FUTEX_PI))
2953                 return -ENOSYS;
2954
2955 retry:
2956         if (get_user(uval, uaddr))
2957                 return -EFAULT;
2958         /*
2959          * We release only a lock we actually own:
2960          */
2961         if ((uval & FUTEX_TID_MASK) != vpid)
2962                 return -EPERM;
2963
2964         ret = get_futex_key(uaddr, flags & FLAGS_SHARED, &key, FUTEX_WRITE);
2965         if (ret)
2966                 return ret;
2967
2968         hb = hash_futex(&key);
2969         spin_lock(&hb->lock);
2970
2971         /*
2972          * Check waiters first. We do not trust user space values at
2973          * all and we at least want to know if user space fiddled
2974          * with the futex value instead of blindly unlocking.
2975          */
2976         top_waiter = futex_top_waiter(hb, &key);
2977         if (top_waiter) {
2978                 struct futex_pi_state *pi_state = top_waiter->pi_state;
2979
2980                 ret = -EINVAL;
2981                 if (!pi_state)
2982                         goto out_unlock;
2983
2984                 /*
2985                  * If current does not own the pi_state then the futex is
2986                  * inconsistent and user space fiddled with the futex value.
2987                  */
2988                 if (pi_state->owner != current)
2989                         goto out_unlock;
2990
2991                 get_pi_state(pi_state);
2992                 /*
2993                  * By taking wait_lock while still holding hb->lock, we ensure
2994                  * there is no point where we hold neither; and therefore
2995                  * wake_futex_pi() must observe a state consistent with what we
2996                  * observed.
2997                  *
2998                  * In particular; this forces __rt_mutex_start_proxy() to
2999                  * complete such that we're guaranteed to observe the
3000                  * rt_waiter. Also see the WARN in wake_futex_pi().
3001                  */
3002                 raw_spin_lock_irq(&pi_state->pi_mutex.wait_lock);
3003                 spin_unlock(&hb->lock);
3004
3005                 /* drops pi_state->pi_mutex.wait_lock */
3006                 ret = wake_futex_pi(uaddr, uval, pi_state);
3007
3008                 put_pi_state(pi_state);
3009
3010                 /*
3011                  * Success, we're done! No tricky corner cases.
3012                  */
3013                 if (!ret)
3014                         goto out_putkey;
3015                 /*
3016                  * The atomic access to the futex value generated a
3017                  * pagefault, so retry the user-access and the wakeup:
3018                  */
3019                 if (ret == -EFAULT)
3020                         goto pi_faulted;
3021                 /*
3022                  * A unconditional UNLOCK_PI op raced against a waiter
3023                  * setting the FUTEX_WAITERS bit. Try again.
3024                  */
3025                 if (ret == -EAGAIN)
3026                         goto pi_retry;
3027                 /*
3028                  * wake_futex_pi has detected invalid state. Tell user
3029                  * space.
3030                  */
3031                 goto out_putkey;
3032         }
3033
3034         /*
3035          * We have no kernel internal state, i.e. no waiters in the
3036          * kernel. Waiters which are about to queue themselves are stuck
3037          * on hb->lock. So we can safely ignore them. We do neither
3038          * preserve the WAITERS bit not the OWNER_DIED one. We are the
3039          * owner.
3040          */
3041         if ((ret = cmpxchg_futex_value_locked(&curval, uaddr, uval, 0))) {
3042                 spin_unlock(&hb->lock);
3043                 switch (ret) {
3044                 case -EFAULT:
3045                         goto pi_faulted;
3046
3047                 case -EAGAIN:
3048                         goto pi_retry;
3049
3050                 default:
3051                         WARN_ON_ONCE(1);
3052                         goto out_putkey;
3053                 }
3054         }
3055
3056         /*
3057          * If uval has changed, let user space handle it.
3058          */
3059         ret = (curval == uval) ? 0 : -EAGAIN;
3060
3061 out_unlock:
3062         spin_unlock(&hb->lock);
3063 out_putkey:
3064         return ret;
3065
3066 pi_retry:
3067         cond_resched();
3068         goto retry;
3069
3070 pi_faulted:
3071
3072         ret = fault_in_user_writeable(uaddr);
3073         if (!ret)
3074                 goto retry;
3075
3076         return ret;
3077 }
3078
3079 /**
3080  * handle_early_requeue_pi_wakeup() - Detect early wakeup on the initial futex
3081  * @hb:         the hash_bucket futex_q was original enqueued on
3082  * @q:          the futex_q woken while waiting to be requeued
3083  * @key2:       the futex_key of the requeue target futex
3084  * @timeout:    the timeout associated with the wait (NULL if none)
3085  *
3086  * Detect if the task was woken on the initial futex as opposed to the requeue
3087  * target futex.  If so, determine if it was a timeout or a signal that caused
3088  * the wakeup and return the appropriate error code to the caller.  Must be
3089  * called with the hb lock held.
3090  *
3091  * Return:
3092  *  -  0 = no early wakeup detected;
3093  *  - <0 = -ETIMEDOUT or -ERESTARTNOINTR
3094  */
3095 static inline
3096 int handle_early_requeue_pi_wakeup(struct futex_hash_bucket *hb,
3097                                    struct futex_q *q, union futex_key *key2,
3098                                    struct hrtimer_sleeper *timeout)
3099 {
3100         int ret = 0;
3101
3102         /*
3103          * With the hb lock held, we avoid races while we process the wakeup.
3104          * We only need to hold hb (and not hb2) to ensure atomicity as the
3105          * wakeup code can't change q.key from uaddr to uaddr2 if we hold hb.
3106          * It can't be requeued from uaddr2 to something else since we don't
3107          * support a PI aware source futex for requeue.
3108          */
3109         if (!match_futex(&q->key, key2)) {
3110                 WARN_ON(q->lock_ptr && (&hb->lock != q->lock_ptr));
3111                 /*
3112                  * We were woken prior to requeue by a timeout or a signal.
3113                  * Unqueue the futex_q and determine which it was.
3114                  */
3115                 plist_del(&q->list, &hb->chain);
3116                 hb_waiters_dec(hb);
3117
3118                 /* Handle spurious wakeups gracefully */
3119                 ret = -EWOULDBLOCK;
3120                 if (timeout && !timeout->task)
3121                         ret = -ETIMEDOUT;
3122                 else if (signal_pending(current))
3123                         ret = -ERESTARTNOINTR;
3124         }
3125         return ret;
3126 }
3127
3128 /**
3129  * futex_wait_requeue_pi() - Wait on uaddr and take uaddr2
3130  * @uaddr:      the futex we initially wait on (non-pi)
3131  * @flags:      futex flags (FLAGS_SHARED, FLAGS_CLOCKRT, etc.), they must be
3132  *              the same type, no requeueing from private to shared, etc.
3133  * @val:        the expected value of uaddr
3134  * @abs_time:   absolute timeout
3135  * @bitset:     32 bit wakeup bitset set by userspace, defaults to all
3136  * @uaddr2:     the pi futex we will take prior to returning to user-space
3137  *
3138  * The caller will wait on uaddr and will be requeued by futex_requeue() to
3139  * uaddr2 which must be PI aware and unique from uaddr.  Normal wakeup will wake
3140  * on uaddr2 and complete the acquisition of the rt_mutex prior to returning to
3141  * userspace.  This ensures the rt_mutex maintains an owner when it has waiters;
3142  * without one, the pi logic would not know which task to boost/deboost, if
3143  * there was a need to.
3144  *
3145  * We call schedule in futex_wait_queue_me() when we enqueue and return there
3146  * via the following--
3147  * 1) wakeup on uaddr2 after an atomic lock acquisition by futex_requeue()
3148  * 2) wakeup on uaddr2 after a requeue
3149  * 3) signal
3150  * 4) timeout
3151  *
3152  * If 3, cleanup and return -ERESTARTNOINTR.
3153  *
3154  * If 2, we may then block on trying to take the rt_mutex and return via:
3155  * 5) successful lock
3156  * 6) signal
3157  * 7) timeout
3158  * 8) other lock acquisition failure
3159  *
3160  * If 6, return -EWOULDBLOCK (restarting the syscall would do the same).
3161  *
3162  * If 4 or 7, we cleanup and return with -ETIMEDOUT.
3163  *
3164  * Return:
3165  *  -  0 - On success;
3166  *  - <0 - On error
3167  */
3168 static int futex_wait_requeue_pi(u32 __user *uaddr, unsigned int flags,
3169                                  u32 val, ktime_t *abs_time, u32 bitset,
3170                                  u32 __user *uaddr2)
3171 {
3172         struct hrtimer_sleeper timeout, *to;
3173         struct futex_pi_state *pi_state = NULL;
3174         struct rt_mutex_waiter rt_waiter;
3175         struct futex_hash_bucket *hb;
3176         union futex_key key2 = FUTEX_KEY_INIT;
3177         struct futex_q q = futex_q_init;
3178         int res, ret;
3179
3180         if (!IS_ENABLED(CONFIG_FUTEX_PI))
3181                 return -ENOSYS;
3182
3183         if (uaddr == uaddr2)
3184                 return -EINVAL;
3185
3186         if (!bitset)
3187                 return -EINVAL;
3188
3189         to = futex_setup_timer(abs_time, &timeout, flags,
3190                                current->timer_slack_ns);
3191
3192         /*
3193          * The waiter is allocated on our stack, manipulated by the requeue
3194          * code while we sleep on uaddr.
3195          */
3196         rt_mutex_init_waiter(&rt_waiter);
3197
3198         ret = get_futex_key(uaddr2, flags & FLAGS_SHARED, &key2, FUTEX_WRITE);
3199         if (unlikely(ret != 0))
3200                 goto out;
3201
3202         q.bitset = bitset;
3203         q.rt_waiter = &rt_waiter;
3204         q.requeue_pi_key = &key2;
3205
3206         /*
3207          * Prepare to wait on uaddr. On success, increments q.key (key1) ref
3208          * count.
3209          */
3210         ret = futex_wait_setup(uaddr, val, flags, &q, &hb);
3211         if (ret)
3212                 goto out;
3213
3214         /*
3215          * The check above which compares uaddrs is not sufficient for
3216          * shared futexes. We need to compare the keys:
3217          */
3218         if (match_futex(&q.key, &key2)) {
3219                 queue_unlock(hb);
3220                 ret = -EINVAL;
3221                 goto out;
3222         }
3223
3224         /* Queue the futex_q, drop the hb lock, wait for wakeup. */
3225         futex_wait_queue_me(hb, &q, to);
3226
3227         spin_lock(&hb->lock);
3228         ret = handle_early_requeue_pi_wakeup(hb, &q, &key2, to);
3229         spin_unlock(&hb->lock);
3230         if (ret)
3231                 goto out;
3232
3233         /*
3234          * In order for us to be here, we know our q.key == key2, and since
3235          * we took the hb->lock above, we also know that futex_requeue() has
3236          * completed and we no longer have to concern ourselves with a wakeup
3237          * race with the atomic proxy lock acquisition by the requeue code. The
3238          * futex_requeue dropped our key1 reference and incremented our key2
3239          * reference count.
3240          */
3241
3242         /* Check if the requeue code acquired the second futex for us. */
3243         if (!q.rt_waiter) {
3244                 /*
3245                  * Got the lock. We might not be the anticipated owner if we
3246                  * did a lock-steal - fix up the PI-state in that case.
3247                  */
3248                 if (q.pi_state && (q.pi_state->owner != current)) {
3249                         spin_lock(q.lock_ptr);
3250                         ret = fixup_pi_state_owner(uaddr2, &q, current);
3251                         if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
3252                                 pi_state = q.pi_state;
3253                                 get_pi_state(pi_state);
3254                         }
3255                         /*
3256                          * Drop the reference to the pi state which
3257                          * the requeue_pi() code acquired for us.
3258                          */
3259                         put_pi_state(q.pi_state);
3260                         spin_unlock(q.lock_ptr);
3261                 }
3262         } else {
3263                 struct rt_mutex *pi_mutex;
3264
3265                 /*
3266                  * We have been woken up by futex_unlock_pi(), a timeout, or a
3267                  * signal.  futex_unlock_pi() will not destroy the lock_ptr nor
3268                  * the pi_state.
3269                  */
3270                 WARN_ON(!q.pi_state);
3271                 pi_mutex = &q.pi_state->pi_mutex;
3272                 ret = rt_mutex_wait_proxy_lock(pi_mutex, to, &rt_waiter);
3273
3274                 spin_lock(q.lock_ptr);
3275                 if (ret && !rt_mutex_cleanup_proxy_lock(pi_mutex, &rt_waiter))
3276                         ret = 0;
3277
3278                 debug_rt_mutex_free_waiter(&rt_waiter);
3279                 /*
3280                  * Fixup the pi_state owner and possibly acquire the lock if we
3281                  * haven't already.
3282                  */
3283                 res = fixup_owner(uaddr2, &q, !ret);
3284                 /*
3285                  * If fixup_owner() returned an error, proprogate that.  If it
3286                  * acquired the lock, clear -ETIMEDOUT or -EINTR.
3287                  */
3288                 if (res)
3289                         ret = (res < 0) ? res : 0;
3290
3291                 /*
3292                  * If fixup_pi_state_owner() faulted and was unable to handle
3293                  * the fault, unlock the rt_mutex and return the fault to
3294                  * userspace.
3295                  */
3296                 if (ret && rt_mutex_owner(&q.pi_state->pi_mutex) == current) {
3297                         pi_state = q.pi_state;
3298                         get_pi_state(pi_state);
3299                 }
3300
3301                 /* Unqueue and drop the lock. */
3302                 unqueue_me_pi(&q);
3303         }
3304
3305         if (pi_state) {
3306                 rt_mutex_futex_unlock(&pi_state->pi_mutex);
3307                 put_pi_state(pi_state);
3308         }
3309
3310         if (ret == -EINTR) {
3311                 /*
3312                  * We've already been requeued, but cannot restart by calling
3313                  * futex_lock_pi() directly. We could restart this syscall, but
3314                  * it would detect that the user space "val" changed and return
3315                  * -EWOULDBLOCK.  Save the overhead of the restart and return
3316                  * -EWOULDBLOCK directly.
3317                  */
3318                 ret = -EWOULDBLOCK;
3319         }
3320
3321 out:
3322         if (to) {
3323                 hrtimer_cancel(&to->timer);
3324                 destroy_hrtimer_on_stack(&to->timer);
3325         }
3326         return ret;
3327 }
3328
3329 /*
3330  * Support for robust futexes: the kernel cleans up held futexes at
3331  * thread exit time.
3332  *
3333  * Implementation: user-space maintains a per-thread list of locks it
3334  * is holding. Upon do_exit(), the kernel carefully walks this list,
3335  * and marks all locks that are owned by this thread with the
3336  * FUTEX_OWNER_DIED bit, and wakes up a waiter (if any). The list is
3337  * always manipulated with the lock held, so the list is private and
3338  * per-thread. Userspace also maintains a per-thread 'list_op_pending'
3339  * field, to allow the kernel to clean up if the thread dies after
3340  * acquiring the lock, but just before it could have added itself to
3341  * the list. There can only be one such pending lock.
3342  */
3343
3344 /**
3345  * sys_set_robust_list() - Set the robust-futex list head of a task
3346  * @head:       pointer to the list-head
3347  * @len:        length of the list-head, as userspace expects
3348  */
3349 SYSCALL_DEFINE2(set_robust_list, struct robust_list_head __user *, head,
3350                 size_t, len)
3351 {
3352         if (!futex_cmpxchg_enabled)
3353                 return -ENOSYS;
3354         /*
3355          * The kernel knows only one size for now:
3356          */
3357         if (unlikely(len != sizeof(*head)))
3358                 return -EINVAL;
3359
3360         current->robust_list = head;
3361
3362         return 0;
3363 }
3364
3365 /**
3366  * sys_get_robust_list() - Get the robust-futex list head of a task
3367  * @pid:        pid of the process [zero for current task]
3368  * @head_ptr:   pointer to a list-head pointer, the kernel fills it in
3369  * @len_ptr:    pointer to a length field, the kernel fills in the header size
3370  */
3371 SYSCALL_DEFINE3(get_robust_list, int, pid,
3372                 struct robust_list_head __user * __user *, head_ptr,
3373                 size_t __user *, len_ptr)
3374 {
3375         struct robust_list_head __user *head;
3376         unsigned long ret;
3377         struct task_struct *p;
3378
3379         if (!futex_cmpxchg_enabled)
3380                 return -ENOSYS;
3381
3382         rcu_read_lock();
3383
3384         ret = -ESRCH;
3385         if (!pid)
3386                 p = current;
3387         else {
3388                 p = find_task_by_vpid(pid);
3389                 if (!p)
3390                         goto err_unlock;
3391         }
3392
3393         ret = -EPERM;
3394         if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
3395                 goto err_unlock;
3396
3397         head = p->robust_list;
3398         rcu_read_unlock();
3399
3400         if (put_user(sizeof(*head), len_ptr))
3401                 return -EFAULT;
3402         return put_user(head, head_ptr);
3403
3404 err_unlock:
3405         rcu_read_unlock();
3406
3407         return ret;
3408 }
3409
3410 /* Constants for the pending_op argument of handle_futex_death */
3411 #define HANDLE_DEATH_PENDING    true
3412 #define HANDLE_DEATH_LIST       false
3413
3414 /*
3415  * Process a futex-list entry, check whether it's owned by the
3416  * dying task, and do notification if so:
3417  */
3418 static int handle_futex_death(u32 __user *uaddr, struct task_struct *curr,
3419                               bool pi, bool pending_op)
3420 {
3421         u32 uval, nval, mval;
3422         int err;
3423
3424         /* Futex address must be 32bit aligned */
3425         if ((((unsigned long)uaddr) % sizeof(*uaddr)) != 0)
3426                 return -1;
3427
3428 retry:
3429         if (get_user(uval, uaddr))
3430                 return -1;
3431
3432         /*
3433          * Special case for regular (non PI) futexes. The unlock path in
3434          * user space has two race scenarios:
3435          *
3436          * 1. The unlock path releases the user space futex value and
3437          *    before it can execute the futex() syscall to wake up
3438          *    waiters it is killed.
3439          *
3440          * 2. A woken up waiter is killed before it can acquire the
3441          *    futex in user space.
3442          *
3443          * In both cases the TID validation below prevents a wakeup of
3444          * potential waiters which can cause these waiters to block
3445          * forever.
3446          *
3447          * In both cases the following conditions are met:
3448          *
3449          *      1) task->robust_list->list_op_pending != NULL
3450          *         @pending_op == true
3451          *      2) User space futex value == 0
3452          *      3) Regular futex: @pi == false
3453          *
3454          * If these conditions are met, it is safe to attempt waking up a
3455          * potential waiter without touching the user space futex value and
3456          * trying to set the OWNER_DIED bit. The user space futex value is
3457          * uncontended and the rest of the user space mutex state is
3458          * consistent, so a woken waiter will just take over the
3459          * uncontended futex. Setting the OWNER_DIED bit would create
3460          * inconsistent state and malfunction of the user space owner died
3461          * handling.
3462          */
3463         if (pending_op && !pi && !uval) {
3464                 futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
3465                 return 0;
3466         }
3467
3468         if ((uval & FUTEX_TID_MASK) != task_pid_vnr(curr))
3469                 return 0;
3470
3471         /*
3472          * Ok, this dying thread is truly holding a futex
3473          * of interest. Set the OWNER_DIED bit atomically
3474          * via cmpxchg, and if the value had FUTEX_WAITERS
3475          * set, wake up a waiter (if any). (We have to do a
3476          * futex_wake() even if OWNER_DIED is already set -
3477          * to handle the rare but possible case of recursive
3478          * thread-death.) The rest of the cleanup is done in
3479          * userspace.
3480          */
3481         mval = (uval & FUTEX_WAITERS) | FUTEX_OWNER_DIED;
3482
3483         /*
3484          * We are not holding a lock here, but we want to have
3485          * the pagefault_disable/enable() protection because
3486          * we want to handle the fault gracefully. If the
3487          * access fails we try to fault in the futex with R/W
3488          * verification via get_user_pages. get_user() above
3489          * does not guarantee R/W access. If that fails we
3490          * give up and leave the futex locked.
3491          */
3492         if ((err = cmpxchg_futex_value_locked(&nval, uaddr, uval, mval))) {
3493                 switch (err) {
3494                 case -EFAULT:
3495                         if (fault_in_user_writeable(uaddr))
3496                                 return -1;
3497                         goto retry;
3498
3499                 case -EAGAIN:
3500                         cond_resched();
3501                         goto retry;
3502
3503                 default:
3504                         WARN_ON_ONCE(1);
3505                         return err;
3506                 }
3507         }
3508
3509         if (nval != uval)
3510                 goto retry;
3511
3512         /*
3513          * Wake robust non-PI futexes here. The wakeup of
3514          * PI futexes happens in exit_pi_state():
3515          */
3516         if (!pi && (uval & FUTEX_WAITERS))
3517                 futex_wake(uaddr, 1, 1, FUTEX_BITSET_MATCH_ANY);
3518
3519         return 0;
3520 }
3521
3522 /*
3523  * Fetch a robust-list pointer. Bit 0 signals PI futexes:
3524  */
3525 static inline int fetch_robust_entry(struct robust_list __user **entry,
3526                                      struct robust_list __user * __user *head,
3527                                      unsigned int *pi)
3528 {
3529         unsigned long uentry;
3530
3531         if (get_user(uentry, (unsigned long __user *)head))
3532                 return -EFAULT;
3533
3534         *entry = (void __user *)(uentry & ~1UL);
3535         *pi = uentry & 1;
3536
3537         return 0;
3538 }
3539
3540 /*
3541  * Walk curr->robust_list (very carefully, it's a userspace list!)
3542  * and mark any locks found there dead, and notify any waiters.
3543  *
3544  * We silently return on any sign of list-walking problem.
3545  */
3546 static void exit_robust_list(struct task_struct *curr)
3547 {
3548         struct robust_list_head __user *head = curr->robust_list;
3549         struct robust_list __user *entry, *next_entry, *pending;
3550         unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
3551         unsigned int next_pi;
3552         unsigned long futex_offset;
3553         int rc;
3554
3555         if (!futex_cmpxchg_enabled)
3556                 return;
3557
3558         /*
3559          * Fetch the list head (which was registered earlier, via
3560          * sys_set_robust_list()):
3561          */
3562         if (fetch_robust_entry(&entry, &head->list.next, &pi))
3563                 return;
3564         /*
3565          * Fetch the relative futex offset:
3566          */
3567         if (get_user(futex_offset, &head->futex_offset))
3568                 return;
3569         /*
3570          * Fetch any possibly pending lock-add first, and handle it
3571          * if it exists:
3572          */
3573         if (fetch_robust_entry(&pending, &head->list_op_pending, &pip))
3574                 return;
3575
3576         next_entry = NULL;      /* avoid warning with gcc */
3577         while (entry != &head->list) {
3578                 /*
3579                  * Fetch the next entry in the list before calling
3580                  * handle_futex_death:
3581                  */
3582                 rc = fetch_robust_entry(&next_entry, &entry->next, &next_pi);
3583                 /*
3584                  * A pending lock might already be on the list, so
3585                  * don't process it twice:
3586                  */
3587                 if (entry != pending) {
3588                         if (handle_futex_death((void __user *)entry + futex_offset,
3589                                                 curr, pi, HANDLE_DEATH_LIST))
3590                                 return;
3591                 }
3592                 if (rc)
3593                         return;
3594                 entry = next_entry;
3595                 pi = next_pi;
3596                 /*
3597                  * Avoid excessively long or circular lists:
3598                  */
3599                 if (!--limit)
3600                         break;
3601
3602                 cond_resched();
3603         }
3604
3605         if (pending) {
3606                 handle_futex_death((void __user *)pending + futex_offset,
3607                                    curr, pip, HANDLE_DEATH_PENDING);
3608         }
3609 }
3610
3611 static void futex_cleanup(struct task_struct *tsk)
3612 {
3613         if (unlikely(tsk->robust_list)) {
3614                 exit_robust_list(tsk);
3615                 tsk->robust_list = NULL;
3616         }
3617
3618 #ifdef CONFIG_COMPAT
3619         if (unlikely(tsk->compat_robust_list)) {
3620                 compat_exit_robust_list(tsk);
3621                 tsk->compat_robust_list = NULL;
3622         }
3623 #endif
3624
3625         if (unlikely(!list_empty(&tsk->pi_state_list)))
3626                 exit_pi_state_list(tsk);
3627 }
3628
3629 /**
3630  * futex_exit_recursive - Set the tasks futex state to FUTEX_STATE_DEAD
3631  * @tsk:        task to set the state on
3632  *
3633  * Set the futex exit state of the task lockless. The futex waiter code
3634  * observes that state when a task is exiting and loops until the task has
3635  * actually finished the futex cleanup. The worst case for this is that the
3636  * waiter runs through the wait loop until the state becomes visible.
3637  *
3638  * This is called from the recursive fault handling path in do_exit().
3639  *
3640  * This is best effort. Either the futex exit code has run already or
3641  * not. If the OWNER_DIED bit has been set on the futex then the waiter can
3642  * take it over. If not, the problem is pushed back to user space. If the
3643  * futex exit code did not run yet, then an already queued waiter might
3644  * block forever, but there is nothing which can be done about that.
3645  */
3646 void futex_exit_recursive(struct task_struct *tsk)
3647 {
3648         /* If the state is FUTEX_STATE_EXITING then futex_exit_mutex is held */
3649         if (tsk->futex_state == FUTEX_STATE_EXITING)
3650                 mutex_unlock(&tsk->futex_exit_mutex);
3651         tsk->futex_state = FUTEX_STATE_DEAD;
3652 }
3653
3654 static void futex_cleanup_begin(struct task_struct *tsk)
3655 {
3656         /*
3657          * Prevent various race issues against a concurrent incoming waiter
3658          * including live locks by forcing the waiter to block on
3659          * tsk->futex_exit_mutex when it observes FUTEX_STATE_EXITING in
3660          * attach_to_pi_owner().
3661          */
3662         mutex_lock(&tsk->futex_exit_mutex);
3663
3664         /*
3665          * Switch the state to FUTEX_STATE_EXITING under tsk->pi_lock.
3666          *
3667          * This ensures that all subsequent checks of tsk->futex_state in
3668          * attach_to_pi_owner() must observe FUTEX_STATE_EXITING with
3669          * tsk->pi_lock held.
3670          *
3671          * It guarantees also that a pi_state which was queued right before
3672          * the state change under tsk->pi_lock by a concurrent waiter must
3673          * be observed in exit_pi_state_list().
3674          */
3675         raw_spin_lock_irq(&tsk->pi_lock);
3676         tsk->futex_state = FUTEX_STATE_EXITING;
3677         raw_spin_unlock_irq(&tsk->pi_lock);
3678 }
3679
3680 static void futex_cleanup_end(struct task_struct *tsk, int state)
3681 {
3682         /*
3683          * Lockless store. The only side effect is that an observer might
3684          * take another loop until it becomes visible.
3685          */
3686         tsk->futex_state = state;
3687         /*
3688          * Drop the exit protection. This unblocks waiters which observed
3689          * FUTEX_STATE_EXITING to reevaluate the state.
3690          */
3691         mutex_unlock(&tsk->futex_exit_mutex);
3692 }
3693
3694 void futex_exec_release(struct task_struct *tsk)
3695 {
3696         /*
3697          * The state handling is done for consistency, but in the case of
3698          * exec() there is no way to prevent futher damage as the PID stays
3699          * the same. But for the unlikely and arguably buggy case that a
3700          * futex is held on exec(), this provides at least as much state
3701          * consistency protection which is possible.
3702          */
3703         futex_cleanup_begin(tsk);
3704         futex_cleanup(tsk);
3705         /*
3706          * Reset the state to FUTEX_STATE_OK. The task is alive and about
3707          * exec a new binary.
3708          */
3709         futex_cleanup_end(tsk, FUTEX_STATE_OK);
3710 }
3711
3712 void futex_exit_release(struct task_struct *tsk)
3713 {
3714         futex_cleanup_begin(tsk);
3715         futex_cleanup(tsk);
3716         futex_cleanup_end(tsk, FUTEX_STATE_DEAD);
3717 }
3718
3719 long do_futex(u32 __user *uaddr, int op, u32 val, ktime_t *timeout,
3720                 u32 __user *uaddr2, u32 val2, u32 val3)
3721 {
3722         int cmd = op & FUTEX_CMD_MASK;
3723         unsigned int flags = 0;
3724
3725         if (!(op & FUTEX_PRIVATE_FLAG))
3726                 flags |= FLAGS_SHARED;
3727
3728         if (op & FUTEX_CLOCK_REALTIME) {
3729                 flags |= FLAGS_CLOCKRT;
3730                 if (cmd != FUTEX_WAIT && cmd != FUTEX_WAIT_BITSET && \
3731                     cmd != FUTEX_WAIT_REQUEUE_PI)
3732                         return -ENOSYS;
3733         }
3734
3735         switch (cmd) {
3736         case FUTEX_LOCK_PI:
3737         case FUTEX_UNLOCK_PI:
3738         case FUTEX_TRYLOCK_PI:
3739         case FUTEX_WAIT_REQUEUE_PI:
3740         case FUTEX_CMP_REQUEUE_PI:
3741                 if (!futex_cmpxchg_enabled)
3742                         return -ENOSYS;
3743         }
3744
3745         switch (cmd) {
3746         case FUTEX_WAIT:
3747                 val3 = FUTEX_BITSET_MATCH_ANY;
3748                 fallthrough;
3749         case FUTEX_WAIT_BITSET:
3750                 return futex_wait(uaddr, flags, val, timeout, val3);
3751         case FUTEX_WAKE:
3752                 val3 = FUTEX_BITSET_MATCH_ANY;
3753                 fallthrough;
3754         case FUTEX_WAKE_BITSET:
3755                 return futex_wake(uaddr, flags, val, val3);
3756         case FUTEX_REQUEUE:
3757                 return futex_requeue(uaddr, flags, uaddr2, val, val2, NULL, 0);
3758         case FUTEX_CMP_REQUEUE:
3759                 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 0);
3760         case FUTEX_WAKE_OP:
3761                 return futex_wake_op(uaddr, flags, uaddr2, val, val2, val3);
3762         case FUTEX_LOCK_PI:
3763                 return futex_lock_pi(uaddr, flags, timeout, 0);
3764         case FUTEX_UNLOCK_PI:
3765                 return futex_unlock_pi(uaddr, flags);
3766         case FUTEX_TRYLOCK_PI:
3767                 return futex_lock_pi(uaddr, flags, NULL, 1);
3768         case FUTEX_WAIT_REQUEUE_PI:
3769                 val3 = FUTEX_BITSET_MATCH_ANY;
3770                 return futex_wait_requeue_pi(uaddr, flags, val, timeout, val3,
3771                                              uaddr2);
3772         case FUTEX_CMP_REQUEUE_PI:
3773                 return futex_requeue(uaddr, flags, uaddr2, val, val2, &val3, 1);
3774         }
3775         return -ENOSYS;
3776 }
3777
3778
3779 SYSCALL_DEFINE6(futex, u32 __user *, uaddr, int, op, u32, val,
3780                 struct __kernel_timespec __user *, utime, u32 __user *, uaddr2,
3781                 u32, val3)
3782 {
3783         struct timespec64 ts;
3784         ktime_t t, *tp = NULL;
3785         u32 val2 = 0;
3786         int cmd = op & FUTEX_CMD_MASK;
3787
3788         if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
3789                       cmd == FUTEX_WAIT_BITSET ||
3790                       cmd == FUTEX_WAIT_REQUEUE_PI)) {
3791                 if (unlikely(should_fail_futex(!(op & FUTEX_PRIVATE_FLAG))))
3792                         return -EFAULT;
3793                 if (get_timespec64(&ts, utime))
3794                         return -EFAULT;
3795                 if (!timespec64_valid(&ts))
3796                         return -EINVAL;
3797
3798                 t = timespec64_to_ktime(ts);
3799                 if (cmd == FUTEX_WAIT)
3800                         t = ktime_add_safe(ktime_get(), t);
3801                 else if (!(op & FUTEX_CLOCK_REALTIME))
3802                         t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
3803                 tp = &t;
3804         }
3805         /*
3806          * requeue parameter in 'utime' if cmd == FUTEX_*_REQUEUE_*.
3807          * number of waiters to wake in 'utime' if cmd == FUTEX_WAKE_OP.
3808          */
3809         if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
3810             cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
3811                 val2 = (u32) (unsigned long) utime;
3812
3813         return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
3814 }
3815
3816 #ifdef CONFIG_COMPAT
3817 /*
3818  * Fetch a robust-list pointer. Bit 0 signals PI futexes:
3819  */
3820 static inline int
3821 compat_fetch_robust_entry(compat_uptr_t *uentry, struct robust_list __user **entry,
3822                    compat_uptr_t __user *head, unsigned int *pi)
3823 {
3824         if (get_user(*uentry, head))
3825                 return -EFAULT;
3826
3827         *entry = compat_ptr((*uentry) & ~1);
3828         *pi = (unsigned int)(*uentry) & 1;
3829
3830         return 0;
3831 }
3832
3833 static void __user *futex_uaddr(struct robust_list __user *entry,
3834                                 compat_long_t futex_offset)
3835 {
3836         compat_uptr_t base = ptr_to_compat(entry);
3837         void __user *uaddr = compat_ptr(base + futex_offset);
3838
3839         return uaddr;
3840 }
3841
3842 /*
3843  * Walk curr->robust_list (very carefully, it's a userspace list!)
3844  * and mark any locks found there dead, and notify any waiters.
3845  *
3846  * We silently return on any sign of list-walking problem.
3847  */
3848 static void compat_exit_robust_list(struct task_struct *curr)
3849 {
3850         struct compat_robust_list_head __user *head = curr->compat_robust_list;
3851         struct robust_list __user *entry, *next_entry, *pending;
3852         unsigned int limit = ROBUST_LIST_LIMIT, pi, pip;
3853         unsigned int next_pi;
3854         compat_uptr_t uentry, next_uentry, upending;
3855         compat_long_t futex_offset;
3856         int rc;
3857
3858         if (!futex_cmpxchg_enabled)
3859                 return;
3860
3861         /*
3862          * Fetch the list head (which was registered earlier, via
3863          * sys_set_robust_list()):
3864          */
3865         if (compat_fetch_robust_entry(&uentry, &entry, &head->list.next, &pi))
3866                 return;
3867         /*
3868          * Fetch the relative futex offset:
3869          */
3870         if (get_user(futex_offset, &head->futex_offset))
3871                 return;
3872         /*
3873          * Fetch any possibly pending lock-add first, and handle it
3874          * if it exists:
3875          */
3876         if (compat_fetch_robust_entry(&upending, &pending,
3877                                &head->list_op_pending, &pip))
3878                 return;
3879
3880         next_entry = NULL;      /* avoid warning with gcc */
3881         while (entry != (struct robust_list __user *) &head->list) {
3882                 /*
3883                  * Fetch the next entry in the list before calling
3884                  * handle_futex_death:
3885                  */
3886                 rc = compat_fetch_robust_entry(&next_uentry, &next_entry,
3887                         (compat_uptr_t __user *)&entry->next, &next_pi);
3888                 /*
3889                  * A pending lock might already be on the list, so
3890                  * dont process it twice:
3891                  */
3892                 if (entry != pending) {
3893                         void __user *uaddr = futex_uaddr(entry, futex_offset);
3894
3895                         if (handle_futex_death(uaddr, curr, pi,
3896                                                HANDLE_DEATH_LIST))
3897                                 return;
3898                 }
3899                 if (rc)
3900                         return;
3901                 uentry = next_uentry;
3902                 entry = next_entry;
3903                 pi = next_pi;
3904                 /*
3905                  * Avoid excessively long or circular lists:
3906                  */
3907                 if (!--limit)
3908                         break;
3909
3910                 cond_resched();
3911         }
3912         if (pending) {
3913                 void __user *uaddr = futex_uaddr(pending, futex_offset);
3914
3915                 handle_futex_death(uaddr, curr, pip, HANDLE_DEATH_PENDING);
3916         }
3917 }
3918
3919 COMPAT_SYSCALL_DEFINE2(set_robust_list,
3920                 struct compat_robust_list_head __user *, head,
3921                 compat_size_t, len)
3922 {
3923         if (!futex_cmpxchg_enabled)
3924                 return -ENOSYS;
3925
3926         if (unlikely(len != sizeof(*head)))
3927                 return -EINVAL;
3928
3929         current->compat_robust_list = head;
3930
3931         return 0;
3932 }
3933
3934 COMPAT_SYSCALL_DEFINE3(get_robust_list, int, pid,
3935                         compat_uptr_t __user *, head_ptr,
3936                         compat_size_t __user *, len_ptr)
3937 {
3938         struct compat_robust_list_head __user *head;
3939         unsigned long ret;
3940         struct task_struct *p;
3941
3942         if (!futex_cmpxchg_enabled)
3943                 return -ENOSYS;
3944
3945         rcu_read_lock();
3946
3947         ret = -ESRCH;
3948         if (!pid)
3949                 p = current;
3950         else {
3951                 p = find_task_by_vpid(pid);
3952                 if (!p)
3953                         goto err_unlock;
3954         }
3955
3956         ret = -EPERM;
3957         if (!ptrace_may_access(p, PTRACE_MODE_READ_REALCREDS))
3958                 goto err_unlock;
3959
3960         head = p->compat_robust_list;
3961         rcu_read_unlock();
3962
3963         if (put_user(sizeof(*head), len_ptr))
3964                 return -EFAULT;
3965         return put_user(ptr_to_compat(head), head_ptr);
3966
3967 err_unlock:
3968         rcu_read_unlock();
3969
3970         return ret;
3971 }
3972 #endif /* CONFIG_COMPAT */
3973
3974 #ifdef CONFIG_COMPAT_32BIT_TIME
3975 SYSCALL_DEFINE6(futex_time32, u32 __user *, uaddr, int, op, u32, val,
3976                 struct old_timespec32 __user *, utime, u32 __user *, uaddr2,
3977                 u32, val3)
3978 {
3979         struct timespec64 ts;
3980         ktime_t t, *tp = NULL;
3981         int val2 = 0;
3982         int cmd = op & FUTEX_CMD_MASK;
3983
3984         if (utime && (cmd == FUTEX_WAIT || cmd == FUTEX_LOCK_PI ||
3985                       cmd == FUTEX_WAIT_BITSET ||
3986                       cmd == FUTEX_WAIT_REQUEUE_PI)) {
3987                 if (get_old_timespec32(&ts, utime))
3988                         return -EFAULT;
3989                 if (!timespec64_valid(&ts))
3990                         return -EINVAL;
3991
3992                 t = timespec64_to_ktime(ts);
3993                 if (cmd == FUTEX_WAIT)
3994                         t = ktime_add_safe(ktime_get(), t);
3995                 else if (!(op & FUTEX_CLOCK_REALTIME))
3996                         t = timens_ktime_to_host(CLOCK_MONOTONIC, t);
3997                 tp = &t;
3998         }
3999         if (cmd == FUTEX_REQUEUE || cmd == FUTEX_CMP_REQUEUE ||
4000             cmd == FUTEX_CMP_REQUEUE_PI || cmd == FUTEX_WAKE_OP)
4001                 val2 = (int) (unsigned long) utime;
4002
4003         return do_futex(uaddr, op, val, tp, uaddr2, val2, val3);
4004 }
4005 #endif /* CONFIG_COMPAT_32BIT_TIME */
4006
4007 static void __init futex_detect_cmpxchg(void)
4008 {
4009 #ifndef CONFIG_HAVE_FUTEX_CMPXCHG
4010         u32 curval;
4011
4012         /*
4013          * This will fail and we want it. Some arch implementations do
4014          * runtime detection of the futex_atomic_cmpxchg_inatomic()
4015          * functionality. We want to know that before we call in any
4016          * of the complex code paths. Also we want to prevent
4017          * registration of robust lists in that case. NULL is
4018          * guaranteed to fault and we get -EFAULT on functional
4019          * implementation, the non-functional ones will return
4020          * -ENOSYS.
4021          */
4022         if (cmpxchg_futex_value_locked(&curval, NULL, 0, 0) == -EFAULT)
4023                 futex_cmpxchg_enabled = 1;
4024 #endif
4025 }
4026
4027 static int __init futex_init(void)
4028 {
4029         unsigned int futex_shift;
4030         unsigned long i;
4031
4032 #if CONFIG_BASE_SMALL
4033         futex_hashsize = 16;
4034 #else
4035         futex_hashsize = roundup_pow_of_two(256 * num_possible_cpus());
4036 #endif
4037
4038         futex_queues = alloc_large_system_hash("futex", sizeof(*futex_queues),
4039                                                futex_hashsize, 0,
4040                                                futex_hashsize < 256 ? HASH_SMALL : 0,
4041                                                &futex_shift, NULL,
4042                                                futex_hashsize, futex_hashsize);
4043         futex_hashsize = 1UL << futex_shift;
4044
4045         futex_detect_cmpxchg();
4046
4047         for (i = 0; i < futex_hashsize; i++) {
4048                 atomic_set(&futex_queues[i].waiters, 0);
4049                 plist_head_init(&futex_queues[i].chain);
4050                 spin_lock_init(&futex_queues[i].lock);
4051         }
4052
4053         return 0;
4054 }
4055 core_initcall(futex_init);