kernel/locking/qrwlock.c

   1 // SPDX-License-Identifier: GPL-2.0-or-later
   2 /*
   3  * Queued read/write locks
   4  *
   5  * (C) Copyright 2013-2014 Hewlett-Packard Development Company, L.P.
   6  *
   7  * Authors: Waiman Long <waiman.long@hp.com>
   8  */
   9 #include <linux/smp.h>
  10 #include <linux/bug.h>
  11 #include <linux/cpumask.h>
  12 #include <linux/percpu.h>
  13 #include <linux/hardirq.h>
  14 #include <linux/spinlock.h>
  15
  16 /**
  17  * queued_read_lock_slowpath - acquire read lock of a queue rwlock
  18  * @lock: Pointer to queue rwlock structure
  19  */
  20 void queued_read_lock_slowpath(struct qrwlock *lock)
  21 {
  22         /*
  23          * Readers come here when they cannot get the lock without waiting
  24          */
  25         if (unlikely(in_interrupt())) {
  26                 /*
  27                  * Readers in interrupt context will get the lock immediately
  28                  * if the writer is just waiting (not holding the lock yet),
  29                  * so spin with ACQUIRE semantics until the lock is available
  30                  * without waiting in the queue.
  31                  */
  32                 atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
  33                 return;
  34         }
  35         atomic_sub(_QR_BIAS, &lock->cnts);
  36
  37         /*
  38          * Put the reader into the wait queue
  39          */
  40         arch_spin_lock(&lock->wait_lock);
  41         atomic_add(_QR_BIAS, &lock->cnts);
  42
  43         /*
  44          * The ACQUIRE semantics of the following spinning code ensure
  45          * that accesses can't leak upwards out of our subsequent critical
  46          * section in the case that the lock is currently held for write.
  47          */
  48         atomic_cond_read_acquire(&lock->cnts, !(VAL & _QW_LOCKED));
  49
  50         /*
  51          * Signal the next one in queue to become queue head
  52          */
  53         arch_spin_unlock(&lock->wait_lock);
  54 }
  55 EXPORT_SYMBOL(queued_read_lock_slowpath);
  56
  57 /**
  58  * queued_write_lock_slowpath - acquire write lock of a queue rwlock
  59  * @lock : Pointer to queue rwlock structure
  60  */
  61 void queued_write_lock_slowpath(struct qrwlock *lock)
  62 {
  63         int cnts;
  64
  65         /* Put the writer into the wait queue */
  66         arch_spin_lock(&lock->wait_lock);
  67
  68         /* Try to acquire the lock directly if no reader is present */
  69         if (!(cnts = atomic_read(&lock->cnts)) &&
  70             atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED))
  71                 goto unlock;
  72
  73         /* Set the waiting flag to notify readers that a writer is pending */
  74         atomic_or(_QW_WAITING, &lock->cnts);
  75
  76         /* When no more readers or writers, set the locked flag */
  77         do {
  78                 cnts = atomic_cond_read_relaxed(&lock->cnts, VAL == _QW_WAITING);
  79         } while (!atomic_try_cmpxchg_acquire(&lock->cnts, &cnts, _QW_LOCKED));
  80 unlock:
  81         arch_spin_unlock(&lock->wait_lock);
  82 }
  83 EXPORT_SYMBOL(queued_write_lock_slowpath);