arch/powerpc/include/asm/spinlock.h

   1 /* SPDX-License-Identifier: GPL-2.0-or-later */
   2 #ifndef __ASM_SPINLOCK_H
   3 #define __ASM_SPINLOCK_H
   4 #ifdef __KERNEL__
   5
   6 /*
   7  * Simple spin lock operations.
   8  *
   9  * Copyright (C) 2001-2004 Paul Mackerras <paulus@au.ibm.com>, IBM
  10  * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM
  11  * Copyright (C) 2002 Dave Engebretsen <engebret@us.ibm.com>, IBM
  12  *      Rework to support virtual processors
  13  *
  14  * Type of int is used as a full 64b word is not necessary.
  15  *
  16  * (the type definitions are in asm/spinlock_types.h)
  17  */
  18 #include <linux/jump_label.h>
  19 #include <linux/irqflags.h>
  20 #ifdef CONFIG_PPC64
  21 #include <asm/paca.h>
  22 #include <asm/hvcall.h>
  23 #endif
  24 #include <asm/synch.h>
  25 #include <asm/ppc-opcode.h>
  26
  27 #ifdef CONFIG_PPC64
  28 /* use 0x800000yy when locked, where yy == CPU number */
  29 #ifdef __BIG_ENDIAN__
  30 #define LOCK_TOKEN      (*(u32 *)(&get_paca()->lock_token))
  31 #else
  32 #define LOCK_TOKEN      (*(u32 *)(&get_paca()->paca_index))
  33 #endif
  34 #else
  35 #define LOCK_TOKEN      1
  36 #endif
  37
  38 #ifdef CONFIG_PPC_PSERIES
  39 DECLARE_STATIC_KEY_FALSE(shared_processor);
  40
  41 #define vcpu_is_preempted vcpu_is_preempted
  42 static inline bool vcpu_is_preempted(int cpu)
  43 {
  44         if (!static_branch_unlikely(&shared_processor))
  45                 return false;
  46         return !!(be32_to_cpu(lppaca_of(cpu).yield_count) & 1);
  47 }
  48 #endif
  49
  50 static __always_inline int arch_spin_value_unlocked(arch_spinlock_t lock)
  51 {
  52         return lock.slock == 0;
  53 }
  54
  55 static inline int arch_spin_is_locked(arch_spinlock_t *lock)
  56 {
  57         smp_mb();
  58         return !arch_spin_value_unlocked(*lock);
  59 }
  60
  61 /*
  62  * This returns the old value in the lock, so we succeeded
  63  * in getting the lock if the return value is 0.
  64  */
  65 static inline unsigned long __arch_spin_trylock(arch_spinlock_t *lock)
  66 {
  67         unsigned long tmp, token;
  68
  69         token = LOCK_TOKEN;
  70         __asm__ __volatile__(
  71 "1:     " PPC_LWARX(%0,0,%2,1) "\n\
  72         cmpwi           0,%0,0\n\
  73         bne-            2f\n\
  74         stwcx.          %1,0,%2\n\
  75         bne-            1b\n"
  76         PPC_ACQUIRE_BARRIER
  77 "2:"
  78         : "=&r" (tmp)
  79         : "r" (token), "r" (&lock->slock)
  80         : "cr0", "memory");
  81
  82         return tmp;
  83 }
  84
  85 static inline int arch_spin_trylock(arch_spinlock_t *lock)
  86 {
  87         return __arch_spin_trylock(lock) == 0;
  88 }
  89
  90 /*
  91  * On a system with shared processors (that is, where a physical
  92  * processor is multiplexed between several virtual processors),
  93  * there is no point spinning on a lock if the holder of the lock
  94  * isn't currently scheduled on a physical processor.  Instead
  95  * we detect this situation and ask the hypervisor to give the
  96  * rest of our timeslice to the lock holder.
  97  *
  98  * So that we can tell which virtual processor is holding a lock,
  99  * we put 0x80000000 | smp_processor_id() in the lock when it is
 100  * held.  Conveniently, we have a word in the paca that holds this
 101  * value.
 102  */
 103
 104 #if defined(CONFIG_PPC_SPLPAR)
 105 /* We only yield to the hypervisor if we are in shared processor mode */
 106 void splpar_spin_yield(arch_spinlock_t *lock);
 107 void splpar_rw_yield(arch_rwlock_t *lock);
 108 #else /* SPLPAR */
 109 static inline void splpar_spin_yield(arch_spinlock_t *lock) {};
 110 static inline void splpar_rw_yield(arch_rwlock_t *lock) {};
 111 #endif
 112
 113 static inline bool is_shared_processor(void)
 114 {
 115 #ifdef CONFIG_PPC_SPLPAR
 116         return static_branch_unlikely(&shared_processor);
 117 #else
 118         return false;
 119 #endif
 120 }
 121
 122 static inline void spin_yield(arch_spinlock_t *lock)
 123 {
 124         if (is_shared_processor())
 125                 splpar_spin_yield(lock);
 126         else
 127                 barrier();
 128 }
 129
 130 static inline void rw_yield(arch_rwlock_t *lock)
 131 {
 132         if (is_shared_processor())
 133                 splpar_rw_yield(lock);
 134         else
 135                 barrier();
 136 }
 137
 138 static inline void arch_spin_lock(arch_spinlock_t *lock)
 139 {
 140         while (1) {
 141                 if (likely(__arch_spin_trylock(lock) == 0))
 142                         break;
 143                 do {
 144                         HMT_low();
 145                         if (is_shared_processor())
 146                                 splpar_spin_yield(lock);
 147                 } while (unlikely(lock->slock != 0));
 148                 HMT_medium();
 149         }
 150 }
 151
 152 static inline
 153 void arch_spin_lock_flags(arch_spinlock_t *lock, unsigned long flags)
 154 {
 155         unsigned long flags_dis;
 156
 157         while (1) {
 158                 if (likely(__arch_spin_trylock(lock) == 0))
 159                         break;
 160                 local_save_flags(flags_dis);
 161                 local_irq_restore(flags);
 162                 do {
 163                         HMT_low();
 164                         if (is_shared_processor())
 165                                 splpar_spin_yield(lock);
 166                 } while (unlikely(lock->slock != 0));
 167                 HMT_medium();
 168                 local_irq_restore(flags_dis);
 169         }
 170 }
 171 #define arch_spin_lock_flags arch_spin_lock_flags
 172
 173 static inline void arch_spin_unlock(arch_spinlock_t *lock)
 174 {
 175         __asm__ __volatile__("# arch_spin_unlock\n\t"
 176                                 PPC_RELEASE_BARRIER: : :"memory");
 177         lock->slock = 0;
 178 }
 179
 180 /*
 181  * Read-write spinlocks, allowing multiple readers
 182  * but only one writer.
 183  *
 184  * NOTE! it is quite common to have readers in interrupts
 185  * but no interrupt writers. For those circumstances we
 186  * can "mix" irq-safe locks - any writer needs to get a
 187  * irq-safe write-lock, but readers can get non-irqsafe
 188  * read-locks.
 189  */
 190
 191 #ifdef CONFIG_PPC64
 192 #define __DO_SIGN_EXTEND        "extsw  %0,%0\n"
 193 #define WRLOCK_TOKEN            LOCK_TOKEN      /* it's negative */
 194 #else
 195 #define __DO_SIGN_EXTEND
 196 #define WRLOCK_TOKEN            (-1)
 197 #endif
 198
 199 /*
 200  * This returns the old value in the lock + 1,
 201  * so we got a read lock if the return value is > 0.
 202  */
 203 static inline long __arch_read_trylock(arch_rwlock_t *rw)
 204 {
 205         long tmp;
 206
 207         __asm__ __volatile__(
 208 "1:     " PPC_LWARX(%0,0,%1,1) "\n"
 209         __DO_SIGN_EXTEND
 210 "       addic.          %0,%0,1\n\
 211         ble-            2f\n"
 212 "       stwcx.          %0,0,%1\n\
 213         bne-            1b\n"
 214         PPC_ACQUIRE_BARRIER
 215 "2:"    : "=&r" (tmp)
 216         : "r" (&rw->lock)
 217         : "cr0", "xer", "memory");
 218
 219         return tmp;
 220 }
 221
 222 /*
 223  * This returns the old value in the lock,
 224  * so we got the write lock if the return value is 0.
 225  */
 226 static inline long __arch_write_trylock(arch_rwlock_t *rw)
 227 {
 228         long tmp, token;
 229
 230         token = WRLOCK_TOKEN;
 231         __asm__ __volatile__(
 232 "1:     " PPC_LWARX(%0,0,%2,1) "\n\
 233         cmpwi           0,%0,0\n\
 234         bne-            2f\n"
 235 "       stwcx.          %1,0,%2\n\
 236         bne-            1b\n"
 237         PPC_ACQUIRE_BARRIER
 238 "2:"    : "=&r" (tmp)
 239         : "r" (token), "r" (&rw->lock)
 240         : "cr0", "memory");
 241
 242         return tmp;
 243 }
 244
 245 static inline void arch_read_lock(arch_rwlock_t *rw)
 246 {
 247         while (1) {
 248                 if (likely(__arch_read_trylock(rw) > 0))
 249                         break;
 250                 do {
 251                         HMT_low();
 252                         if (is_shared_processor())
 253                                 splpar_rw_yield(rw);
 254                 } while (unlikely(rw->lock < 0));
 255                 HMT_medium();
 256         }
 257 }
 258
 259 static inline void arch_write_lock(arch_rwlock_t *rw)
 260 {
 261         while (1) {
 262                 if (likely(__arch_write_trylock(rw) == 0))
 263                         break;
 264                 do {
 265                         HMT_low();
 266                         if (is_shared_processor())
 267                                 splpar_rw_yield(rw);
 268                 } while (unlikely(rw->lock != 0));
 269                 HMT_medium();
 270         }
 271 }
 272
 273 static inline int arch_read_trylock(arch_rwlock_t *rw)
 274 {
 275         return __arch_read_trylock(rw) > 0;
 276 }
 277
 278 static inline int arch_write_trylock(arch_rwlock_t *rw)
 279 {
 280         return __arch_write_trylock(rw) == 0;
 281 }
 282
 283 static inline void arch_read_unlock(arch_rwlock_t *rw)
 284 {
 285         long tmp;
 286
 287         __asm__ __volatile__(
 288         "# read_unlock\n\t"
 289         PPC_RELEASE_BARRIER
 290 "1:     lwarx           %0,0,%1\n\
 291         addic           %0,%0,-1\n"
 292 "       stwcx.          %0,0,%1\n\
 293         bne-            1b"
 294         : "=&r"(tmp)
 295         : "r"(&rw->lock)
 296         : "cr0", "xer", "memory");
 297 }
 298
 299 static inline void arch_write_unlock(arch_rwlock_t *rw)
 300 {
 301         __asm__ __volatile__("# write_unlock\n\t"
 302                                 PPC_RELEASE_BARRIER: : :"memory");
 303         rw->lock = 0;
 304 }
 305
 306 #define arch_spin_relax(lock)   spin_yield(lock)
 307 #define arch_read_relax(lock)   rw_yield(lock)
 308 #define arch_write_relax(lock)  rw_yield(lock)
 309
 310 /* See include/linux/spinlock.h */
 311 #define smp_mb__after_spinlock()   smp_mb()
 312
 313 #endif /* __KERNEL__ */
 314 #endif /* __ASM_SPINLOCK_H */