fs/btrfs/locking.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2008 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/spinlock.h>
   9 #include <linux/page-flags.h>
  10 #include <asm/bug.h>
  11 #include "misc.h"
  12 #include "ctree.h"
  13 #include "extent_io.h"
  14 #include "locking.h"
  15
  16 #ifdef CONFIG_BTRFS_DEBUG
  17 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb)
  18 {
  19         WARN_ON(eb->spinning_writers);
  20         eb->spinning_writers++;
  21 }
  22
  23 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb)
  24 {
  25         WARN_ON(eb->spinning_writers != 1);
  26         eb->spinning_writers--;
  27 }
  28
  29 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb)
  30 {
  31         WARN_ON(eb->spinning_writers);
  32 }
  33
  34 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb)
  35 {
  36         atomic_inc(&eb->spinning_readers);
  37 }
  38
  39 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb)
  40 {
  41         WARN_ON(atomic_read(&eb->spinning_readers) == 0);
  42         atomic_dec(&eb->spinning_readers);
  43 }
  44
  45 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb)
  46 {
  47         atomic_inc(&eb->read_locks);
  48 }
  49
  50 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb)
  51 {
  52         atomic_dec(&eb->read_locks);
  53 }
  54
  55 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb)
  56 {
  57         BUG_ON(!atomic_read(&eb->read_locks));
  58 }
  59
  60 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb)
  61 {
  62         eb->write_locks++;
  63 }
  64
  65 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb)
  66 {
  67         eb->write_locks--;
  68 }
  69
  70 void btrfs_assert_tree_locked(struct extent_buffer *eb)
  71 {
  72         BUG_ON(!eb->write_locks);
  73 }
  74
  75 #else
  76 static void btrfs_assert_spinning_writers_get(struct extent_buffer *eb) { }
  77 static void btrfs_assert_spinning_writers_put(struct extent_buffer *eb) { }
  78 static void btrfs_assert_no_spinning_writers(struct extent_buffer *eb) { }
  79 static void btrfs_assert_spinning_readers_put(struct extent_buffer *eb) { }
  80 static void btrfs_assert_spinning_readers_get(struct extent_buffer *eb) { }
  81 static void btrfs_assert_tree_read_locked(struct extent_buffer *eb) { }
  82 static void btrfs_assert_tree_read_locks_get(struct extent_buffer *eb) { }
  83 static void btrfs_assert_tree_read_locks_put(struct extent_buffer *eb) { }
  84 void btrfs_assert_tree_locked(struct extent_buffer *eb) { }
  85 static void btrfs_assert_tree_write_locks_get(struct extent_buffer *eb) { }
  86 static void btrfs_assert_tree_write_locks_put(struct extent_buffer *eb) { }
  87 #endif
  88
  89 void btrfs_set_lock_blocking_read(struct extent_buffer *eb)
  90 {
  91         trace_btrfs_set_lock_blocking_read(eb);
  92         /*
  93          * No lock is required.  The lock owner may change if we have a read
  94          * lock, but it won't change to or away from us.  If we have the write
  95          * lock, we are the owner and it'll never change.
  96          */
  97         if (eb->lock_nested && current->pid == eb->lock_owner)
  98                 return;
  99         btrfs_assert_tree_read_locked(eb);
 100         atomic_inc(&eb->blocking_readers);
 101         btrfs_assert_spinning_readers_put(eb);
 102         read_unlock(&eb->lock);
 103 }
 104
 105 void btrfs_set_lock_blocking_write(struct extent_buffer *eb)
 106 {
 107         trace_btrfs_set_lock_blocking_write(eb);
 108         /*
 109          * No lock is required.  The lock owner may change if we have a read
 110          * lock, but it won't change to or away from us.  If we have the write
 111          * lock, we are the owner and it'll never change.
 112          */
 113         if (eb->lock_nested && current->pid == eb->lock_owner)
 114                 return;
 115         if (eb->blocking_writers == 0) {
 116                 btrfs_assert_spinning_writers_put(eb);
 117                 btrfs_assert_tree_locked(eb);
 118                 eb->blocking_writers++;
 119                 write_unlock(&eb->lock);
 120         }
 121 }
 122
 123 /*
 124  * take a spinning read lock.  This will wait for any blocking
 125  * writers
 126  */
 127 void btrfs_tree_read_lock(struct extent_buffer *eb)
 128 {
 129         u64 start_ns = 0;
 130
 131         if (trace_btrfs_tree_read_lock_enabled())
 132                 start_ns = ktime_get_ns();
 133 again:
 134         read_lock(&eb->lock);
 135         BUG_ON(eb->blocking_writers == 0 &&
 136                current->pid == eb->lock_owner);
 137         if (eb->blocking_writers && current->pid == eb->lock_owner) {
 138                 /*
 139                  * This extent is already write-locked by our thread. We allow
 140                  * an additional read lock to be added because it's for the same
 141                  * thread. btrfs_find_all_roots() depends on this as it may be
 142                  * called on a partly (write-)locked tree.
 143                  */
 144                 BUG_ON(eb->lock_nested);
 145                 eb->lock_nested = true;
 146                 read_unlock(&eb->lock);
 147                 trace_btrfs_tree_read_lock(eb, start_ns);
 148                 return;
 149         }
 150         if (eb->blocking_writers) {
 151                 read_unlock(&eb->lock);
 152                 wait_event(eb->write_lock_wq,
 153                            eb->blocking_writers == 0);
 154                 goto again;
 155         }
 156         btrfs_assert_tree_read_locks_get(eb);
 157         btrfs_assert_spinning_readers_get(eb);
 158         trace_btrfs_tree_read_lock(eb, start_ns);
 159 }
 160
 161 /*
 162  * take a spinning read lock.
 163  * returns 1 if we get the read lock and 0 if we don't
 164  * this won't wait for blocking writers
 165  */
 166 int btrfs_tree_read_lock_atomic(struct extent_buffer *eb)
 167 {
 168         if (eb->blocking_writers)
 169                 return 0;
 170
 171         read_lock(&eb->lock);
 172         if (eb->blocking_writers) {
 173                 read_unlock(&eb->lock);
 174                 return 0;
 175         }
 176         btrfs_assert_tree_read_locks_get(eb);
 177         btrfs_assert_spinning_readers_get(eb);
 178         trace_btrfs_tree_read_lock_atomic(eb);
 179         return 1;
 180 }
 181
 182 /*
 183  * returns 1 if we get the read lock and 0 if we don't
 184  * this won't wait for blocking writers
 185  */
 186 int btrfs_try_tree_read_lock(struct extent_buffer *eb)
 187 {
 188         if (eb->blocking_writers)
 189                 return 0;
 190
 191         if (!read_trylock(&eb->lock))
 192                 return 0;
 193
 194         if (eb->blocking_writers) {
 195                 read_unlock(&eb->lock);
 196                 return 0;
 197         }
 198         btrfs_assert_tree_read_locks_get(eb);
 199         btrfs_assert_spinning_readers_get(eb);
 200         trace_btrfs_try_tree_read_lock(eb);
 201         return 1;
 202 }
 203
 204 /*
 205  * returns 1 if we get the read lock and 0 if we don't
 206  * this won't wait for blocking writers or readers
 207  */
 208 int btrfs_try_tree_write_lock(struct extent_buffer *eb)
 209 {
 210         if (eb->blocking_writers || atomic_read(&eb->blocking_readers))
 211                 return 0;
 212
 213         write_lock(&eb->lock);
 214         if (eb->blocking_writers || atomic_read(&eb->blocking_readers)) {
 215                 write_unlock(&eb->lock);
 216                 return 0;
 217         }
 218         btrfs_assert_tree_write_locks_get(eb);
 219         btrfs_assert_spinning_writers_get(eb);
 220         eb->lock_owner = current->pid;
 221         trace_btrfs_try_tree_write_lock(eb);
 222         return 1;
 223 }
 224
 225 /*
 226  * drop a spinning read lock
 227  */
 228 void btrfs_tree_read_unlock(struct extent_buffer *eb)
 229 {
 230         trace_btrfs_tree_read_unlock(eb);
 231         /*
 232          * if we're nested, we have the write lock.  No new locking
 233          * is needed as long as we are the lock owner.
 234          * The write unlock will do a barrier for us, and the lock_nested
 235          * field only matters to the lock owner.
 236          */
 237         if (eb->lock_nested && current->pid == eb->lock_owner) {
 238                 eb->lock_nested = false;
 239                 return;
 240         }
 241         btrfs_assert_tree_read_locked(eb);
 242         btrfs_assert_spinning_readers_put(eb);
 243         btrfs_assert_tree_read_locks_put(eb);
 244         read_unlock(&eb->lock);
 245 }
 246
 247 /*
 248  * drop a blocking read lock
 249  */
 250 void btrfs_tree_read_unlock_blocking(struct extent_buffer *eb)
 251 {
 252         trace_btrfs_tree_read_unlock_blocking(eb);
 253         /*
 254          * if we're nested, we have the write lock.  No new locking
 255          * is needed as long as we are the lock owner.
 256          * The write unlock will do a barrier for us, and the lock_nested
 257          * field only matters to the lock owner.
 258          */
 259         if (eb->lock_nested && current->pid == eb->lock_owner) {
 260                 eb->lock_nested = false;
 261                 return;
 262         }
 263         btrfs_assert_tree_read_locked(eb);
 264         WARN_ON(atomic_read(&eb->blocking_readers) == 0);
 265         /* atomic_dec_and_test implies a barrier */
 266         if (atomic_dec_and_test(&eb->blocking_readers))
 267                 cond_wake_up_nomb(&eb->read_lock_wq);
 268         btrfs_assert_tree_read_locks_put(eb);
 269 }
 270
 271 /*
 272  * take a spinning write lock.  This will wait for both
 273  * blocking readers or writers
 274  */
 275 void btrfs_tree_lock(struct extent_buffer *eb)
 276 {
 277         u64 start_ns = 0;
 278
 279         if (trace_btrfs_tree_lock_enabled())
 280                 start_ns = ktime_get_ns();
 281
 282         WARN_ON(eb->lock_owner == current->pid);
 283 again:
 284         wait_event(eb->read_lock_wq, atomic_read(&eb->blocking_readers) == 0);
 285         wait_event(eb->write_lock_wq, eb->blocking_writers == 0);
 286         write_lock(&eb->lock);
 287         if (atomic_read(&eb->blocking_readers) || eb->blocking_writers) {
 288                 write_unlock(&eb->lock);
 289                 goto again;
 290         }
 291         btrfs_assert_spinning_writers_get(eb);
 292         btrfs_assert_tree_write_locks_get(eb);
 293         eb->lock_owner = current->pid;
 294         trace_btrfs_tree_lock(eb, start_ns);
 295 }
 296
 297 /*
 298  * drop a spinning or a blocking write lock.
 299  */
 300 void btrfs_tree_unlock(struct extent_buffer *eb)
 301 {
 302         int blockers = eb->blocking_writers;
 303
 304         BUG_ON(blockers > 1);
 305
 306         btrfs_assert_tree_locked(eb);
 307         trace_btrfs_tree_unlock(eb);
 308         eb->lock_owner = 0;
 309         btrfs_assert_tree_write_locks_put(eb);
 310
 311         if (blockers) {
 312                 btrfs_assert_no_spinning_writers(eb);
 313                 eb->blocking_writers--;
 314                 /*
 315                  * We need to order modifying blocking_writers above with
 316                  * actually waking up the sleepers to ensure they see the
 317                  * updated value of blocking_writers
 318                  */
 319                 cond_wake_up(&eb->write_lock_wq);
 320         } else {
 321                 btrfs_assert_spinning_writers_put(eb);
 322                 write_unlock(&eb->lock);
 323         }
 324 }