fs/btrfs/subpage.c

   1 // SPDX-License-Identifier: GPL-2.0
   2
   3 #include <linux/slab.h>
   4 #include "ctree.h"
   5 #include "subpage.h"
   6 #include "btrfs_inode.h"
   7
   8 /*
   9  * Subpage (sectorsize < PAGE_SIZE) support overview:
  10  *
  11  * Limitations:
  12  *
  13  * - Only support 64K page size for now
  14  *   This is to make metadata handling easier, as 64K page would ensure
  15  *   all nodesize would fit inside one page, thus we don't need to handle
  16  *   cases where a tree block crosses several pages.
  17  *
  18  * - Only metadata read-write for now
  19  *   The data read-write part is in development.
  20  *
  21  * - Metadata can't cross 64K page boundary
  22  *   btrfs-progs and kernel have done that for a while, thus only ancient
  23  *   filesystems could have such problem.  For such case, do a graceful
  24  *   rejection.
  25  *
  26  * Special behavior:
  27  *
  28  * - Metadata
  29  *   Metadata read is fully supported.
  30  *   Meaning when reading one tree block will only trigger the read for the
  31  *   needed range, other unrelated range in the same page will not be touched.
  32  *
  33  *   Metadata write support is partial.
  34  *   The writeback is still for the full page, but we will only submit
  35  *   the dirty extent buffers in the page.
  36  *
  37  *   This means, if we have a metadata page like this:
  38  *
  39  *   Page offset
  40  *   0         16K         32K         48K        64K
  41  *   |/////////|           |///////////|
  42  *        \- Tree block A        \- Tree block B
  43  *
  44  *   Even if we just want to writeback tree block A, we will also writeback
  45  *   tree block B if it's also dirty.
  46  *
  47  *   This may cause extra metadata writeback which results more COW.
  48  *
  49  * Implementation:
  50  *
  51  * - Common
  52  *   Both metadata and data will use a new structure, btrfs_subpage, to
  53  *   record the status of each sector inside a page.  This provides the extra
  54  *   granularity needed.
  55  *
  56  * - Metadata
  57  *   Since we have multiple tree blocks inside one page, we can't rely on page
  58  *   locking anymore, or we will have greatly reduced concurrency or even
  59  *   deadlocks (hold one tree lock while trying to lock another tree lock in
  60  *   the same page).
  61  *
  62  *   Thus for metadata locking, subpage support relies on io_tree locking only.
  63  *   This means a slightly higher tree locking latency.
  64  */
  65
  66 void btrfs_init_subpage_info(struct btrfs_subpage_info *subpage_info, u32 sectorsize)
  67 {
  68         unsigned int cur = 0;
  69         unsigned int nr_bits;
  70
  71         ASSERT(IS_ALIGNED(PAGE_SIZE, sectorsize));
  72
  73         nr_bits = PAGE_SIZE / sectorsize;
  74         subpage_info->bitmap_nr_bits = nr_bits;
  75
  76         subpage_info->uptodate_offset = cur;
  77         cur += nr_bits;
  78
  79         subpage_info->error_offset = cur;
  80         cur += nr_bits;
  81
  82         subpage_info->dirty_offset = cur;
  83         cur += nr_bits;
  84
  85         subpage_info->writeback_offset = cur;
  86         cur += nr_bits;
  87
  88         subpage_info->ordered_offset = cur;
  89         cur += nr_bits;
  90
  91         subpage_info->checked_offset = cur;
  92         cur += nr_bits;
  93
  94         subpage_info->total_nr_bits = cur;
  95 }
  96
  97 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
  98                          struct page *page, enum btrfs_subpage_type type)
  99 {
 100         struct btrfs_subpage *subpage;
 101
 102         /*
 103          * We have cases like a dummy extent buffer page, which is not mappped
 104          * and doesn't need to be locked.
 105          */
 106         if (page->mapping)
 107                 ASSERT(PageLocked(page));
 108
 109         /* Either not subpage, or the page already has private attached */
 110         if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page))
 111                 return 0;
 112
 113         subpage = btrfs_alloc_subpage(fs_info, type);
 114         if (IS_ERR(subpage))
 115                 return  PTR_ERR(subpage);
 116
 117         attach_page_private(page, subpage);
 118         return 0;
 119 }
 120
 121 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info,
 122                           struct page *page)
 123 {
 124         struct btrfs_subpage *subpage;
 125
 126         /* Either not subpage, or already detached */
 127         if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page))
 128                 return;
 129
 130         subpage = (struct btrfs_subpage *)detach_page_private(page);
 131         ASSERT(subpage);
 132         btrfs_free_subpage(subpage);
 133 }
 134
 135 struct btrfs_subpage *btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
 136                                           enum btrfs_subpage_type type)
 137 {
 138         struct btrfs_subpage *ret;
 139         unsigned int real_size;
 140
 141         ASSERT(fs_info->sectorsize < PAGE_SIZE);
 142
 143         real_size = struct_size(ret, bitmaps,
 144                         BITS_TO_LONGS(fs_info->subpage_info->total_nr_bits));
 145         ret = kzalloc(real_size, GFP_NOFS);
 146         if (!ret)
 147                 return ERR_PTR(-ENOMEM);
 148
 149         spin_lock_init(&ret->lock);
 150         if (type == BTRFS_SUBPAGE_METADATA) {
 151                 atomic_set(&ret->eb_refs, 0);
 152         } else {
 153                 atomic_set(&ret->readers, 0);
 154                 atomic_set(&ret->writers, 0);
 155         }
 156         return ret;
 157 }
 158
 159 void btrfs_free_subpage(struct btrfs_subpage *subpage)
 160 {
 161         kfree(subpage);
 162 }
 163
 164 /*
 165  * Increase the eb_refs of current subpage.
 166  *
 167  * This is important for eb allocation, to prevent race with last eb freeing
 168  * of the same page.
 169  * With the eb_refs increased before the eb inserted into radix tree,
 170  * detach_extent_buffer_page() won't detach the page private while we're still
 171  * allocating the extent buffer.
 172  */
 173 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
 174                             struct page *page)
 175 {
 176         struct btrfs_subpage *subpage;
 177
 178         if (fs_info->sectorsize == PAGE_SIZE)
 179                 return;
 180
 181         ASSERT(PagePrivate(page) && page->mapping);
 182         lockdep_assert_held(&page->mapping->private_lock);
 183
 184         subpage = (struct btrfs_subpage *)page->private;
 185         atomic_inc(&subpage->eb_refs);
 186 }
 187
 188 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
 189                             struct page *page)
 190 {
 191         struct btrfs_subpage *subpage;
 192
 193         if (fs_info->sectorsize == PAGE_SIZE)
 194                 return;
 195
 196         ASSERT(PagePrivate(page) && page->mapping);
 197         lockdep_assert_held(&page->mapping->private_lock);
 198
 199         subpage = (struct btrfs_subpage *)page->private;
 200         ASSERT(atomic_read(&subpage->eb_refs));
 201         atomic_dec(&subpage->eb_refs);
 202 }
 203
 204 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
 205                 struct page *page, u64 start, u32 len)
 206 {
 207         /* Basic checks */
 208         ASSERT(PagePrivate(page) && page->private);
 209         ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
 210                IS_ALIGNED(len, fs_info->sectorsize));
 211         /*
 212          * The range check only works for mapped page, we can still have
 213          * unmapped page like dummy extent buffer pages.
 214          */
 215         if (page->mapping)
 216                 ASSERT(page_offset(page) <= start &&
 217                        start + len <= page_offset(page) + PAGE_SIZE);
 218 }
 219
 220 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
 221                 struct page *page, u64 start, u32 len)
 222 {
 223         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 224         const int nbits = len >> fs_info->sectorsize_bits;
 225
 226         btrfs_subpage_assert(fs_info, page, start, len);
 227
 228         atomic_add(nbits, &subpage->readers);
 229 }
 230
 231 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
 232                 struct page *page, u64 start, u32 len)
 233 {
 234         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 235         const int nbits = len >> fs_info->sectorsize_bits;
 236         bool is_data;
 237         bool last;
 238
 239         btrfs_subpage_assert(fs_info, page, start, len);
 240         is_data = is_data_inode(page->mapping->host);
 241         ASSERT(atomic_read(&subpage->readers) >= nbits);
 242         last = atomic_sub_and_test(nbits, &subpage->readers);
 243
 244         /*
 245          * For data we need to unlock the page if the last read has finished.
 246          *
 247          * And please don't replace @last with atomic_sub_and_test() call
 248          * inside if () condition.
 249          * As we want the atomic_sub_and_test() to be always executed.
 250          */
 251         if (is_data && last)
 252                 unlock_page(page);
 253 }
 254
 255 static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len)
 256 {
 257         u64 orig_start = *start;
 258         u32 orig_len = *len;
 259
 260         *start = max_t(u64, page_offset(page), orig_start);
 261         /*
 262          * For certain call sites like btrfs_drop_pages(), we may have pages
 263          * beyond the target range. In that case, just set @len to 0, subpage
 264          * helpers can handle @len == 0 without any problem.
 265          */
 266         if (page_offset(page) >= orig_start + orig_len)
 267                 *len = 0;
 268         else
 269                 *len = min_t(u64, page_offset(page) + PAGE_SIZE,
 270                              orig_start + orig_len) - *start;
 271 }
 272
 273 void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,
 274                 struct page *page, u64 start, u32 len)
 275 {
 276         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 277         const int nbits = (len >> fs_info->sectorsize_bits);
 278         int ret;
 279
 280         btrfs_subpage_assert(fs_info, page, start, len);
 281
 282         ASSERT(atomic_read(&subpage->readers) == 0);
 283         ret = atomic_add_return(nbits, &subpage->writers);
 284         ASSERT(ret == nbits);
 285 }
 286
 287 bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,
 288                 struct page *page, u64 start, u32 len)
 289 {
 290         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 291         const int nbits = (len >> fs_info->sectorsize_bits);
 292
 293         btrfs_subpage_assert(fs_info, page, start, len);
 294
 295         /*
 296          * We have call sites passing @lock_page into
 297          * extent_clear_unlock_delalloc() for compression path.
 298          *
 299          * This @locked_page is locked by plain lock_page(), thus its
 300          * subpage::writers is 0.  Handle them in a special way.
 301          */
 302         if (atomic_read(&subpage->writers) == 0)
 303                 return true;
 304
 305         ASSERT(atomic_read(&subpage->writers) >= nbits);
 306         return atomic_sub_and_test(nbits, &subpage->writers);
 307 }
 308
 309 /*
 310  * Lock a page for delalloc page writeback.
 311  *
 312  * Return -EAGAIN if the page is not properly initialized.
 313  * Return 0 with the page locked, and writer counter updated.
 314  *
 315  * Even with 0 returned, the page still need extra check to make sure
 316  * it's really the correct page, as the caller is using
 317  * find_get_pages_contig(), which can race with page invalidating.
 318  */
 319 int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info,
 320                 struct page *page, u64 start, u32 len)
 321 {
 322         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {
 323                 lock_page(page);
 324                 return 0;
 325         }
 326         lock_page(page);
 327         if (!PagePrivate(page) || !page->private) {
 328                 unlock_page(page);
 329                 return -EAGAIN;
 330         }
 331         btrfs_subpage_clamp_range(page, &start, &len);
 332         btrfs_subpage_start_writer(fs_info, page, start, len);
 333         return 0;
 334 }
 335
 336 void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info,
 337                 struct page *page, u64 start, u32 len)
 338 {
 339         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)
 340                 return unlock_page(page);
 341         btrfs_subpage_clamp_range(page, &start, &len);
 342         if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len))
 343                 unlock_page(page);
 344 }
 345
 346 static bool bitmap_test_range_all_set(unsigned long *addr, unsigned int start,
 347                                       unsigned int nbits)
 348 {
 349         unsigned int found_zero;
 350
 351         found_zero = find_next_zero_bit(addr, start + nbits, start);
 352         if (found_zero == start + nbits)
 353                 return true;
 354         return false;
 355 }
 356
 357 static bool bitmap_test_range_all_zero(unsigned long *addr, unsigned int start,
 358                                        unsigned int nbits)
 359 {
 360         unsigned int found_set;
 361
 362         found_set = find_next_bit(addr, start + nbits, start);
 363         if (found_set == start + nbits)
 364                 return true;
 365         return false;
 366 }
 367
 368 #define subpage_calc_start_bit(fs_info, page, name, start, len)         \
 369 ({                                                                      \
 370         unsigned int start_bit;                                         \
 371                                                                         \
 372         btrfs_subpage_assert(fs_info, page, start, len);                \
 373         start_bit = offset_in_page(start) >> fs_info->sectorsize_bits;  \
 374         start_bit += fs_info->subpage_info->name##_offset;              \
 375         start_bit;                                                      \
 376 })
 377
 378 #define subpage_test_bitmap_all_set(fs_info, subpage, name)             \
 379         bitmap_test_range_all_set(subpage->bitmaps,                     \
 380                         fs_info->subpage_info->name##_offset,           \
 381                         fs_info->subpage_info->bitmap_nr_bits)
 382
 383 #define subpage_test_bitmap_all_zero(fs_info, subpage, name)            \
 384         bitmap_test_range_all_zero(subpage->bitmaps,                    \
 385                         fs_info->subpage_info->name##_offset,           \
 386                         fs_info->subpage_info->bitmap_nr_bits)
 387
 388 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
 389                 struct page *page, u64 start, u32 len)
 390 {
 391         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 392         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 393                                                         uptodate, start, len);
 394         unsigned long flags;
 395
 396         spin_lock_irqsave(&subpage->lock, flags);
 397         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 398         if (subpage_test_bitmap_all_set(fs_info, subpage, uptodate))
 399                 SetPageUptodate(page);
 400         spin_unlock_irqrestore(&subpage->lock, flags);
 401 }
 402
 403 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
 404                 struct page *page, u64 start, u32 len)
 405 {
 406         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 407         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 408                                                         uptodate, start, len);
 409         unsigned long flags;
 410
 411         spin_lock_irqsave(&subpage->lock, flags);
 412         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 413         ClearPageUptodate(page);
 414         spin_unlock_irqrestore(&subpage->lock, flags);
 415 }
 416
 417 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
 418                 struct page *page, u64 start, u32 len)
 419 {
 420         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 421         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 422                                                         error, start, len);
 423         unsigned long flags;
 424
 425         spin_lock_irqsave(&subpage->lock, flags);
 426         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 427         SetPageError(page);
 428         spin_unlock_irqrestore(&subpage->lock, flags);
 429 }
 430
 431 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
 432                 struct page *page, u64 start, u32 len)
 433 {
 434         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 435         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 436                                                         error, start, len);
 437         unsigned long flags;
 438
 439         spin_lock_irqsave(&subpage->lock, flags);
 440         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 441         if (subpage_test_bitmap_all_zero(fs_info, subpage, error))
 442                 ClearPageError(page);
 443         spin_unlock_irqrestore(&subpage->lock, flags);
 444 }
 445
 446 void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
 447                 struct page *page, u64 start, u32 len)
 448 {
 449         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 450         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 451                                                         dirty, start, len);
 452         unsigned long flags;
 453
 454         spin_lock_irqsave(&subpage->lock, flags);
 455         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 456         spin_unlock_irqrestore(&subpage->lock, flags);
 457         set_page_dirty(page);
 458 }
 459
 460 /*
 461  * Extra clear_and_test function for subpage dirty bitmap.
 462  *
 463  * Return true if we're the last bits in the dirty_bitmap and clear the
 464  * dirty_bitmap.
 465  * Return false otherwise.
 466  *
 467  * NOTE: Callers should manually clear page dirty for true case, as we have
 468  * extra handling for tree blocks.
 469  */
 470 bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
 471                 struct page *page, u64 start, u32 len)
 472 {
 473         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 474         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 475                                                         dirty, start, len);
 476         unsigned long flags;
 477         bool last = false;
 478
 479         spin_lock_irqsave(&subpage->lock, flags);
 480         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 481         if (subpage_test_bitmap_all_zero(fs_info, subpage, dirty))
 482                 last = true;
 483         spin_unlock_irqrestore(&subpage->lock, flags);
 484         return last;
 485 }
 486
 487 void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,
 488                 struct page *page, u64 start, u32 len)
 489 {
 490         bool last;
 491
 492         last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len);
 493         if (last)
 494                 clear_page_dirty_for_io(page);
 495 }
 496
 497 void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
 498                 struct page *page, u64 start, u32 len)
 499 {
 500         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 501         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 502                                                         writeback, start, len);
 503         unsigned long flags;
 504
 505         spin_lock_irqsave(&subpage->lock, flags);
 506         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 507         set_page_writeback(page);
 508         spin_unlock_irqrestore(&subpage->lock, flags);
 509 }
 510
 511 void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
 512                 struct page *page, u64 start, u32 len)
 513 {
 514         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 515         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 516                                                         writeback, start, len);
 517         unsigned long flags;
 518
 519         spin_lock_irqsave(&subpage->lock, flags);
 520         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 521         if (subpage_test_bitmap_all_zero(fs_info, subpage, writeback)) {
 522                 ASSERT(PageWriteback(page));
 523                 end_page_writeback(page);
 524         }
 525         spin_unlock_irqrestore(&subpage->lock, flags);
 526 }
 527
 528 void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
 529                 struct page *page, u64 start, u32 len)
 530 {
 531         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 532         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 533                                                         ordered, start, len);
 534         unsigned long flags;
 535
 536         spin_lock_irqsave(&subpage->lock, flags);
 537         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 538         SetPageOrdered(page);
 539         spin_unlock_irqrestore(&subpage->lock, flags);
 540 }
 541
 542 void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
 543                 struct page *page, u64 start, u32 len)
 544 {
 545         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 546         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 547                                                         ordered, start, len);
 548         unsigned long flags;
 549
 550         spin_lock_irqsave(&subpage->lock, flags);
 551         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 552         if (subpage_test_bitmap_all_zero(fs_info, subpage, ordered))
 553                 ClearPageOrdered(page);
 554         spin_unlock_irqrestore(&subpage->lock, flags);
 555 }
 556
 557 void btrfs_subpage_set_checked(const struct btrfs_fs_info *fs_info,
 558                                struct page *page, u64 start, u32 len)
 559 {
 560         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 561         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 562                                                         checked, start, len);
 563         unsigned long flags;
 564
 565         spin_lock_irqsave(&subpage->lock, flags);
 566         bitmap_set(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 567         if (subpage_test_bitmap_all_set(fs_info, subpage, checked))
 568                 SetPageChecked(page);
 569         spin_unlock_irqrestore(&subpage->lock, flags);
 570 }
 571
 572 void btrfs_subpage_clear_checked(const struct btrfs_fs_info *fs_info,
 573                                  struct page *page, u64 start, u32 len)
 574 {
 575         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 576         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,
 577                                                         checked, start, len);
 578         unsigned long flags;
 579
 580         spin_lock_irqsave(&subpage->lock, flags);
 581         bitmap_clear(subpage->bitmaps, start_bit, len >> fs_info->sectorsize_bits);
 582         ClearPageChecked(page);
 583         spin_unlock_irqrestore(&subpage->lock, flags);
 584 }
 585
 586 /*
 587  * Unlike set/clear which is dependent on each page status, for test all bits
 588  * are tested in the same way.
 589  */
 590 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name)                           \
 591 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info,     \
 592                 struct page *page, u64 start, u32 len)                  \
 593 {                                                                       \
 594         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
 595         unsigned int start_bit = subpage_calc_start_bit(fs_info, page,  \
 596                                                 name, start, len);      \
 597         unsigned long flags;                                            \
 598         bool ret;                                                       \
 599                                                                         \
 600         spin_lock_irqsave(&subpage->lock, flags);                       \
 601         ret = bitmap_test_range_all_set(subpage->bitmaps, start_bit,    \
 602                                 len >> fs_info->sectorsize_bits);       \
 603         spin_unlock_irqrestore(&subpage->lock, flags);                  \
 604         return ret;                                                     \
 605 }
 606 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
 607 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error);
 608 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
 609 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
 610 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);
 611 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(checked);
 612
 613 /*
 614  * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
 615  * in.  We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
 616  * back to regular sectorsize branch.
 617  */
 618 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func,  \
 619                                test_page_func)                          \
 620 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info,         \
 621                 struct page *page, u64 start, u32 len)                  \
 622 {                                                                       \
 623         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {   \
 624                 set_page_func(page);                                    \
 625                 return;                                                 \
 626         }                                                               \
 627         btrfs_subpage_set_##name(fs_info, page, start, len);            \
 628 }                                                                       \
 629 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info,       \
 630                 struct page *page, u64 start, u32 len)                  \
 631 {                                                                       \
 632         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {   \
 633                 clear_page_func(page);                                  \
 634                 return;                                                 \
 635         }                                                               \
 636         btrfs_subpage_clear_##name(fs_info, page, start, len);          \
 637 }                                                                       \
 638 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info,        \
 639                 struct page *page, u64 start, u32 len)                  \
 640 {                                                                       \
 641         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)     \
 642                 return test_page_func(page);                            \
 643         return btrfs_subpage_test_##name(fs_info, page, start, len);    \
 644 }                                                                       \
 645 void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info,   \
 646                 struct page *page, u64 start, u32 len)                  \
 647 {                                                                       \
 648         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {   \
 649                 set_page_func(page);                                    \
 650                 return;                                                 \
 651         }                                                               \
 652         btrfs_subpage_clamp_range(page, &start, &len);                  \
 653         btrfs_subpage_set_##name(fs_info, page, start, len);            \
 654 }                                                                       \
 655 void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \
 656                 struct page *page, u64 start, u32 len)                  \
 657 {                                                                       \
 658         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {   \
 659                 clear_page_func(page);                                  \
 660                 return;                                                 \
 661         }                                                               \
 662         btrfs_subpage_clamp_range(page, &start, &len);                  \
 663         btrfs_subpage_clear_##name(fs_info, page, start, len);          \
 664 }                                                                       \
 665 bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info,  \
 666                 struct page *page, u64 start, u32 len)                  \
 667 {                                                                       \
 668         if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)     \
 669                 return test_page_func(page);                            \
 670         btrfs_subpage_clamp_range(page, &start, &len);                  \
 671         return btrfs_subpage_test_##name(fs_info, page, start, len);    \
 672 }
 673 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate,
 674                          PageUptodate);
 675 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError);
 676 IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io,
 677                          PageDirty);
 678 IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback,
 679                          PageWriteback);
 680 IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered,
 681                          PageOrdered);
 682 IMPLEMENT_BTRFS_PAGE_OPS(checked, SetPageChecked, ClearPageChecked, PageChecked);
 683
 684 /*
 685  * Make sure not only the page dirty bit is cleared, but also subpage dirty bit
 686  * is cleared.
 687  */
 688 void btrfs_page_assert_not_dirty(const struct btrfs_fs_info *fs_info,
 689                                  struct page *page)
 690 {
 691         struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
 692
 693         if (!IS_ENABLED(CONFIG_BTRFS_ASSERT))
 694                 return;
 695
 696         ASSERT(!PageDirty(page));
 697         if (fs_info->sectorsize == PAGE_SIZE)
 698                 return;
 699
 700         ASSERT(PagePrivate(page) && page->private);
 701         ASSERT(subpage_test_bitmap_all_zero(fs_info, subpage, dirty));
 702 }
 703
 704 /*
 705  * Handle different locked pages with different page sizes:
 706  *
 707  * - Page locked by plain lock_page()
 708  *   It should not have any subpage::writers count.
 709  *   Can be unlocked by unlock_page().
 710  *   This is the most common locked page for __extent_writepage() called
 711  *   inside extent_write_cache_pages() or extent_write_full_page().
 712  *   Rarer cases include the @locked_page from extent_write_locked_range().
 713  *
 714  * - Page locked by lock_delalloc_pages()
 715  *   There is only one caller, all pages except @locked_page for
 716  *   extent_write_locked_range().
 717  *   In this case, we have to call subpage helper to handle the case.
 718  */
 719 void btrfs_page_unlock_writer(struct btrfs_fs_info *fs_info, struct page *page,
 720                               u64 start, u32 len)
 721 {
 722         struct btrfs_subpage *subpage;
 723
 724         ASSERT(PageLocked(page));
 725         /* For regular page size case, we just unlock the page */
 726         if (fs_info->sectorsize == PAGE_SIZE)
 727                 return unlock_page(page);
 728
 729         ASSERT(PagePrivate(page) && page->private);
 730         subpage = (struct btrfs_subpage *)page->private;
 731
 732         /*
 733          * For subpage case, there are two types of locked page.  With or
 734          * without writers number.
 735          *
 736          * Since we own the page lock, no one else could touch subpage::writers
 737          * and we are safe to do several atomic operations without spinlock.
 738          */
 739         if (atomic_read(&subpage->writers) == 0)
 740                 /* No writers, locked by plain lock_page() */
 741                 return unlock_page(page);
 742
 743         /* Have writers, use proper subpage helper to end it */
 744         btrfs_page_end_writer_lock(fs_info, page, start, len);
 745 }