fs/btrfs/scrub.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2011, 2012 STRATO.  All rights reserved.
   4  */
   5
   6 #include <linux/blkdev.h>
   7 #include <linux/ratelimit.h>
   8 #include <linux/sched/mm.h>
   9 #include <crypto/hash.h>
  10 #include "ctree.h"
  11 #include "discard.h"
  12 #include "volumes.h"
  13 #include "disk-io.h"
  14 #include "ordered-data.h"
  15 #include "transaction.h"
  16 #include "backref.h"
  17 #include "extent_io.h"
  18 #include "dev-replace.h"
  19 #include "check-integrity.h"
  20 #include "rcu-string.h"
  21 #include "raid56.h"
  22 #include "block-group.h"
  23 #include "zoned.h"
  24
  25 /*
  26  * This is only the first step towards a full-features scrub. It reads all
  27  * extent and super block and verifies the checksums. In case a bad checksum
  28  * is found or the extent cannot be read, good data will be written back if
  29  * any can be found.
  30  *
  31  * Future enhancements:
  32  *  - In case an unrepairable extent is encountered, track which files are
  33  *    affected and report them
  34  *  - track and record media errors, throw out bad devices
  35  *  - add a mode to also read unallocated space
  36  */
  37
  38 struct scrub_block;
  39 struct scrub_ctx;
  40
  41 /*
  42  * the following three values only influence the performance.
  43  * The last one configures the number of parallel and outstanding I/O
  44  * operations. The first two values configure an upper limit for the number
  45  * of (dynamically allocated) pages that are added to a bio.
  46  */
  47 #define SCRUB_PAGES_PER_RD_BIO  32      /* 128k per bio */
  48 #define SCRUB_PAGES_PER_WR_BIO  32      /* 128k per bio */
  49 #define SCRUB_BIOS_PER_SCTX     64      /* 8MB per device in flight */
  50
  51 /*
  52  * the following value times PAGE_SIZE needs to be large enough to match the
  53  * largest node/leaf/sector size that shall be supported.
  54  * Values larger than BTRFS_STRIPE_LEN are not supported.
  55  */
  56 #define SCRUB_MAX_PAGES_PER_BLOCK       16      /* 64k per node/leaf/sector */
  57
  58 struct scrub_recover {
  59         refcount_t              refs;
  60         struct btrfs_bio        *bbio;
  61         u64                     map_length;
  62 };
  63
  64 struct scrub_page {
  65         struct scrub_block      *sblock;
  66         struct page             *page;
  67         struct btrfs_device     *dev;
  68         struct list_head        list;
  69         u64                     flags;  /* extent flags */
  70         u64                     generation;
  71         u64                     logical;
  72         u64                     physical;
  73         u64                     physical_for_dev_replace;
  74         atomic_t                refs;
  75         u8                      mirror_num;
  76         int                     have_csum:1;
  77         int                     io_error:1;
  78         u8                      csum[BTRFS_CSUM_SIZE];
  79
  80         struct scrub_recover    *recover;
  81 };
  82
  83 struct scrub_bio {
  84         int                     index;
  85         struct scrub_ctx        *sctx;
  86         struct btrfs_device     *dev;
  87         struct bio              *bio;
  88         blk_status_t            status;
  89         u64                     logical;
  90         u64                     physical;
  91 #if SCRUB_PAGES_PER_WR_BIO >= SCRUB_PAGES_PER_RD_BIO
  92         struct scrub_page       *pagev[SCRUB_PAGES_PER_WR_BIO];
  93 #else
  94         struct scrub_page       *pagev[SCRUB_PAGES_PER_RD_BIO];
  95 #endif
  96         int                     page_count;
  97         int                     next_free;
  98         struct btrfs_work       work;
  99 };
 100
 101 struct scrub_block {
 102         struct scrub_page       *pagev[SCRUB_MAX_PAGES_PER_BLOCK];
 103         int                     page_count;
 104         atomic_t                outstanding_pages;
 105         refcount_t              refs; /* free mem on transition to zero */
 106         struct scrub_ctx        *sctx;
 107         struct scrub_parity     *sparity;
 108         struct {
 109                 unsigned int    header_error:1;
 110                 unsigned int    checksum_error:1;
 111                 unsigned int    no_io_error_seen:1;
 112                 unsigned int    generation_error:1; /* also sets header_error */
 113
 114                 /* The following is for the data used to check parity */
 115                 /* It is for the data with checksum */
 116                 unsigned int    data_corrected:1;
 117         };
 118         struct btrfs_work       work;
 119 };
 120
 121 /* Used for the chunks with parity stripe such RAID5/6 */
 122 struct scrub_parity {
 123         struct scrub_ctx        *sctx;
 124
 125         struct btrfs_device     *scrub_dev;
 126
 127         u64                     logic_start;
 128
 129         u64                     logic_end;
 130
 131         int                     nsectors;
 132
 133         u32                     stripe_len;
 134
 135         refcount_t              refs;
 136
 137         struct list_head        spages;
 138
 139         /* Work of parity check and repair */
 140         struct btrfs_work       work;
 141
 142         /* Mark the parity blocks which have data */
 143         unsigned long           *dbitmap;
 144
 145         /*
 146          * Mark the parity blocks which have data, but errors happen when
 147          * read data or check data
 148          */
 149         unsigned long           *ebitmap;
 150
 151         unsigned long           bitmap[];
 152 };
 153
 154 struct scrub_ctx {
 155         struct scrub_bio        *bios[SCRUB_BIOS_PER_SCTX];
 156         struct btrfs_fs_info    *fs_info;
 157         int                     first_free;
 158         int                     curr;
 159         atomic_t                bios_in_flight;
 160         atomic_t                workers_pending;
 161         spinlock_t              list_lock;
 162         wait_queue_head_t       list_wait;
 163         struct list_head        csum_list;
 164         atomic_t                cancel_req;
 165         int                     readonly;
 166         int                     pages_per_rd_bio;
 167
 168         int                     is_dev_replace;
 169
 170         struct scrub_bio        *wr_curr_bio;
 171         struct mutex            wr_lock;
 172         int                     pages_per_wr_bio; /* <= SCRUB_PAGES_PER_WR_BIO */
 173         struct btrfs_device     *wr_tgtdev;
 174         bool                    flush_all_writes;
 175
 176         /*
 177          * statistics
 178          */
 179         struct btrfs_scrub_progress stat;
 180         spinlock_t              stat_lock;
 181
 182         /*
 183          * Use a ref counter to avoid use-after-free issues. Scrub workers
 184          * decrement bios_in_flight and workers_pending and then do a wakeup
 185          * on the list_wait wait queue. We must ensure the main scrub task
 186          * doesn't free the scrub context before or while the workers are
 187          * doing the wakeup() call.
 188          */
 189         refcount_t              refs;
 190 };
 191
 192 struct scrub_warning {
 193         struct btrfs_path       *path;
 194         u64                     extent_item_size;
 195         const char              *errstr;
 196         u64                     physical;
 197         u64                     logical;
 198         struct btrfs_device     *dev;
 199 };
 200
 201 struct full_stripe_lock {
 202         struct rb_node node;
 203         u64 logical;
 204         u64 refs;
 205         struct mutex mutex;
 206 };
 207
 208 static void scrub_pending_bio_inc(struct scrub_ctx *sctx);
 209 static void scrub_pending_bio_dec(struct scrub_ctx *sctx);
 210 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check);
 211 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
 212                                      struct scrub_block *sblocks_for_recheck);
 213 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
 214                                 struct scrub_block *sblock,
 215                                 int retry_failed_mirror);
 216 static void scrub_recheck_block_checksum(struct scrub_block *sblock);
 217 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
 218                                              struct scrub_block *sblock_good);
 219 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
 220                                             struct scrub_block *sblock_good,
 221                                             int page_num, int force_write);
 222 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock);
 223 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
 224                                            int page_num);
 225 static int scrub_checksum_data(struct scrub_block *sblock);
 226 static int scrub_checksum_tree_block(struct scrub_block *sblock);
 227 static int scrub_checksum_super(struct scrub_block *sblock);
 228 static void scrub_block_get(struct scrub_block *sblock);
 229 static void scrub_block_put(struct scrub_block *sblock);
 230 static void scrub_page_get(struct scrub_page *spage);
 231 static void scrub_page_put(struct scrub_page *spage);
 232 static void scrub_parity_get(struct scrub_parity *sparity);
 233 static void scrub_parity_put(struct scrub_parity *sparity);
 234 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
 235                                     struct scrub_page *spage);
 236 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len,
 237                        u64 physical, struct btrfs_device *dev, u64 flags,
 238                        u64 gen, int mirror_num, u8 *csum,
 239                        u64 physical_for_dev_replace);
 240 static void scrub_bio_end_io(struct bio *bio);
 241 static void scrub_bio_end_io_worker(struct btrfs_work *work);
 242 static void scrub_block_complete(struct scrub_block *sblock);
 243 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
 244                                u64 extent_logical, u32 extent_len,
 245                                u64 *extent_physical,
 246                                struct btrfs_device **extent_dev,
 247                                int *extent_mirror_num);
 248 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
 249                                     struct scrub_page *spage);
 250 static void scrub_wr_submit(struct scrub_ctx *sctx);
 251 static void scrub_wr_bio_end_io(struct bio *bio);
 252 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work);
 253 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 254 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info);
 255 static void scrub_put_ctx(struct scrub_ctx *sctx);
 256
 257 static inline int scrub_is_page_on_raid56(struct scrub_page *spage)
 258 {
 259         return spage->recover &&
 260                (spage->recover->bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK);
 261 }
 262
 263 static void scrub_pending_bio_inc(struct scrub_ctx *sctx)
 264 {
 265         refcount_inc(&sctx->refs);
 266         atomic_inc(&sctx->bios_in_flight);
 267 }
 268
 269 static void scrub_pending_bio_dec(struct scrub_ctx *sctx)
 270 {
 271         atomic_dec(&sctx->bios_in_flight);
 272         wake_up(&sctx->list_wait);
 273         scrub_put_ctx(sctx);
 274 }
 275
 276 static void __scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 277 {
 278         while (atomic_read(&fs_info->scrub_pause_req)) {
 279                 mutex_unlock(&fs_info->scrub_lock);
 280                 wait_event(fs_info->scrub_pause_wait,
 281                    atomic_read(&fs_info->scrub_pause_req) == 0);
 282                 mutex_lock(&fs_info->scrub_lock);
 283         }
 284 }
 285
 286 static void scrub_pause_on(struct btrfs_fs_info *fs_info)
 287 {
 288         atomic_inc(&fs_info->scrubs_paused);
 289         wake_up(&fs_info->scrub_pause_wait);
 290 }
 291
 292 static void scrub_pause_off(struct btrfs_fs_info *fs_info)
 293 {
 294         mutex_lock(&fs_info->scrub_lock);
 295         __scrub_blocked_if_needed(fs_info);
 296         atomic_dec(&fs_info->scrubs_paused);
 297         mutex_unlock(&fs_info->scrub_lock);
 298
 299         wake_up(&fs_info->scrub_pause_wait);
 300 }
 301
 302 static void scrub_blocked_if_needed(struct btrfs_fs_info *fs_info)
 303 {
 304         scrub_pause_on(fs_info);
 305         scrub_pause_off(fs_info);
 306 }
 307
 308 /*
 309  * Insert new full stripe lock into full stripe locks tree
 310  *
 311  * Return pointer to existing or newly inserted full_stripe_lock structure if
 312  * everything works well.
 313  * Return ERR_PTR(-ENOMEM) if we failed to allocate memory
 314  *
 315  * NOTE: caller must hold full_stripe_locks_root->lock before calling this
 316  * function
 317  */
 318 static struct full_stripe_lock *insert_full_stripe_lock(
 319                 struct btrfs_full_stripe_locks_tree *locks_root,
 320                 u64 fstripe_logical)
 321 {
 322         struct rb_node **p;
 323         struct rb_node *parent = NULL;
 324         struct full_stripe_lock *entry;
 325         struct full_stripe_lock *ret;
 326
 327         lockdep_assert_held(&locks_root->lock);
 328
 329         p = &locks_root->root.rb_node;
 330         while (*p) {
 331                 parent = *p;
 332                 entry = rb_entry(parent, struct full_stripe_lock, node);
 333                 if (fstripe_logical < entry->logical) {
 334                         p = &(*p)->rb_left;
 335                 } else if (fstripe_logical > entry->logical) {
 336                         p = &(*p)->rb_right;
 337                 } else {
 338                         entry->refs++;
 339                         return entry;
 340                 }
 341         }
 342
 343         /*
 344          * Insert new lock.
 345          */
 346         ret = kmalloc(sizeof(*ret), GFP_KERNEL);
 347         if (!ret)
 348                 return ERR_PTR(-ENOMEM);
 349         ret->logical = fstripe_logical;
 350         ret->refs = 1;
 351         mutex_init(&ret->mutex);
 352
 353         rb_link_node(&ret->node, parent, p);
 354         rb_insert_color(&ret->node, &locks_root->root);
 355         return ret;
 356 }
 357
 358 /*
 359  * Search for a full stripe lock of a block group
 360  *
 361  * Return pointer to existing full stripe lock if found
 362  * Return NULL if not found
 363  */
 364 static struct full_stripe_lock *search_full_stripe_lock(
 365                 struct btrfs_full_stripe_locks_tree *locks_root,
 366                 u64 fstripe_logical)
 367 {
 368         struct rb_node *node;
 369         struct full_stripe_lock *entry;
 370
 371         lockdep_assert_held(&locks_root->lock);
 372
 373         node = locks_root->root.rb_node;
 374         while (node) {
 375                 entry = rb_entry(node, struct full_stripe_lock, node);
 376                 if (fstripe_logical < entry->logical)
 377                         node = node->rb_left;
 378                 else if (fstripe_logical > entry->logical)
 379                         node = node->rb_right;
 380                 else
 381                         return entry;
 382         }
 383         return NULL;
 384 }
 385
 386 /*
 387  * Helper to get full stripe logical from a normal bytenr.
 388  *
 389  * Caller must ensure @cache is a RAID56 block group.
 390  */
 391 static u64 get_full_stripe_logical(struct btrfs_block_group *cache, u64 bytenr)
 392 {
 393         u64 ret;
 394
 395         /*
 396          * Due to chunk item size limit, full stripe length should not be
 397          * larger than U32_MAX. Just a sanity check here.
 398          */
 399         WARN_ON_ONCE(cache->full_stripe_len >= U32_MAX);
 400
 401         /*
 402          * round_down() can only handle power of 2, while RAID56 full
 403          * stripe length can be 64KiB * n, so we need to manually round down.
 404          */
 405         ret = div64_u64(bytenr - cache->start, cache->full_stripe_len) *
 406                         cache->full_stripe_len + cache->start;
 407         return ret;
 408 }
 409
 410 /*
 411  * Lock a full stripe to avoid concurrency of recovery and read
 412  *
 413  * It's only used for profiles with parities (RAID5/6), for other profiles it
 414  * does nothing.
 415  *
 416  * Return 0 if we locked full stripe covering @bytenr, with a mutex held.
 417  * So caller must call unlock_full_stripe() at the same context.
 418  *
 419  * Return <0 if encounters error.
 420  */
 421 static int lock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
 422                             bool *locked_ret)
 423 {
 424         struct btrfs_block_group *bg_cache;
 425         struct btrfs_full_stripe_locks_tree *locks_root;
 426         struct full_stripe_lock *existing;
 427         u64 fstripe_start;
 428         int ret = 0;
 429
 430         *locked_ret = false;
 431         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
 432         if (!bg_cache) {
 433                 ASSERT(0);
 434                 return -ENOENT;
 435         }
 436
 437         /* Profiles not based on parity don't need full stripe lock */
 438         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
 439                 goto out;
 440         locks_root = &bg_cache->full_stripe_locks_root;
 441
 442         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
 443
 444         /* Now insert the full stripe lock */
 445         mutex_lock(&locks_root->lock);
 446         existing = insert_full_stripe_lock(locks_root, fstripe_start);
 447         mutex_unlock(&locks_root->lock);
 448         if (IS_ERR(existing)) {
 449                 ret = PTR_ERR(existing);
 450                 goto out;
 451         }
 452         mutex_lock(&existing->mutex);
 453         *locked_ret = true;
 454 out:
 455         btrfs_put_block_group(bg_cache);
 456         return ret;
 457 }
 458
 459 /*
 460  * Unlock a full stripe.
 461  *
 462  * NOTE: Caller must ensure it's the same context calling corresponding
 463  * lock_full_stripe().
 464  *
 465  * Return 0 if we unlock full stripe without problem.
 466  * Return <0 for error
 467  */
 468 static int unlock_full_stripe(struct btrfs_fs_info *fs_info, u64 bytenr,
 469                               bool locked)
 470 {
 471         struct btrfs_block_group *bg_cache;
 472         struct btrfs_full_stripe_locks_tree *locks_root;
 473         struct full_stripe_lock *fstripe_lock;
 474         u64 fstripe_start;
 475         bool freeit = false;
 476         int ret = 0;
 477
 478         /* If we didn't acquire full stripe lock, no need to continue */
 479         if (!locked)
 480                 return 0;
 481
 482         bg_cache = btrfs_lookup_block_group(fs_info, bytenr);
 483         if (!bg_cache) {
 484                 ASSERT(0);
 485                 return -ENOENT;
 486         }
 487         if (!(bg_cache->flags & BTRFS_BLOCK_GROUP_RAID56_MASK))
 488                 goto out;
 489
 490         locks_root = &bg_cache->full_stripe_locks_root;
 491         fstripe_start = get_full_stripe_logical(bg_cache, bytenr);
 492
 493         mutex_lock(&locks_root->lock);
 494         fstripe_lock = search_full_stripe_lock(locks_root, fstripe_start);
 495         /* Unpaired unlock_full_stripe() detected */
 496         if (!fstripe_lock) {
 497                 WARN_ON(1);
 498                 ret = -ENOENT;
 499                 mutex_unlock(&locks_root->lock);
 500                 goto out;
 501         }
 502
 503         if (fstripe_lock->refs == 0) {
 504                 WARN_ON(1);
 505                 btrfs_warn(fs_info, "full stripe lock at %llu refcount underflow",
 506                         fstripe_lock->logical);
 507         } else {
 508                 fstripe_lock->refs--;
 509         }
 510
 511         if (fstripe_lock->refs == 0) {
 512                 rb_erase(&fstripe_lock->node, &locks_root->root);
 513                 freeit = true;
 514         }
 515         mutex_unlock(&locks_root->lock);
 516
 517         mutex_unlock(&fstripe_lock->mutex);
 518         if (freeit)
 519                 kfree(fstripe_lock);
 520 out:
 521         btrfs_put_block_group(bg_cache);
 522         return ret;
 523 }
 524
 525 static void scrub_free_csums(struct scrub_ctx *sctx)
 526 {
 527         while (!list_empty(&sctx->csum_list)) {
 528                 struct btrfs_ordered_sum *sum;
 529                 sum = list_first_entry(&sctx->csum_list,
 530                                        struct btrfs_ordered_sum, list);
 531                 list_del(&sum->list);
 532                 kfree(sum);
 533         }
 534 }
 535
 536 static noinline_for_stack void scrub_free_ctx(struct scrub_ctx *sctx)
 537 {
 538         int i;
 539
 540         if (!sctx)
 541                 return;
 542
 543         /* this can happen when scrub is cancelled */
 544         if (sctx->curr != -1) {
 545                 struct scrub_bio *sbio = sctx->bios[sctx->curr];
 546
 547                 for (i = 0; i < sbio->page_count; i++) {
 548                         WARN_ON(!sbio->pagev[i]->page);
 549                         scrub_block_put(sbio->pagev[i]->sblock);
 550                 }
 551                 bio_put(sbio->bio);
 552         }
 553
 554         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 555                 struct scrub_bio *sbio = sctx->bios[i];
 556
 557                 if (!sbio)
 558                         break;
 559                 kfree(sbio);
 560         }
 561
 562         kfree(sctx->wr_curr_bio);
 563         scrub_free_csums(sctx);
 564         kfree(sctx);
 565 }
 566
 567 static void scrub_put_ctx(struct scrub_ctx *sctx)
 568 {
 569         if (refcount_dec_and_test(&sctx->refs))
 570                 scrub_free_ctx(sctx);
 571 }
 572
 573 static noinline_for_stack struct scrub_ctx *scrub_setup_ctx(
 574                 struct btrfs_fs_info *fs_info, int is_dev_replace)
 575 {
 576         struct scrub_ctx *sctx;
 577         int             i;
 578
 579         sctx = kzalloc(sizeof(*sctx), GFP_KERNEL);
 580         if (!sctx)
 581                 goto nomem;
 582         refcount_set(&sctx->refs, 1);
 583         sctx->is_dev_replace = is_dev_replace;
 584         sctx->pages_per_rd_bio = SCRUB_PAGES_PER_RD_BIO;
 585         sctx->curr = -1;
 586         sctx->fs_info = fs_info;
 587         INIT_LIST_HEAD(&sctx->csum_list);
 588         for (i = 0; i < SCRUB_BIOS_PER_SCTX; ++i) {
 589                 struct scrub_bio *sbio;
 590
 591                 sbio = kzalloc(sizeof(*sbio), GFP_KERNEL);
 592                 if (!sbio)
 593                         goto nomem;
 594                 sctx->bios[i] = sbio;
 595
 596                 sbio->index = i;
 597                 sbio->sctx = sctx;
 598                 sbio->page_count = 0;
 599                 btrfs_init_work(&sbio->work, scrub_bio_end_io_worker, NULL,
 600                                 NULL);
 601
 602                 if (i != SCRUB_BIOS_PER_SCTX - 1)
 603                         sctx->bios[i]->next_free = i + 1;
 604                 else
 605                         sctx->bios[i]->next_free = -1;
 606         }
 607         sctx->first_free = 0;
 608         atomic_set(&sctx->bios_in_flight, 0);
 609         atomic_set(&sctx->workers_pending, 0);
 610         atomic_set(&sctx->cancel_req, 0);
 611
 612         spin_lock_init(&sctx->list_lock);
 613         spin_lock_init(&sctx->stat_lock);
 614         init_waitqueue_head(&sctx->list_wait);
 615
 616         WARN_ON(sctx->wr_curr_bio != NULL);
 617         mutex_init(&sctx->wr_lock);
 618         sctx->wr_curr_bio = NULL;
 619         if (is_dev_replace) {
 620                 WARN_ON(!fs_info->dev_replace.tgtdev);
 621                 sctx->pages_per_wr_bio = SCRUB_PAGES_PER_WR_BIO;
 622                 sctx->wr_tgtdev = fs_info->dev_replace.tgtdev;
 623                 sctx->flush_all_writes = false;
 624         }
 625
 626         return sctx;
 627
 628 nomem:
 629         scrub_free_ctx(sctx);
 630         return ERR_PTR(-ENOMEM);
 631 }
 632
 633 static int scrub_print_warning_inode(u64 inum, u64 offset, u64 root,
 634                                      void *warn_ctx)
 635 {
 636         u64 isize;
 637         u32 nlink;
 638         int ret;
 639         int i;
 640         unsigned nofs_flag;
 641         struct extent_buffer *eb;
 642         struct btrfs_inode_item *inode_item;
 643         struct scrub_warning *swarn = warn_ctx;
 644         struct btrfs_fs_info *fs_info = swarn->dev->fs_info;
 645         struct inode_fs_paths *ipath = NULL;
 646         struct btrfs_root *local_root;
 647         struct btrfs_key key;
 648
 649         local_root = btrfs_get_fs_root(fs_info, root, true);
 650         if (IS_ERR(local_root)) {
 651                 ret = PTR_ERR(local_root);
 652                 goto err;
 653         }
 654
 655         /*
 656          * this makes the path point to (inum INODE_ITEM ioff)
 657          */
 658         key.objectid = inum;
 659         key.type = BTRFS_INODE_ITEM_KEY;
 660         key.offset = 0;
 661
 662         ret = btrfs_search_slot(NULL, local_root, &key, swarn->path, 0, 0);
 663         if (ret) {
 664                 btrfs_put_root(local_root);
 665                 btrfs_release_path(swarn->path);
 666                 goto err;
 667         }
 668
 669         eb = swarn->path->nodes[0];
 670         inode_item = btrfs_item_ptr(eb, swarn->path->slots[0],
 671                                         struct btrfs_inode_item);
 672         isize = btrfs_inode_size(eb, inode_item);
 673         nlink = btrfs_inode_nlink(eb, inode_item);
 674         btrfs_release_path(swarn->path);
 675
 676         /*
 677          * init_path might indirectly call vmalloc, or use GFP_KERNEL. Scrub
 678          * uses GFP_NOFS in this context, so we keep it consistent but it does
 679          * not seem to be strictly necessary.
 680          */
 681         nofs_flag = memalloc_nofs_save();
 682         ipath = init_ipath(4096, local_root, swarn->path);
 683         memalloc_nofs_restore(nofs_flag);
 684         if (IS_ERR(ipath)) {
 685                 btrfs_put_root(local_root);
 686                 ret = PTR_ERR(ipath);
 687                 ipath = NULL;
 688                 goto err;
 689         }
 690         ret = paths_from_inode(inum, ipath);
 691
 692         if (ret < 0)
 693                 goto err;
 694
 695         /*
 696          * we deliberately ignore the bit ipath might have been too small to
 697          * hold all of the paths here
 698          */
 699         for (i = 0; i < ipath->fspath->elem_cnt; ++i)
 700                 btrfs_warn_in_rcu(fs_info,
 701 "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu, length %llu, links %u (path: %s)",
 702                                   swarn->errstr, swarn->logical,
 703                                   rcu_str_deref(swarn->dev->name),
 704                                   swarn->physical,
 705                                   root, inum, offset,
 706                                   min(isize - offset, (u64)PAGE_SIZE), nlink,
 707                                   (char *)(unsigned long)ipath->fspath->val[i]);
 708
 709         btrfs_put_root(local_root);
 710         free_ipath(ipath);
 711         return 0;
 712
 713 err:
 714         btrfs_warn_in_rcu(fs_info,
 715                           "%s at logical %llu on dev %s, physical %llu, root %llu, inode %llu, offset %llu: path resolving failed with ret=%d",
 716                           swarn->errstr, swarn->logical,
 717                           rcu_str_deref(swarn->dev->name),
 718                           swarn->physical,
 719                           root, inum, offset, ret);
 720
 721         free_ipath(ipath);
 722         return 0;
 723 }
 724
 725 static void scrub_print_warning(const char *errstr, struct scrub_block *sblock)
 726 {
 727         struct btrfs_device *dev;
 728         struct btrfs_fs_info *fs_info;
 729         struct btrfs_path *path;
 730         struct btrfs_key found_key;
 731         struct extent_buffer *eb;
 732         struct btrfs_extent_item *ei;
 733         struct scrub_warning swarn;
 734         unsigned long ptr = 0;
 735         u64 extent_item_pos;
 736         u64 flags = 0;
 737         u64 ref_root;
 738         u32 item_size;
 739         u8 ref_level = 0;
 740         int ret;
 741
 742         WARN_ON(sblock->page_count < 1);
 743         dev = sblock->pagev[0]->dev;
 744         fs_info = sblock->sctx->fs_info;
 745
 746         path = btrfs_alloc_path();
 747         if (!path)
 748                 return;
 749
 750         swarn.physical = sblock->pagev[0]->physical;
 751         swarn.logical = sblock->pagev[0]->logical;
 752         swarn.errstr = errstr;
 753         swarn.dev = NULL;
 754
 755         ret = extent_from_logical(fs_info, swarn.logical, path, &found_key,
 756                                   &flags);
 757         if (ret < 0)
 758                 goto out;
 759
 760         extent_item_pos = swarn.logical - found_key.objectid;
 761         swarn.extent_item_size = found_key.offset;
 762
 763         eb = path->nodes[0];
 764         ei = btrfs_item_ptr(eb, path->slots[0], struct btrfs_extent_item);
 765         item_size = btrfs_item_size_nr(eb, path->slots[0]);
 766
 767         if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
 768                 do {
 769                         ret = tree_backref_for_extent(&ptr, eb, &found_key, ei,
 770                                                       item_size, &ref_root,
 771                                                       &ref_level);
 772                         btrfs_warn_in_rcu(fs_info,
 773 "%s at logical %llu on dev %s, physical %llu: metadata %s (level %d) in tree %llu",
 774                                 errstr, swarn.logical,
 775                                 rcu_str_deref(dev->name),
 776                                 swarn.physical,
 777                                 ref_level ? "node" : "leaf",
 778                                 ret < 0 ? -1 : ref_level,
 779                                 ret < 0 ? -1 : ref_root);
 780                 } while (ret != 1);
 781                 btrfs_release_path(path);
 782         } else {
 783                 btrfs_release_path(path);
 784                 swarn.path = path;
 785                 swarn.dev = dev;
 786                 iterate_extent_inodes(fs_info, found_key.objectid,
 787                                         extent_item_pos, 1,
 788                                         scrub_print_warning_inode, &swarn, false);
 789         }
 790
 791 out:
 792         btrfs_free_path(path);
 793 }
 794
 795 static inline void scrub_get_recover(struct scrub_recover *recover)
 796 {
 797         refcount_inc(&recover->refs);
 798 }
 799
 800 static inline void scrub_put_recover(struct btrfs_fs_info *fs_info,
 801                                      struct scrub_recover *recover)
 802 {
 803         if (refcount_dec_and_test(&recover->refs)) {
 804                 btrfs_bio_counter_dec(fs_info);
 805                 btrfs_put_bbio(recover->bbio);
 806                 kfree(recover);
 807         }
 808 }
 809
 810 /*
 811  * scrub_handle_errored_block gets called when either verification of the
 812  * pages failed or the bio failed to read, e.g. with EIO. In the latter
 813  * case, this function handles all pages in the bio, even though only one
 814  * may be bad.
 815  * The goal of this function is to repair the errored block by using the
 816  * contents of one of the mirrors.
 817  */
 818 static int scrub_handle_errored_block(struct scrub_block *sblock_to_check)
 819 {
 820         struct scrub_ctx *sctx = sblock_to_check->sctx;
 821         struct btrfs_device *dev;
 822         struct btrfs_fs_info *fs_info;
 823         u64 logical;
 824         unsigned int failed_mirror_index;
 825         unsigned int is_metadata;
 826         unsigned int have_csum;
 827         struct scrub_block *sblocks_for_recheck; /* holds one for each mirror */
 828         struct scrub_block *sblock_bad;
 829         int ret;
 830         int mirror_index;
 831         int page_num;
 832         int success;
 833         bool full_stripe_locked;
 834         unsigned int nofs_flag;
 835         static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
 836                                       DEFAULT_RATELIMIT_BURST);
 837
 838         BUG_ON(sblock_to_check->page_count < 1);
 839         fs_info = sctx->fs_info;
 840         if (sblock_to_check->pagev[0]->flags & BTRFS_EXTENT_FLAG_SUPER) {
 841                 /*
 842                  * if we find an error in a super block, we just report it.
 843                  * They will get written with the next transaction commit
 844                  * anyway
 845                  */
 846                 spin_lock(&sctx->stat_lock);
 847                 ++sctx->stat.super_errors;
 848                 spin_unlock(&sctx->stat_lock);
 849                 return 0;
 850         }
 851         logical = sblock_to_check->pagev[0]->logical;
 852         BUG_ON(sblock_to_check->pagev[0]->mirror_num < 1);
 853         failed_mirror_index = sblock_to_check->pagev[0]->mirror_num - 1;
 854         is_metadata = !(sblock_to_check->pagev[0]->flags &
 855                         BTRFS_EXTENT_FLAG_DATA);
 856         have_csum = sblock_to_check->pagev[0]->have_csum;
 857         dev = sblock_to_check->pagev[0]->dev;
 858
 859         /*
 860          * We must use GFP_NOFS because the scrub task might be waiting for a
 861          * worker task executing this function and in turn a transaction commit
 862          * might be waiting the scrub task to pause (which needs to wait for all
 863          * the worker tasks to complete before pausing).
 864          * We do allocations in the workers through insert_full_stripe_lock()
 865          * and scrub_add_page_to_wr_bio(), which happens down the call chain of
 866          * this function.
 867          */
 868         nofs_flag = memalloc_nofs_save();
 869         /*
 870          * For RAID5/6, race can happen for a different device scrub thread.
 871          * For data corruption, Parity and Data threads will both try
 872          * to recovery the data.
 873          * Race can lead to doubly added csum error, or even unrecoverable
 874          * error.
 875          */
 876         ret = lock_full_stripe(fs_info, logical, &full_stripe_locked);
 877         if (ret < 0) {
 878                 memalloc_nofs_restore(nofs_flag);
 879                 spin_lock(&sctx->stat_lock);
 880                 if (ret == -ENOMEM)
 881                         sctx->stat.malloc_errors++;
 882                 sctx->stat.read_errors++;
 883                 sctx->stat.uncorrectable_errors++;
 884                 spin_unlock(&sctx->stat_lock);
 885                 return ret;
 886         }
 887
 888         /*
 889          * read all mirrors one after the other. This includes to
 890          * re-read the extent or metadata block that failed (that was
 891          * the cause that this fixup code is called) another time,
 892          * page by page this time in order to know which pages
 893          * caused I/O errors and which ones are good (for all mirrors).
 894          * It is the goal to handle the situation when more than one
 895          * mirror contains I/O errors, but the errors do not
 896          * overlap, i.e. the data can be repaired by selecting the
 897          * pages from those mirrors without I/O error on the
 898          * particular pages. One example (with blocks >= 2 * PAGE_SIZE)
 899          * would be that mirror #1 has an I/O error on the first page,
 900          * the second page is good, and mirror #2 has an I/O error on
 901          * the second page, but the first page is good.
 902          * Then the first page of the first mirror can be repaired by
 903          * taking the first page of the second mirror, and the
 904          * second page of the second mirror can be repaired by
 905          * copying the contents of the 2nd page of the 1st mirror.
 906          * One more note: if the pages of one mirror contain I/O
 907          * errors, the checksum cannot be verified. In order to get
 908          * the best data for repairing, the first attempt is to find
 909          * a mirror without I/O errors and with a validated checksum.
 910          * Only if this is not possible, the pages are picked from
 911          * mirrors with I/O errors without considering the checksum.
 912          * If the latter is the case, at the end, the checksum of the
 913          * repaired area is verified in order to correctly maintain
 914          * the statistics.
 915          */
 916
 917         sblocks_for_recheck = kcalloc(BTRFS_MAX_MIRRORS,
 918                                       sizeof(*sblocks_for_recheck), GFP_KERNEL);
 919         if (!sblocks_for_recheck) {
 920                 spin_lock(&sctx->stat_lock);
 921                 sctx->stat.malloc_errors++;
 922                 sctx->stat.read_errors++;
 923                 sctx->stat.uncorrectable_errors++;
 924                 spin_unlock(&sctx->stat_lock);
 925                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 926                 goto out;
 927         }
 928
 929         /* setup the context, map the logical blocks and alloc the pages */
 930         ret = scrub_setup_recheck_block(sblock_to_check, sblocks_for_recheck);
 931         if (ret) {
 932                 spin_lock(&sctx->stat_lock);
 933                 sctx->stat.read_errors++;
 934                 sctx->stat.uncorrectable_errors++;
 935                 spin_unlock(&sctx->stat_lock);
 936                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 937                 goto out;
 938         }
 939         BUG_ON(failed_mirror_index >= BTRFS_MAX_MIRRORS);
 940         sblock_bad = sblocks_for_recheck + failed_mirror_index;
 941
 942         /* build and submit the bios for the failed mirror, check checksums */
 943         scrub_recheck_block(fs_info, sblock_bad, 1);
 944
 945         if (!sblock_bad->header_error && !sblock_bad->checksum_error &&
 946             sblock_bad->no_io_error_seen) {
 947                 /*
 948                  * the error disappeared after reading page by page, or
 949                  * the area was part of a huge bio and other parts of the
 950                  * bio caused I/O errors, or the block layer merged several
 951                  * read requests into one and the error is caused by a
 952                  * different bio (usually one of the two latter cases is
 953                  * the cause)
 954                  */
 955                 spin_lock(&sctx->stat_lock);
 956                 sctx->stat.unverified_errors++;
 957                 sblock_to_check->data_corrected = 1;
 958                 spin_unlock(&sctx->stat_lock);
 959
 960                 if (sctx->is_dev_replace)
 961                         scrub_write_block_to_dev_replace(sblock_bad);
 962                 goto out;
 963         }
 964
 965         if (!sblock_bad->no_io_error_seen) {
 966                 spin_lock(&sctx->stat_lock);
 967                 sctx->stat.read_errors++;
 968                 spin_unlock(&sctx->stat_lock);
 969                 if (__ratelimit(&rs))
 970                         scrub_print_warning("i/o error", sblock_to_check);
 971                 btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_READ_ERRS);
 972         } else if (sblock_bad->checksum_error) {
 973                 spin_lock(&sctx->stat_lock);
 974                 sctx->stat.csum_errors++;
 975                 spin_unlock(&sctx->stat_lock);
 976                 if (__ratelimit(&rs))
 977                         scrub_print_warning("checksum error", sblock_to_check);
 978                 btrfs_dev_stat_inc_and_print(dev,
 979                                              BTRFS_DEV_STAT_CORRUPTION_ERRS);
 980         } else if (sblock_bad->header_error) {
 981                 spin_lock(&sctx->stat_lock);
 982                 sctx->stat.verify_errors++;
 983                 spin_unlock(&sctx->stat_lock);
 984                 if (__ratelimit(&rs))
 985                         scrub_print_warning("checksum/header error",
 986                                             sblock_to_check);
 987                 if (sblock_bad->generation_error)
 988                         btrfs_dev_stat_inc_and_print(dev,
 989                                 BTRFS_DEV_STAT_GENERATION_ERRS);
 990                 else
 991                         btrfs_dev_stat_inc_and_print(dev,
 992                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
 993         }
 994
 995         if (sctx->readonly) {
 996                 ASSERT(!sctx->is_dev_replace);
 997                 goto out;
 998         }
 999
1000         /*
1001          * now build and submit the bios for the other mirrors, check
1002          * checksums.
1003          * First try to pick the mirror which is completely without I/O
1004          * errors and also does not have a checksum error.
1005          * If one is found, and if a checksum is present, the full block
1006          * that is known to contain an error is rewritten. Afterwards
1007          * the block is known to be corrected.
1008          * If a mirror is found which is completely correct, and no
1009          * checksum is present, only those pages are rewritten that had
1010          * an I/O error in the block to be repaired, since it cannot be
1011          * determined, which copy of the other pages is better (and it
1012          * could happen otherwise that a correct page would be
1013          * overwritten by a bad one).
1014          */
1015         for (mirror_index = 0; ;mirror_index++) {
1016                 struct scrub_block *sblock_other;
1017
1018                 if (mirror_index == failed_mirror_index)
1019                         continue;
1020
1021                 /* raid56's mirror can be more than BTRFS_MAX_MIRRORS */
1022                 if (!scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1023                         if (mirror_index >= BTRFS_MAX_MIRRORS)
1024                                 break;
1025                         if (!sblocks_for_recheck[mirror_index].page_count)
1026                                 break;
1027
1028                         sblock_other = sblocks_for_recheck + mirror_index;
1029                 } else {
1030                         struct scrub_recover *r = sblock_bad->pagev[0]->recover;
1031                         int max_allowed = r->bbio->num_stripes -
1032                                                 r->bbio->num_tgtdevs;
1033
1034                         if (mirror_index >= max_allowed)
1035                                 break;
1036                         if (!sblocks_for_recheck[1].page_count)
1037                                 break;
1038
1039                         ASSERT(failed_mirror_index == 0);
1040                         sblock_other = sblocks_for_recheck + 1;
1041                         sblock_other->pagev[0]->mirror_num = 1 + mirror_index;
1042                 }
1043
1044                 /* build and submit the bios, check checksums */
1045                 scrub_recheck_block(fs_info, sblock_other, 0);
1046
1047                 if (!sblock_other->header_error &&
1048                     !sblock_other->checksum_error &&
1049                     sblock_other->no_io_error_seen) {
1050                         if (sctx->is_dev_replace) {
1051                                 scrub_write_block_to_dev_replace(sblock_other);
1052                                 goto corrected_error;
1053                         } else {
1054                                 ret = scrub_repair_block_from_good_copy(
1055                                                 sblock_bad, sblock_other);
1056                                 if (!ret)
1057                                         goto corrected_error;
1058                         }
1059                 }
1060         }
1061
1062         if (sblock_bad->no_io_error_seen && !sctx->is_dev_replace)
1063                 goto did_not_correct_error;
1064
1065         /*
1066          * In case of I/O errors in the area that is supposed to be
1067          * repaired, continue by picking good copies of those pages.
1068          * Select the good pages from mirrors to rewrite bad pages from
1069          * the area to fix. Afterwards verify the checksum of the block
1070          * that is supposed to be repaired. This verification step is
1071          * only done for the purpose of statistic counting and for the
1072          * final scrub report, whether errors remain.
1073          * A perfect algorithm could make use of the checksum and try
1074          * all possible combinations of pages from the different mirrors
1075          * until the checksum verification succeeds. For example, when
1076          * the 2nd page of mirror #1 faces I/O errors, and the 2nd page
1077          * of mirror #2 is readable but the final checksum test fails,
1078          * then the 2nd page of mirror #3 could be tried, whether now
1079          * the final checksum succeeds. But this would be a rare
1080          * exception and is therefore not implemented. At least it is
1081          * avoided that the good copy is overwritten.
1082          * A more useful improvement would be to pick the sectors
1083          * without I/O error based on sector sizes (512 bytes on legacy
1084          * disks) instead of on PAGE_SIZE. Then maybe 512 byte of one
1085          * mirror could be repaired by taking 512 byte of a different
1086          * mirror, even if other 512 byte sectors in the same PAGE_SIZE
1087          * area are unreadable.
1088          */
1089         success = 1;
1090         for (page_num = 0; page_num < sblock_bad->page_count;
1091              page_num++) {
1092                 struct scrub_page *spage_bad = sblock_bad->pagev[page_num];
1093                 struct scrub_block *sblock_other = NULL;
1094
1095                 /* skip no-io-error page in scrub */
1096                 if (!spage_bad->io_error && !sctx->is_dev_replace)
1097                         continue;
1098
1099                 if (scrub_is_page_on_raid56(sblock_bad->pagev[0])) {
1100                         /*
1101                          * In case of dev replace, if raid56 rebuild process
1102                          * didn't work out correct data, then copy the content
1103                          * in sblock_bad to make sure target device is identical
1104                          * to source device, instead of writing garbage data in
1105                          * sblock_for_recheck array to target device.
1106                          */
1107                         sblock_other = NULL;
1108                 } else if (spage_bad->io_error) {
1109                         /* try to find no-io-error page in mirrors */
1110                         for (mirror_index = 0;
1111                              mirror_index < BTRFS_MAX_MIRRORS &&
1112                              sblocks_for_recheck[mirror_index].page_count > 0;
1113                              mirror_index++) {
1114                                 if (!sblocks_for_recheck[mirror_index].
1115                                     pagev[page_num]->io_error) {
1116                                         sblock_other = sblocks_for_recheck +
1117                                                        mirror_index;
1118                                         break;
1119                                 }
1120                         }
1121                         if (!sblock_other)
1122                                 success = 0;
1123                 }
1124
1125                 if (sctx->is_dev_replace) {
1126                         /*
1127                          * did not find a mirror to fetch the page
1128                          * from. scrub_write_page_to_dev_replace()
1129                          * handles this case (page->io_error), by
1130                          * filling the block with zeros before
1131                          * submitting the write request
1132                          */
1133                         if (!sblock_other)
1134                                 sblock_other = sblock_bad;
1135
1136                         if (scrub_write_page_to_dev_replace(sblock_other,
1137                                                             page_num) != 0) {
1138                                 atomic64_inc(
1139                                         &fs_info->dev_replace.num_write_errors);
1140                                 success = 0;
1141                         }
1142                 } else if (sblock_other) {
1143                         ret = scrub_repair_page_from_good_copy(sblock_bad,
1144                                                                sblock_other,
1145                                                                page_num, 0);
1146                         if (0 == ret)
1147                                 spage_bad->io_error = 0;
1148                         else
1149                                 success = 0;
1150                 }
1151         }
1152
1153         if (success && !sctx->is_dev_replace) {
1154                 if (is_metadata || have_csum) {
1155                         /*
1156                          * need to verify the checksum now that all
1157                          * sectors on disk are repaired (the write
1158                          * request for data to be repaired is on its way).
1159                          * Just be lazy and use scrub_recheck_block()
1160                          * which re-reads the data before the checksum
1161                          * is verified, but most likely the data comes out
1162                          * of the page cache.
1163                          */
1164                         scrub_recheck_block(fs_info, sblock_bad, 1);
1165                         if (!sblock_bad->header_error &&
1166                             !sblock_bad->checksum_error &&
1167                             sblock_bad->no_io_error_seen)
1168                                 goto corrected_error;
1169                         else
1170                                 goto did_not_correct_error;
1171                 } else {
1172 corrected_error:
1173                         spin_lock(&sctx->stat_lock);
1174                         sctx->stat.corrected_errors++;
1175                         sblock_to_check->data_corrected = 1;
1176                         spin_unlock(&sctx->stat_lock);
1177                         btrfs_err_rl_in_rcu(fs_info,
1178                                 "fixed up error at logical %llu on dev %s",
1179                                 logical, rcu_str_deref(dev->name));
1180                 }
1181         } else {
1182 did_not_correct_error:
1183                 spin_lock(&sctx->stat_lock);
1184                 sctx->stat.uncorrectable_errors++;
1185                 spin_unlock(&sctx->stat_lock);
1186                 btrfs_err_rl_in_rcu(fs_info,
1187                         "unable to fixup (regular) error at logical %llu on dev %s",
1188                         logical, rcu_str_deref(dev->name));
1189         }
1190
1191 out:
1192         if (sblocks_for_recheck) {
1193                 for (mirror_index = 0; mirror_index < BTRFS_MAX_MIRRORS;
1194                      mirror_index++) {
1195                         struct scrub_block *sblock = sblocks_for_recheck +
1196                                                      mirror_index;
1197                         struct scrub_recover *recover;
1198                         int page_index;
1199
1200                         for (page_index = 0; page_index < sblock->page_count;
1201                              page_index++) {
1202                                 sblock->pagev[page_index]->sblock = NULL;
1203                                 recover = sblock->pagev[page_index]->recover;
1204                                 if (recover) {
1205                                         scrub_put_recover(fs_info, recover);
1206                                         sblock->pagev[page_index]->recover =
1207                                                                         NULL;
1208                                 }
1209                                 scrub_page_put(sblock->pagev[page_index]);
1210                         }
1211                 }
1212                 kfree(sblocks_for_recheck);
1213         }
1214
1215         ret = unlock_full_stripe(fs_info, logical, full_stripe_locked);
1216         memalloc_nofs_restore(nofs_flag);
1217         if (ret < 0)
1218                 return ret;
1219         return 0;
1220 }
1221
1222 static inline int scrub_nr_raid_mirrors(struct btrfs_bio *bbio)
1223 {
1224         if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID5)
1225                 return 2;
1226         else if (bbio->map_type & BTRFS_BLOCK_GROUP_RAID6)
1227                 return 3;
1228         else
1229                 return (int)bbio->num_stripes;
1230 }
1231
1232 static inline void scrub_stripe_index_and_offset(u64 logical, u64 map_type,
1233                                                  u64 *raid_map,
1234                                                  u64 mapped_length,
1235                                                  int nstripes, int mirror,
1236                                                  int *stripe_index,
1237                                                  u64 *stripe_offset)
1238 {
1239         int i;
1240
1241         if (map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
1242                 /* RAID5/6 */
1243                 for (i = 0; i < nstripes; i++) {
1244                         if (raid_map[i] == RAID6_Q_STRIPE ||
1245                             raid_map[i] == RAID5_P_STRIPE)
1246                                 continue;
1247
1248                         if (logical >= raid_map[i] &&
1249                             logical < raid_map[i] + mapped_length)
1250                                 break;
1251                 }
1252
1253                 *stripe_index = i;
1254                 *stripe_offset = logical - raid_map[i];
1255         } else {
1256                 /* The other RAID type */
1257                 *stripe_index = mirror;
1258                 *stripe_offset = 0;
1259         }
1260 }
1261
1262 static int scrub_setup_recheck_block(struct scrub_block *original_sblock,
1263                                      struct scrub_block *sblocks_for_recheck)
1264 {
1265         struct scrub_ctx *sctx = original_sblock->sctx;
1266         struct btrfs_fs_info *fs_info = sctx->fs_info;
1267         u64 length = original_sblock->page_count * PAGE_SIZE;
1268         u64 logical = original_sblock->pagev[0]->logical;
1269         u64 generation = original_sblock->pagev[0]->generation;
1270         u64 flags = original_sblock->pagev[0]->flags;
1271         u64 have_csum = original_sblock->pagev[0]->have_csum;
1272         struct scrub_recover *recover;
1273         struct btrfs_bio *bbio;
1274         u64 sublen;
1275         u64 mapped_length;
1276         u64 stripe_offset;
1277         int stripe_index;
1278         int page_index = 0;
1279         int mirror_index;
1280         int nmirrors;
1281         int ret;
1282
1283         /*
1284          * note: the two members refs and outstanding_pages
1285          * are not used (and not set) in the blocks that are used for
1286          * the recheck procedure
1287          */
1288
1289         while (length > 0) {
1290                 sublen = min_t(u64, length, PAGE_SIZE);
1291                 mapped_length = sublen;
1292                 bbio = NULL;
1293
1294                 /*
1295                  * with a length of PAGE_SIZE, each returned stripe
1296                  * represents one mirror
1297                  */
1298                 btrfs_bio_counter_inc_blocked(fs_info);
1299                 ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
1300                                 logical, &mapped_length, &bbio);
1301                 if (ret || !bbio || mapped_length < sublen) {
1302                         btrfs_put_bbio(bbio);
1303                         btrfs_bio_counter_dec(fs_info);
1304                         return -EIO;
1305                 }
1306
1307                 recover = kzalloc(sizeof(struct scrub_recover), GFP_NOFS);
1308                 if (!recover) {
1309                         btrfs_put_bbio(bbio);
1310                         btrfs_bio_counter_dec(fs_info);
1311                         return -ENOMEM;
1312                 }
1313
1314                 refcount_set(&recover->refs, 1);
1315                 recover->bbio = bbio;
1316                 recover->map_length = mapped_length;
1317
1318                 BUG_ON(page_index >= SCRUB_MAX_PAGES_PER_BLOCK);
1319
1320                 nmirrors = min(scrub_nr_raid_mirrors(bbio), BTRFS_MAX_MIRRORS);
1321
1322                 for (mirror_index = 0; mirror_index < nmirrors;
1323                      mirror_index++) {
1324                         struct scrub_block *sblock;
1325                         struct scrub_page *spage;
1326
1327                         sblock = sblocks_for_recheck + mirror_index;
1328                         sblock->sctx = sctx;
1329
1330                         spage = kzalloc(sizeof(*spage), GFP_NOFS);
1331                         if (!spage) {
1332 leave_nomem:
1333                                 spin_lock(&sctx->stat_lock);
1334                                 sctx->stat.malloc_errors++;
1335                                 spin_unlock(&sctx->stat_lock);
1336                                 scrub_put_recover(fs_info, recover);
1337                                 return -ENOMEM;
1338                         }
1339                         scrub_page_get(spage);
1340                         sblock->pagev[page_index] = spage;
1341                         spage->sblock = sblock;
1342                         spage->flags = flags;
1343                         spage->generation = generation;
1344                         spage->logical = logical;
1345                         spage->have_csum = have_csum;
1346                         if (have_csum)
1347                                 memcpy(spage->csum,
1348                                        original_sblock->pagev[0]->csum,
1349                                        sctx->fs_info->csum_size);
1350
1351                         scrub_stripe_index_and_offset(logical,
1352                                                       bbio->map_type,
1353                                                       bbio->raid_map,
1354                                                       mapped_length,
1355                                                       bbio->num_stripes -
1356                                                       bbio->num_tgtdevs,
1357                                                       mirror_index,
1358                                                       &stripe_index,
1359                                                       &stripe_offset);
1360                         spage->physical = bbio->stripes[stripe_index].physical +
1361                                          stripe_offset;
1362                         spage->dev = bbio->stripes[stripe_index].dev;
1363
1364                         BUG_ON(page_index >= original_sblock->page_count);
1365                         spage->physical_for_dev_replace =
1366                                 original_sblock->pagev[page_index]->
1367                                 physical_for_dev_replace;
1368                         /* for missing devices, dev->bdev is NULL */
1369                         spage->mirror_num = mirror_index + 1;
1370                         sblock->page_count++;
1371                         spage->page = alloc_page(GFP_NOFS);
1372                         if (!spage->page)
1373                                 goto leave_nomem;
1374
1375                         scrub_get_recover(recover);
1376                         spage->recover = recover;
1377                 }
1378                 scrub_put_recover(fs_info, recover);
1379                 length -= sublen;
1380                 logical += sublen;
1381                 page_index++;
1382         }
1383
1384         return 0;
1385 }
1386
1387 static void scrub_bio_wait_endio(struct bio *bio)
1388 {
1389         complete(bio->bi_private);
1390 }
1391
1392 static int scrub_submit_raid56_bio_wait(struct btrfs_fs_info *fs_info,
1393                                         struct bio *bio,
1394                                         struct scrub_page *spage)
1395 {
1396         DECLARE_COMPLETION_ONSTACK(done);
1397         int ret;
1398         int mirror_num;
1399
1400         bio->bi_iter.bi_sector = spage->logical >> 9;
1401         bio->bi_private = &done;
1402         bio->bi_end_io = scrub_bio_wait_endio;
1403
1404         mirror_num = spage->sblock->pagev[0]->mirror_num;
1405         ret = raid56_parity_recover(fs_info, bio, spage->recover->bbio,
1406                                     spage->recover->map_length,
1407                                     mirror_num, 0);
1408         if (ret)
1409                 return ret;
1410
1411         wait_for_completion_io(&done);
1412         return blk_status_to_errno(bio->bi_status);
1413 }
1414
1415 static void scrub_recheck_block_on_raid56(struct btrfs_fs_info *fs_info,
1416                                           struct scrub_block *sblock)
1417 {
1418         struct scrub_page *first_page = sblock->pagev[0];
1419         struct bio *bio;
1420         int page_num;
1421
1422         /* All pages in sblock belong to the same stripe on the same device. */
1423         ASSERT(first_page->dev);
1424         if (!first_page->dev->bdev)
1425                 goto out;
1426
1427         bio = btrfs_io_bio_alloc(BIO_MAX_PAGES);
1428         bio_set_dev(bio, first_page->dev->bdev);
1429
1430         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1431                 struct scrub_page *spage = sblock->pagev[page_num];
1432
1433                 WARN_ON(!spage->page);
1434                 bio_add_page(bio, spage->page, PAGE_SIZE, 0);
1435         }
1436
1437         if (scrub_submit_raid56_bio_wait(fs_info, bio, first_page)) {
1438                 bio_put(bio);
1439                 goto out;
1440         }
1441
1442         bio_put(bio);
1443
1444         scrub_recheck_block_checksum(sblock);
1445
1446         return;
1447 out:
1448         for (page_num = 0; page_num < sblock->page_count; page_num++)
1449                 sblock->pagev[page_num]->io_error = 1;
1450
1451         sblock->no_io_error_seen = 0;
1452 }
1453
1454 /*
1455  * this function will check the on disk data for checksum errors, header
1456  * errors and read I/O errors. If any I/O errors happen, the exact pages
1457  * which are errored are marked as being bad. The goal is to enable scrub
1458  * to take those pages that are not errored from all the mirrors so that
1459  * the pages that are errored in the just handled mirror can be repaired.
1460  */
1461 static void scrub_recheck_block(struct btrfs_fs_info *fs_info,
1462                                 struct scrub_block *sblock,
1463                                 int retry_failed_mirror)
1464 {
1465         int page_num;
1466
1467         sblock->no_io_error_seen = 1;
1468
1469         /* short cut for raid56 */
1470         if (!retry_failed_mirror && scrub_is_page_on_raid56(sblock->pagev[0]))
1471                 return scrub_recheck_block_on_raid56(fs_info, sblock);
1472
1473         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1474                 struct bio *bio;
1475                 struct scrub_page *spage = sblock->pagev[page_num];
1476
1477                 if (spage->dev->bdev == NULL) {
1478                         spage->io_error = 1;
1479                         sblock->no_io_error_seen = 0;
1480                         continue;
1481                 }
1482
1483                 WARN_ON(!spage->page);
1484                 bio = btrfs_io_bio_alloc(1);
1485                 bio_set_dev(bio, spage->dev->bdev);
1486
1487                 bio_add_page(bio, spage->page, PAGE_SIZE, 0);
1488                 bio->bi_iter.bi_sector = spage->physical >> 9;
1489                 bio->bi_opf = REQ_OP_READ;
1490
1491                 if (btrfsic_submit_bio_wait(bio)) {
1492                         spage->io_error = 1;
1493                         sblock->no_io_error_seen = 0;
1494                 }
1495
1496                 bio_put(bio);
1497         }
1498
1499         if (sblock->no_io_error_seen)
1500                 scrub_recheck_block_checksum(sblock);
1501 }
1502
1503 static inline int scrub_check_fsid(u8 fsid[],
1504                                    struct scrub_page *spage)
1505 {
1506         struct btrfs_fs_devices *fs_devices = spage->dev->fs_devices;
1507         int ret;
1508
1509         ret = memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
1510         return !ret;
1511 }
1512
1513 static void scrub_recheck_block_checksum(struct scrub_block *sblock)
1514 {
1515         sblock->header_error = 0;
1516         sblock->checksum_error = 0;
1517         sblock->generation_error = 0;
1518
1519         if (sblock->pagev[0]->flags & BTRFS_EXTENT_FLAG_DATA)
1520                 scrub_checksum_data(sblock);
1521         else
1522                 scrub_checksum_tree_block(sblock);
1523 }
1524
1525 static int scrub_repair_block_from_good_copy(struct scrub_block *sblock_bad,
1526                                              struct scrub_block *sblock_good)
1527 {
1528         int page_num;
1529         int ret = 0;
1530
1531         for (page_num = 0; page_num < sblock_bad->page_count; page_num++) {
1532                 int ret_sub;
1533
1534                 ret_sub = scrub_repair_page_from_good_copy(sblock_bad,
1535                                                            sblock_good,
1536                                                            page_num, 1);
1537                 if (ret_sub)
1538                         ret = ret_sub;
1539         }
1540
1541         return ret;
1542 }
1543
1544 static int scrub_repair_page_from_good_copy(struct scrub_block *sblock_bad,
1545                                             struct scrub_block *sblock_good,
1546                                             int page_num, int force_write)
1547 {
1548         struct scrub_page *spage_bad = sblock_bad->pagev[page_num];
1549         struct scrub_page *spage_good = sblock_good->pagev[page_num];
1550         struct btrfs_fs_info *fs_info = sblock_bad->sctx->fs_info;
1551
1552         BUG_ON(spage_bad->page == NULL);
1553         BUG_ON(spage_good->page == NULL);
1554         if (force_write || sblock_bad->header_error ||
1555             sblock_bad->checksum_error || spage_bad->io_error) {
1556                 struct bio *bio;
1557                 int ret;
1558
1559                 if (!spage_bad->dev->bdev) {
1560                         btrfs_warn_rl(fs_info,
1561                                 "scrub_repair_page_from_good_copy(bdev == NULL) is unexpected");
1562                         return -EIO;
1563                 }
1564
1565                 bio = btrfs_io_bio_alloc(1);
1566                 bio_set_dev(bio, spage_bad->dev->bdev);
1567                 bio->bi_iter.bi_sector = spage_bad->physical >> 9;
1568                 bio->bi_opf = REQ_OP_WRITE;
1569
1570                 ret = bio_add_page(bio, spage_good->page, PAGE_SIZE, 0);
1571                 if (PAGE_SIZE != ret) {
1572                         bio_put(bio);
1573                         return -EIO;
1574                 }
1575
1576                 if (btrfsic_submit_bio_wait(bio)) {
1577                         btrfs_dev_stat_inc_and_print(spage_bad->dev,
1578                                 BTRFS_DEV_STAT_WRITE_ERRS);
1579                         atomic64_inc(&fs_info->dev_replace.num_write_errors);
1580                         bio_put(bio);
1581                         return -EIO;
1582                 }
1583                 bio_put(bio);
1584         }
1585
1586         return 0;
1587 }
1588
1589 static void scrub_write_block_to_dev_replace(struct scrub_block *sblock)
1590 {
1591         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
1592         int page_num;
1593
1594         /*
1595          * This block is used for the check of the parity on the source device,
1596          * so the data needn't be written into the destination device.
1597          */
1598         if (sblock->sparity)
1599                 return;
1600
1601         for (page_num = 0; page_num < sblock->page_count; page_num++) {
1602                 int ret;
1603
1604                 ret = scrub_write_page_to_dev_replace(sblock, page_num);
1605                 if (ret)
1606                         atomic64_inc(&fs_info->dev_replace.num_write_errors);
1607         }
1608 }
1609
1610 static int scrub_write_page_to_dev_replace(struct scrub_block *sblock,
1611                                            int page_num)
1612 {
1613         struct scrub_page *spage = sblock->pagev[page_num];
1614
1615         BUG_ON(spage->page == NULL);
1616         if (spage->io_error)
1617                 clear_page(page_address(spage->page));
1618
1619         return scrub_add_page_to_wr_bio(sblock->sctx, spage);
1620 }
1621
1622 static int scrub_add_page_to_wr_bio(struct scrub_ctx *sctx,
1623                                     struct scrub_page *spage)
1624 {
1625         struct scrub_bio *sbio;
1626         int ret;
1627
1628         mutex_lock(&sctx->wr_lock);
1629 again:
1630         if (!sctx->wr_curr_bio) {
1631                 sctx->wr_curr_bio = kzalloc(sizeof(*sctx->wr_curr_bio),
1632                                               GFP_KERNEL);
1633                 if (!sctx->wr_curr_bio) {
1634                         mutex_unlock(&sctx->wr_lock);
1635                         return -ENOMEM;
1636                 }
1637                 sctx->wr_curr_bio->sctx = sctx;
1638                 sctx->wr_curr_bio->page_count = 0;
1639         }
1640         sbio = sctx->wr_curr_bio;
1641         if (sbio->page_count == 0) {
1642                 struct bio *bio;
1643
1644                 sbio->physical = spage->physical_for_dev_replace;
1645                 sbio->logical = spage->logical;
1646                 sbio->dev = sctx->wr_tgtdev;
1647                 bio = sbio->bio;
1648                 if (!bio) {
1649                         bio = btrfs_io_bio_alloc(sctx->pages_per_wr_bio);
1650                         sbio->bio = bio;
1651                 }
1652
1653                 bio->bi_private = sbio;
1654                 bio->bi_end_io = scrub_wr_bio_end_io;
1655                 bio_set_dev(bio, sbio->dev->bdev);
1656                 bio->bi_iter.bi_sector = sbio->physical >> 9;
1657                 bio->bi_opf = REQ_OP_WRITE;
1658                 sbio->status = 0;
1659         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
1660                    spage->physical_for_dev_replace ||
1661                    sbio->logical + sbio->page_count * PAGE_SIZE !=
1662                    spage->logical) {
1663                 scrub_wr_submit(sctx);
1664                 goto again;
1665         }
1666
1667         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
1668         if (ret != PAGE_SIZE) {
1669                 if (sbio->page_count < 1) {
1670                         bio_put(sbio->bio);
1671                         sbio->bio = NULL;
1672                         mutex_unlock(&sctx->wr_lock);
1673                         return -EIO;
1674                 }
1675                 scrub_wr_submit(sctx);
1676                 goto again;
1677         }
1678
1679         sbio->pagev[sbio->page_count] = spage;
1680         scrub_page_get(spage);
1681         sbio->page_count++;
1682         if (sbio->page_count == sctx->pages_per_wr_bio)
1683                 scrub_wr_submit(sctx);
1684         mutex_unlock(&sctx->wr_lock);
1685
1686         return 0;
1687 }
1688
1689 static void scrub_wr_submit(struct scrub_ctx *sctx)
1690 {
1691         struct scrub_bio *sbio;
1692
1693         if (!sctx->wr_curr_bio)
1694                 return;
1695
1696         sbio = sctx->wr_curr_bio;
1697         sctx->wr_curr_bio = NULL;
1698         WARN_ON(!sbio->bio->bi_disk);
1699         scrub_pending_bio_inc(sctx);
1700         /* process all writes in a single worker thread. Then the block layer
1701          * orders the requests before sending them to the driver which
1702          * doubled the write performance on spinning disks when measured
1703          * with Linux 3.5 */
1704         btrfsic_submit_bio(sbio->bio);
1705 }
1706
1707 static void scrub_wr_bio_end_io(struct bio *bio)
1708 {
1709         struct scrub_bio *sbio = bio->bi_private;
1710         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
1711
1712         sbio->status = bio->bi_status;
1713         sbio->bio = bio;
1714
1715         btrfs_init_work(&sbio->work, scrub_wr_bio_end_io_worker, NULL, NULL);
1716         btrfs_queue_work(fs_info->scrub_wr_completion_workers, &sbio->work);
1717 }
1718
1719 static void scrub_wr_bio_end_io_worker(struct btrfs_work *work)
1720 {
1721         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
1722         struct scrub_ctx *sctx = sbio->sctx;
1723         int i;
1724
1725         WARN_ON(sbio->page_count > SCRUB_PAGES_PER_WR_BIO);
1726         if (sbio->status) {
1727                 struct btrfs_dev_replace *dev_replace =
1728                         &sbio->sctx->fs_info->dev_replace;
1729
1730                 for (i = 0; i < sbio->page_count; i++) {
1731                         struct scrub_page *spage = sbio->pagev[i];
1732
1733                         spage->io_error = 1;
1734                         atomic64_inc(&dev_replace->num_write_errors);
1735                 }
1736         }
1737
1738         for (i = 0; i < sbio->page_count; i++)
1739                 scrub_page_put(sbio->pagev[i]);
1740
1741         bio_put(sbio->bio);
1742         kfree(sbio);
1743         scrub_pending_bio_dec(sctx);
1744 }
1745
1746 static int scrub_checksum(struct scrub_block *sblock)
1747 {
1748         u64 flags;
1749         int ret;
1750
1751         /*
1752          * No need to initialize these stats currently,
1753          * because this function only use return value
1754          * instead of these stats value.
1755          *
1756          * Todo:
1757          * always use stats
1758          */
1759         sblock->header_error = 0;
1760         sblock->generation_error = 0;
1761         sblock->checksum_error = 0;
1762
1763         WARN_ON(sblock->page_count < 1);
1764         flags = sblock->pagev[0]->flags;
1765         ret = 0;
1766         if (flags & BTRFS_EXTENT_FLAG_DATA)
1767                 ret = scrub_checksum_data(sblock);
1768         else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK)
1769                 ret = scrub_checksum_tree_block(sblock);
1770         else if (flags & BTRFS_EXTENT_FLAG_SUPER)
1771                 (void)scrub_checksum_super(sblock);
1772         else
1773                 WARN_ON(1);
1774         if (ret)
1775                 scrub_handle_errored_block(sblock);
1776
1777         return ret;
1778 }
1779
1780 static int scrub_checksum_data(struct scrub_block *sblock)
1781 {
1782         struct scrub_ctx *sctx = sblock->sctx;
1783         struct btrfs_fs_info *fs_info = sctx->fs_info;
1784         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1785         u8 csum[BTRFS_CSUM_SIZE];
1786         struct scrub_page *spage;
1787         char *kaddr;
1788
1789         BUG_ON(sblock->page_count < 1);
1790         spage = sblock->pagev[0];
1791         if (!spage->have_csum)
1792                 return 0;
1793
1794         kaddr = page_address(spage->page);
1795
1796         shash->tfm = fs_info->csum_shash;
1797         crypto_shash_init(shash);
1798
1799         /*
1800          * In scrub_pages() and scrub_pages_for_parity() we ensure each spage
1801          * only contains one sector of data.
1802          */
1803         crypto_shash_digest(shash, kaddr, fs_info->sectorsize, csum);
1804
1805         if (memcmp(csum, spage->csum, fs_info->csum_size))
1806                 sblock->checksum_error = 1;
1807         return sblock->checksum_error;
1808 }
1809
1810 static int scrub_checksum_tree_block(struct scrub_block *sblock)
1811 {
1812         struct scrub_ctx *sctx = sblock->sctx;
1813         struct btrfs_header *h;
1814         struct btrfs_fs_info *fs_info = sctx->fs_info;
1815         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1816         u8 calculated_csum[BTRFS_CSUM_SIZE];
1817         u8 on_disk_csum[BTRFS_CSUM_SIZE];
1818         /*
1819          * This is done in sectorsize steps even for metadata as there's a
1820          * constraint for nodesize to be aligned to sectorsize. This will need
1821          * to change so we don't misuse data and metadata units like that.
1822          */
1823         const u32 sectorsize = sctx->fs_info->sectorsize;
1824         const int num_sectors = fs_info->nodesize >> fs_info->sectorsize_bits;
1825         int i;
1826         struct scrub_page *spage;
1827         char *kaddr;
1828
1829         BUG_ON(sblock->page_count < 1);
1830
1831         /* Each member in pagev is just one block, not a full page */
1832         ASSERT(sblock->page_count == num_sectors);
1833
1834         spage = sblock->pagev[0];
1835         kaddr = page_address(spage->page);
1836         h = (struct btrfs_header *)kaddr;
1837         memcpy(on_disk_csum, h->csum, sctx->fs_info->csum_size);
1838
1839         /*
1840          * we don't use the getter functions here, as we
1841          * a) don't have an extent buffer and
1842          * b) the page is already kmapped
1843          */
1844         if (spage->logical != btrfs_stack_header_bytenr(h))
1845                 sblock->header_error = 1;
1846
1847         if (spage->generation != btrfs_stack_header_generation(h)) {
1848                 sblock->header_error = 1;
1849                 sblock->generation_error = 1;
1850         }
1851
1852         if (!scrub_check_fsid(h->fsid, spage))
1853                 sblock->header_error = 1;
1854
1855         if (memcmp(h->chunk_tree_uuid, fs_info->chunk_tree_uuid,
1856                    BTRFS_UUID_SIZE))
1857                 sblock->header_error = 1;
1858
1859         shash->tfm = fs_info->csum_shash;
1860         crypto_shash_init(shash);
1861         crypto_shash_update(shash, kaddr + BTRFS_CSUM_SIZE,
1862                             sectorsize - BTRFS_CSUM_SIZE);
1863
1864         for (i = 1; i < num_sectors; i++) {
1865                 kaddr = page_address(sblock->pagev[i]->page);
1866                 crypto_shash_update(shash, kaddr, sectorsize);
1867         }
1868
1869         crypto_shash_final(shash, calculated_csum);
1870         if (memcmp(calculated_csum, on_disk_csum, sctx->fs_info->csum_size))
1871                 sblock->checksum_error = 1;
1872
1873         return sblock->header_error || sblock->checksum_error;
1874 }
1875
1876 static int scrub_checksum_super(struct scrub_block *sblock)
1877 {
1878         struct btrfs_super_block *s;
1879         struct scrub_ctx *sctx = sblock->sctx;
1880         struct btrfs_fs_info *fs_info = sctx->fs_info;
1881         SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
1882         u8 calculated_csum[BTRFS_CSUM_SIZE];
1883         struct scrub_page *spage;
1884         char *kaddr;
1885         int fail_gen = 0;
1886         int fail_cor = 0;
1887
1888         BUG_ON(sblock->page_count < 1);
1889         spage = sblock->pagev[0];
1890         kaddr = page_address(spage->page);
1891         s = (struct btrfs_super_block *)kaddr;
1892
1893         if (spage->logical != btrfs_super_bytenr(s))
1894                 ++fail_cor;
1895
1896         if (spage->generation != btrfs_super_generation(s))
1897                 ++fail_gen;
1898
1899         if (!scrub_check_fsid(s->fsid, spage))
1900                 ++fail_cor;
1901
1902         shash->tfm = fs_info->csum_shash;
1903         crypto_shash_init(shash);
1904         crypto_shash_digest(shash, kaddr + BTRFS_CSUM_SIZE,
1905                         BTRFS_SUPER_INFO_SIZE - BTRFS_CSUM_SIZE, calculated_csum);
1906
1907         if (memcmp(calculated_csum, s->csum, sctx->fs_info->csum_size))
1908                 ++fail_cor;
1909
1910         if (fail_cor + fail_gen) {
1911                 /*
1912                  * if we find an error in a super block, we just report it.
1913                  * They will get written with the next transaction commit
1914                  * anyway
1915                  */
1916                 spin_lock(&sctx->stat_lock);
1917                 ++sctx->stat.super_errors;
1918                 spin_unlock(&sctx->stat_lock);
1919                 if (fail_cor)
1920                         btrfs_dev_stat_inc_and_print(spage->dev,
1921                                 BTRFS_DEV_STAT_CORRUPTION_ERRS);
1922                 else
1923                         btrfs_dev_stat_inc_and_print(spage->dev,
1924                                 BTRFS_DEV_STAT_GENERATION_ERRS);
1925         }
1926
1927         return fail_cor + fail_gen;
1928 }
1929
1930 static void scrub_block_get(struct scrub_block *sblock)
1931 {
1932         refcount_inc(&sblock->refs);
1933 }
1934
1935 static void scrub_block_put(struct scrub_block *sblock)
1936 {
1937         if (refcount_dec_and_test(&sblock->refs)) {
1938                 int i;
1939
1940                 if (sblock->sparity)
1941                         scrub_parity_put(sblock->sparity);
1942
1943                 for (i = 0; i < sblock->page_count; i++)
1944                         scrub_page_put(sblock->pagev[i]);
1945                 kfree(sblock);
1946         }
1947 }
1948
1949 static void scrub_page_get(struct scrub_page *spage)
1950 {
1951         atomic_inc(&spage->refs);
1952 }
1953
1954 static void scrub_page_put(struct scrub_page *spage)
1955 {
1956         if (atomic_dec_and_test(&spage->refs)) {
1957                 if (spage->page)
1958                         __free_page(spage->page);
1959                 kfree(spage);
1960         }
1961 }
1962
1963 static void scrub_submit(struct scrub_ctx *sctx)
1964 {
1965         struct scrub_bio *sbio;
1966
1967         if (sctx->curr == -1)
1968                 return;
1969
1970         sbio = sctx->bios[sctx->curr];
1971         sctx->curr = -1;
1972         scrub_pending_bio_inc(sctx);
1973         btrfsic_submit_bio(sbio->bio);
1974 }
1975
1976 static int scrub_add_page_to_rd_bio(struct scrub_ctx *sctx,
1977                                     struct scrub_page *spage)
1978 {
1979         struct scrub_block *sblock = spage->sblock;
1980         struct scrub_bio *sbio;
1981         int ret;
1982
1983 again:
1984         /*
1985          * grab a fresh bio or wait for one to become available
1986          */
1987         while (sctx->curr == -1) {
1988                 spin_lock(&sctx->list_lock);
1989                 sctx->curr = sctx->first_free;
1990                 if (sctx->curr != -1) {
1991                         sctx->first_free = sctx->bios[sctx->curr]->next_free;
1992                         sctx->bios[sctx->curr]->next_free = -1;
1993                         sctx->bios[sctx->curr]->page_count = 0;
1994                         spin_unlock(&sctx->list_lock);
1995                 } else {
1996                         spin_unlock(&sctx->list_lock);
1997                         wait_event(sctx->list_wait, sctx->first_free != -1);
1998                 }
1999         }
2000         sbio = sctx->bios[sctx->curr];
2001         if (sbio->page_count == 0) {
2002                 struct bio *bio;
2003
2004                 sbio->physical = spage->physical;
2005                 sbio->logical = spage->logical;
2006                 sbio->dev = spage->dev;
2007                 bio = sbio->bio;
2008                 if (!bio) {
2009                         bio = btrfs_io_bio_alloc(sctx->pages_per_rd_bio);
2010                         sbio->bio = bio;
2011                 }
2012
2013                 bio->bi_private = sbio;
2014                 bio->bi_end_io = scrub_bio_end_io;
2015                 bio_set_dev(bio, sbio->dev->bdev);
2016                 bio->bi_iter.bi_sector = sbio->physical >> 9;
2017                 bio->bi_opf = REQ_OP_READ;
2018                 sbio->status = 0;
2019         } else if (sbio->physical + sbio->page_count * PAGE_SIZE !=
2020                    spage->physical ||
2021                    sbio->logical + sbio->page_count * PAGE_SIZE !=
2022                    spage->logical ||
2023                    sbio->dev != spage->dev) {
2024                 scrub_submit(sctx);
2025                 goto again;
2026         }
2027
2028         sbio->pagev[sbio->page_count] = spage;
2029         ret = bio_add_page(sbio->bio, spage->page, PAGE_SIZE, 0);
2030         if (ret != PAGE_SIZE) {
2031                 if (sbio->page_count < 1) {
2032                         bio_put(sbio->bio);
2033                         sbio->bio = NULL;
2034                         return -EIO;
2035                 }
2036                 scrub_submit(sctx);
2037                 goto again;
2038         }
2039
2040         scrub_block_get(sblock); /* one for the page added to the bio */
2041         atomic_inc(&sblock->outstanding_pages);
2042         sbio->page_count++;
2043         if (sbio->page_count == sctx->pages_per_rd_bio)
2044                 scrub_submit(sctx);
2045
2046         return 0;
2047 }
2048
2049 static void scrub_missing_raid56_end_io(struct bio *bio)
2050 {
2051         struct scrub_block *sblock = bio->bi_private;
2052         struct btrfs_fs_info *fs_info = sblock->sctx->fs_info;
2053
2054         if (bio->bi_status)
2055                 sblock->no_io_error_seen = 0;
2056
2057         bio_put(bio);
2058
2059         btrfs_queue_work(fs_info->scrub_workers, &sblock->work);
2060 }
2061
2062 static void scrub_missing_raid56_worker(struct btrfs_work *work)
2063 {
2064         struct scrub_block *sblock = container_of(work, struct scrub_block, work);
2065         struct scrub_ctx *sctx = sblock->sctx;
2066         struct btrfs_fs_info *fs_info = sctx->fs_info;
2067         u64 logical;
2068         struct btrfs_device *dev;
2069
2070         logical = sblock->pagev[0]->logical;
2071         dev = sblock->pagev[0]->dev;
2072
2073         if (sblock->no_io_error_seen)
2074                 scrub_recheck_block_checksum(sblock);
2075
2076         if (!sblock->no_io_error_seen) {
2077                 spin_lock(&sctx->stat_lock);
2078                 sctx->stat.read_errors++;
2079                 spin_unlock(&sctx->stat_lock);
2080                 btrfs_err_rl_in_rcu(fs_info,
2081                         "IO error rebuilding logical %llu for dev %s",
2082                         logical, rcu_str_deref(dev->name));
2083         } else if (sblock->header_error || sblock->checksum_error) {
2084                 spin_lock(&sctx->stat_lock);
2085                 sctx->stat.uncorrectable_errors++;
2086                 spin_unlock(&sctx->stat_lock);
2087                 btrfs_err_rl_in_rcu(fs_info,
2088                         "failed to rebuild valid logical %llu for dev %s",
2089                         logical, rcu_str_deref(dev->name));
2090         } else {
2091                 scrub_write_block_to_dev_replace(sblock);
2092         }
2093
2094         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2095                 mutex_lock(&sctx->wr_lock);
2096                 scrub_wr_submit(sctx);
2097                 mutex_unlock(&sctx->wr_lock);
2098         }
2099
2100         scrub_block_put(sblock);
2101         scrub_pending_bio_dec(sctx);
2102 }
2103
2104 static void scrub_missing_raid56_pages(struct scrub_block *sblock)
2105 {
2106         struct scrub_ctx *sctx = sblock->sctx;
2107         struct btrfs_fs_info *fs_info = sctx->fs_info;
2108         u64 length = sblock->page_count * PAGE_SIZE;
2109         u64 logical = sblock->pagev[0]->logical;
2110         struct btrfs_bio *bbio = NULL;
2111         struct bio *bio;
2112         struct btrfs_raid_bio *rbio;
2113         int ret;
2114         int i;
2115
2116         btrfs_bio_counter_inc_blocked(fs_info);
2117         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_GET_READ_MIRRORS, logical,
2118                         &length, &bbio);
2119         if (ret || !bbio || !bbio->raid_map)
2120                 goto bbio_out;
2121
2122         if (WARN_ON(!sctx->is_dev_replace ||
2123                     !(bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK))) {
2124                 /*
2125                  * We shouldn't be scrubbing a missing device. Even for dev
2126                  * replace, we should only get here for RAID 5/6. We either
2127                  * managed to mount something with no mirrors remaining or
2128                  * there's a bug in scrub_remap_extent()/btrfs_map_block().
2129                  */
2130                 goto bbio_out;
2131         }
2132
2133         bio = btrfs_io_bio_alloc(0);
2134         bio->bi_iter.bi_sector = logical >> 9;
2135         bio->bi_private = sblock;
2136         bio->bi_end_io = scrub_missing_raid56_end_io;
2137
2138         rbio = raid56_alloc_missing_rbio(fs_info, bio, bbio, length);
2139         if (!rbio)
2140                 goto rbio_out;
2141
2142         for (i = 0; i < sblock->page_count; i++) {
2143                 struct scrub_page *spage = sblock->pagev[i];
2144
2145                 raid56_add_scrub_pages(rbio, spage->page, spage->logical);
2146         }
2147
2148         btrfs_init_work(&sblock->work, scrub_missing_raid56_worker, NULL, NULL);
2149         scrub_block_get(sblock);
2150         scrub_pending_bio_inc(sctx);
2151         raid56_submit_missing_rbio(rbio);
2152         return;
2153
2154 rbio_out:
2155         bio_put(bio);
2156 bbio_out:
2157         btrfs_bio_counter_dec(fs_info);
2158         btrfs_put_bbio(bbio);
2159         spin_lock(&sctx->stat_lock);
2160         sctx->stat.malloc_errors++;
2161         spin_unlock(&sctx->stat_lock);
2162 }
2163
2164 static int scrub_pages(struct scrub_ctx *sctx, u64 logical, u32 len,
2165                        u64 physical, struct btrfs_device *dev, u64 flags,
2166                        u64 gen, int mirror_num, u8 *csum,
2167                        u64 physical_for_dev_replace)
2168 {
2169         struct scrub_block *sblock;
2170         const u32 sectorsize = sctx->fs_info->sectorsize;
2171         int index;
2172
2173         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2174         if (!sblock) {
2175                 spin_lock(&sctx->stat_lock);
2176                 sctx->stat.malloc_errors++;
2177                 spin_unlock(&sctx->stat_lock);
2178                 return -ENOMEM;
2179         }
2180
2181         /* one ref inside this function, plus one for each page added to
2182          * a bio later on */
2183         refcount_set(&sblock->refs, 1);
2184         sblock->sctx = sctx;
2185         sblock->no_io_error_seen = 1;
2186
2187         for (index = 0; len > 0; index++) {
2188                 struct scrub_page *spage;
2189                 /*
2190                  * Here we will allocate one page for one sector to scrub.
2191                  * This is fine if PAGE_SIZE == sectorsize, but will cost
2192                  * more memory for PAGE_SIZE > sectorsize case.
2193                  */
2194                 u32 l = min(sectorsize, len);
2195
2196                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2197                 if (!spage) {
2198 leave_nomem:
2199                         spin_lock(&sctx->stat_lock);
2200                         sctx->stat.malloc_errors++;
2201                         spin_unlock(&sctx->stat_lock);
2202                         scrub_block_put(sblock);
2203                         return -ENOMEM;
2204                 }
2205                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2206                 scrub_page_get(spage);
2207                 sblock->pagev[index] = spage;
2208                 spage->sblock = sblock;
2209                 spage->dev = dev;
2210                 spage->flags = flags;
2211                 spage->generation = gen;
2212                 spage->logical = logical;
2213                 spage->physical = physical;
2214                 spage->physical_for_dev_replace = physical_for_dev_replace;
2215                 spage->mirror_num = mirror_num;
2216                 if (csum) {
2217                         spage->have_csum = 1;
2218                         memcpy(spage->csum, csum, sctx->fs_info->csum_size);
2219                 } else {
2220                         spage->have_csum = 0;
2221                 }
2222                 sblock->page_count++;
2223                 spage->page = alloc_page(GFP_KERNEL);
2224                 if (!spage->page)
2225                         goto leave_nomem;
2226                 len -= l;
2227                 logical += l;
2228                 physical += l;
2229                 physical_for_dev_replace += l;
2230         }
2231
2232         WARN_ON(sblock->page_count == 0);
2233         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2234                 /*
2235                  * This case should only be hit for RAID 5/6 device replace. See
2236                  * the comment in scrub_missing_raid56_pages() for details.
2237                  */
2238                 scrub_missing_raid56_pages(sblock);
2239         } else {
2240                 for (index = 0; index < sblock->page_count; index++) {
2241                         struct scrub_page *spage = sblock->pagev[index];
2242                         int ret;
2243
2244                         ret = scrub_add_page_to_rd_bio(sctx, spage);
2245                         if (ret) {
2246                                 scrub_block_put(sblock);
2247                                 return ret;
2248                         }
2249                 }
2250
2251                 if (flags & BTRFS_EXTENT_FLAG_SUPER)
2252                         scrub_submit(sctx);
2253         }
2254
2255         /* last one frees, either here or in bio completion for last page */
2256         scrub_block_put(sblock);
2257         return 0;
2258 }
2259
2260 static void scrub_bio_end_io(struct bio *bio)
2261 {
2262         struct scrub_bio *sbio = bio->bi_private;
2263         struct btrfs_fs_info *fs_info = sbio->dev->fs_info;
2264
2265         sbio->status = bio->bi_status;
2266         sbio->bio = bio;
2267
2268         btrfs_queue_work(fs_info->scrub_workers, &sbio->work);
2269 }
2270
2271 static void scrub_bio_end_io_worker(struct btrfs_work *work)
2272 {
2273         struct scrub_bio *sbio = container_of(work, struct scrub_bio, work);
2274         struct scrub_ctx *sctx = sbio->sctx;
2275         int i;
2276
2277         BUG_ON(sbio->page_count > SCRUB_PAGES_PER_RD_BIO);
2278         if (sbio->status) {
2279                 for (i = 0; i < sbio->page_count; i++) {
2280                         struct scrub_page *spage = sbio->pagev[i];
2281
2282                         spage->io_error = 1;
2283                         spage->sblock->no_io_error_seen = 0;
2284                 }
2285         }
2286
2287         /* now complete the scrub_block items that have all pages completed */
2288         for (i = 0; i < sbio->page_count; i++) {
2289                 struct scrub_page *spage = sbio->pagev[i];
2290                 struct scrub_block *sblock = spage->sblock;
2291
2292                 if (atomic_dec_and_test(&sblock->outstanding_pages))
2293                         scrub_block_complete(sblock);
2294                 scrub_block_put(sblock);
2295         }
2296
2297         bio_put(sbio->bio);
2298         sbio->bio = NULL;
2299         spin_lock(&sctx->list_lock);
2300         sbio->next_free = sctx->first_free;
2301         sctx->first_free = sbio->index;
2302         spin_unlock(&sctx->list_lock);
2303
2304         if (sctx->is_dev_replace && sctx->flush_all_writes) {
2305                 mutex_lock(&sctx->wr_lock);
2306                 scrub_wr_submit(sctx);
2307                 mutex_unlock(&sctx->wr_lock);
2308         }
2309
2310         scrub_pending_bio_dec(sctx);
2311 }
2312
2313 static inline void __scrub_mark_bitmap(struct scrub_parity *sparity,
2314                                        unsigned long *bitmap,
2315                                        u64 start, u32 len)
2316 {
2317         u64 offset;
2318         u32 nsectors;
2319         u32 sectorsize_bits = sparity->sctx->fs_info->sectorsize_bits;
2320
2321         if (len >= sparity->stripe_len) {
2322                 bitmap_set(bitmap, 0, sparity->nsectors);
2323                 return;
2324         }
2325
2326         start -= sparity->logic_start;
2327         start = div64_u64_rem(start, sparity->stripe_len, &offset);
2328         offset = offset >> sectorsize_bits;
2329         nsectors = len >> sectorsize_bits;
2330
2331         if (offset + nsectors <= sparity->nsectors) {
2332                 bitmap_set(bitmap, offset, nsectors);
2333                 return;
2334         }
2335
2336         bitmap_set(bitmap, offset, sparity->nsectors - offset);
2337         bitmap_set(bitmap, 0, nsectors - (sparity->nsectors - offset));
2338 }
2339
2340 static inline void scrub_parity_mark_sectors_error(struct scrub_parity *sparity,
2341                                                    u64 start, u32 len)
2342 {
2343         __scrub_mark_bitmap(sparity, sparity->ebitmap, start, len);
2344 }
2345
2346 static inline void scrub_parity_mark_sectors_data(struct scrub_parity *sparity,
2347                                                   u64 start, u32 len)
2348 {
2349         __scrub_mark_bitmap(sparity, sparity->dbitmap, start, len);
2350 }
2351
2352 static void scrub_block_complete(struct scrub_block *sblock)
2353 {
2354         int corrupted = 0;
2355
2356         if (!sblock->no_io_error_seen) {
2357                 corrupted = 1;
2358                 scrub_handle_errored_block(sblock);
2359         } else {
2360                 /*
2361                  * if has checksum error, write via repair mechanism in
2362                  * dev replace case, otherwise write here in dev replace
2363                  * case.
2364                  */
2365                 corrupted = scrub_checksum(sblock);
2366                 if (!corrupted && sblock->sctx->is_dev_replace)
2367                         scrub_write_block_to_dev_replace(sblock);
2368         }
2369
2370         if (sblock->sparity && corrupted && !sblock->data_corrected) {
2371                 u64 start = sblock->pagev[0]->logical;
2372                 u64 end = sblock->pagev[sblock->page_count - 1]->logical +
2373                           PAGE_SIZE;
2374
2375                 ASSERT(end - start <= U32_MAX);
2376                 scrub_parity_mark_sectors_error(sblock->sparity,
2377                                                 start, end - start);
2378         }
2379 }
2380
2381 static void drop_csum_range(struct scrub_ctx *sctx, struct btrfs_ordered_sum *sum)
2382 {
2383         sctx->stat.csum_discards += sum->len >> sctx->fs_info->sectorsize_bits;
2384         list_del(&sum->list);
2385         kfree(sum);
2386 }
2387
2388 /*
2389  * Find the desired csum for range [logical, logical + sectorsize), and store
2390  * the csum into @csum.
2391  *
2392  * The search source is sctx->csum_list, which is a pre-populated list
2393  * storing bytenr ordered csum ranges.  We're reponsible to cleanup any range
2394  * that is before @logical.
2395  *
2396  * Return 0 if there is no csum for the range.
2397  * Return 1 if there is csum for the range and copied to @csum.
2398  */
2399 static int scrub_find_csum(struct scrub_ctx *sctx, u64 logical, u8 *csum)
2400 {
2401         bool found = false;
2402
2403         while (!list_empty(&sctx->csum_list)) {
2404                 struct btrfs_ordered_sum *sum = NULL;
2405                 unsigned long index;
2406                 unsigned long num_sectors;
2407
2408                 sum = list_first_entry(&sctx->csum_list,
2409                                        struct btrfs_ordered_sum, list);
2410                 /* The current csum range is beyond our range, no csum found */
2411                 if (sum->bytenr > logical)
2412                         break;
2413
2414                 /*
2415                  * The current sum is before our bytenr, since scrub is always
2416                  * done in bytenr order, the csum will never be used anymore,
2417                  * clean it up so that later calls won't bother with the range,
2418                  * and continue search the next range.
2419                  */
2420                 if (sum->bytenr + sum->len <= logical) {
2421                         drop_csum_range(sctx, sum);
2422                         continue;
2423                 }
2424
2425                 /* Now the csum range covers our bytenr, copy the csum */
2426                 found = true;
2427                 index = (logical - sum->bytenr) >> sctx->fs_info->sectorsize_bits;
2428                 num_sectors = sum->len >> sctx->fs_info->sectorsize_bits;
2429
2430                 memcpy(csum, sum->sums + index * sctx->fs_info->csum_size,
2431                        sctx->fs_info->csum_size);
2432
2433                 /* Cleanup the range if we're at the end of the csum range */
2434                 if (index == num_sectors - 1)
2435                         drop_csum_range(sctx, sum);
2436                 break;
2437         }
2438         if (!found)
2439                 return 0;
2440         return 1;
2441 }
2442
2443 /* scrub extent tries to collect up to 64 kB for each bio */
2444 static int scrub_extent(struct scrub_ctx *sctx, struct map_lookup *map,
2445                         u64 logical, u32 len,
2446                         u64 physical, struct btrfs_device *dev, u64 flags,
2447                         u64 gen, int mirror_num, u64 physical_for_dev_replace)
2448 {
2449         int ret;
2450         u8 csum[BTRFS_CSUM_SIZE];
2451         u32 blocksize;
2452
2453         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2454                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2455                         blocksize = map->stripe_len;
2456                 else
2457                         blocksize = sctx->fs_info->sectorsize;
2458                 spin_lock(&sctx->stat_lock);
2459                 sctx->stat.data_extents_scrubbed++;
2460                 sctx->stat.data_bytes_scrubbed += len;
2461                 spin_unlock(&sctx->stat_lock);
2462         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2463                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
2464                         blocksize = map->stripe_len;
2465                 else
2466                         blocksize = sctx->fs_info->nodesize;
2467                 spin_lock(&sctx->stat_lock);
2468                 sctx->stat.tree_extents_scrubbed++;
2469                 sctx->stat.tree_bytes_scrubbed += len;
2470                 spin_unlock(&sctx->stat_lock);
2471         } else {
2472                 blocksize = sctx->fs_info->sectorsize;
2473                 WARN_ON(1);
2474         }
2475
2476         while (len) {
2477                 u32 l = min(len, blocksize);
2478                 int have_csum = 0;
2479
2480                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2481                         /* push csums to sbio */
2482                         have_csum = scrub_find_csum(sctx, logical, csum);
2483                         if (have_csum == 0)
2484                                 ++sctx->stat.no_csum;
2485                 }
2486                 ret = scrub_pages(sctx, logical, l, physical, dev, flags, gen,
2487                                   mirror_num, have_csum ? csum : NULL,
2488                                   physical_for_dev_replace);
2489                 if (ret)
2490                         return ret;
2491                 len -= l;
2492                 logical += l;
2493                 physical += l;
2494                 physical_for_dev_replace += l;
2495         }
2496         return 0;
2497 }
2498
2499 static int scrub_pages_for_parity(struct scrub_parity *sparity,
2500                                   u64 logical, u32 len,
2501                                   u64 physical, struct btrfs_device *dev,
2502                                   u64 flags, u64 gen, int mirror_num, u8 *csum)
2503 {
2504         struct scrub_ctx *sctx = sparity->sctx;
2505         struct scrub_block *sblock;
2506         const u32 sectorsize = sctx->fs_info->sectorsize;
2507         int index;
2508
2509         ASSERT(IS_ALIGNED(len, sectorsize));
2510
2511         sblock = kzalloc(sizeof(*sblock), GFP_KERNEL);
2512         if (!sblock) {
2513                 spin_lock(&sctx->stat_lock);
2514                 sctx->stat.malloc_errors++;
2515                 spin_unlock(&sctx->stat_lock);
2516                 return -ENOMEM;
2517         }
2518
2519         /* one ref inside this function, plus one for each page added to
2520          * a bio later on */
2521         refcount_set(&sblock->refs, 1);
2522         sblock->sctx = sctx;
2523         sblock->no_io_error_seen = 1;
2524         sblock->sparity = sparity;
2525         scrub_parity_get(sparity);
2526
2527         for (index = 0; len > 0; index++) {
2528                 struct scrub_page *spage;
2529
2530                 spage = kzalloc(sizeof(*spage), GFP_KERNEL);
2531                 if (!spage) {
2532 leave_nomem:
2533                         spin_lock(&sctx->stat_lock);
2534                         sctx->stat.malloc_errors++;
2535                         spin_unlock(&sctx->stat_lock);
2536                         scrub_block_put(sblock);
2537                         return -ENOMEM;
2538                 }
2539                 BUG_ON(index >= SCRUB_MAX_PAGES_PER_BLOCK);
2540                 /* For scrub block */
2541                 scrub_page_get(spage);
2542                 sblock->pagev[index] = spage;
2543                 /* For scrub parity */
2544                 scrub_page_get(spage);
2545                 list_add_tail(&spage->list, &sparity->spages);
2546                 spage->sblock = sblock;
2547                 spage->dev = dev;
2548                 spage->flags = flags;
2549                 spage->generation = gen;
2550                 spage->logical = logical;
2551                 spage->physical = physical;
2552                 spage->mirror_num = mirror_num;
2553                 if (csum) {
2554                         spage->have_csum = 1;
2555                         memcpy(spage->csum, csum, sctx->fs_info->csum_size);
2556                 } else {
2557                         spage->have_csum = 0;
2558                 }
2559                 sblock->page_count++;
2560                 spage->page = alloc_page(GFP_KERNEL);
2561                 if (!spage->page)
2562                         goto leave_nomem;
2563
2564
2565                 /* Iterate over the stripe range in sectorsize steps */
2566                 len -= sectorsize;
2567                 logical += sectorsize;
2568                 physical += sectorsize;
2569         }
2570
2571         WARN_ON(sblock->page_count == 0);
2572         for (index = 0; index < sblock->page_count; index++) {
2573                 struct scrub_page *spage = sblock->pagev[index];
2574                 int ret;
2575
2576                 ret = scrub_add_page_to_rd_bio(sctx, spage);
2577                 if (ret) {
2578                         scrub_block_put(sblock);
2579                         return ret;
2580                 }
2581         }
2582
2583         /* last one frees, either here or in bio completion for last page */
2584         scrub_block_put(sblock);
2585         return 0;
2586 }
2587
2588 static int scrub_extent_for_parity(struct scrub_parity *sparity,
2589                                    u64 logical, u32 len,
2590                                    u64 physical, struct btrfs_device *dev,
2591                                    u64 flags, u64 gen, int mirror_num)
2592 {
2593         struct scrub_ctx *sctx = sparity->sctx;
2594         int ret;
2595         u8 csum[BTRFS_CSUM_SIZE];
2596         u32 blocksize;
2597
2598         if (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state)) {
2599                 scrub_parity_mark_sectors_error(sparity, logical, len);
2600                 return 0;
2601         }
2602
2603         if (flags & BTRFS_EXTENT_FLAG_DATA) {
2604                 blocksize = sparity->stripe_len;
2605         } else if (flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) {
2606                 blocksize = sparity->stripe_len;
2607         } else {
2608                 blocksize = sctx->fs_info->sectorsize;
2609                 WARN_ON(1);
2610         }
2611
2612         while (len) {
2613                 u32 l = min(len, blocksize);
2614                 int have_csum = 0;
2615
2616                 if (flags & BTRFS_EXTENT_FLAG_DATA) {
2617                         /* push csums to sbio */
2618                         have_csum = scrub_find_csum(sctx, logical, csum);
2619                         if (have_csum == 0)
2620                                 goto skip;
2621                 }
2622                 ret = scrub_pages_for_parity(sparity, logical, l, physical, dev,
2623                                              flags, gen, mirror_num,
2624                                              have_csum ? csum : NULL);
2625                 if (ret)
2626                         return ret;
2627 skip:
2628                 len -= l;
2629                 logical += l;
2630                 physical += l;
2631         }
2632         return 0;
2633 }
2634
2635 /*
2636  * Given a physical address, this will calculate it's
2637  * logical offset. if this is a parity stripe, it will return
2638  * the most left data stripe's logical offset.
2639  *
2640  * return 0 if it is a data stripe, 1 means parity stripe.
2641  */
2642 static int get_raid56_logic_offset(u64 physical, int num,
2643                                    struct map_lookup *map, u64 *offset,
2644                                    u64 *stripe_start)
2645 {
2646         int i;
2647         int j = 0;
2648         u64 stripe_nr;
2649         u64 last_offset;
2650         u32 stripe_index;
2651         u32 rot;
2652         const int data_stripes = nr_data_stripes(map);
2653
2654         last_offset = (physical - map->stripes[num].physical) * data_stripes;
2655         if (stripe_start)
2656                 *stripe_start = last_offset;
2657
2658         *offset = last_offset;
2659         for (i = 0; i < data_stripes; i++) {
2660                 *offset = last_offset + i * map->stripe_len;
2661
2662                 stripe_nr = div64_u64(*offset, map->stripe_len);
2663                 stripe_nr = div_u64(stripe_nr, data_stripes);
2664
2665                 /* Work out the disk rotation on this stripe-set */
2666                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes, &rot);
2667                 /* calculate which stripe this data locates */
2668                 rot += i;
2669                 stripe_index = rot % map->num_stripes;
2670                 if (stripe_index == num)
2671                         return 0;
2672                 if (stripe_index < num)
2673                         j++;
2674         }
2675         *offset = last_offset + j * map->stripe_len;
2676         return 1;
2677 }
2678
2679 static void scrub_free_parity(struct scrub_parity *sparity)
2680 {
2681         struct scrub_ctx *sctx = sparity->sctx;
2682         struct scrub_page *curr, *next;
2683         int nbits;
2684
2685         nbits = bitmap_weight(sparity->ebitmap, sparity->nsectors);
2686         if (nbits) {
2687                 spin_lock(&sctx->stat_lock);
2688                 sctx->stat.read_errors += nbits;
2689                 sctx->stat.uncorrectable_errors += nbits;
2690                 spin_unlock(&sctx->stat_lock);
2691         }
2692
2693         list_for_each_entry_safe(curr, next, &sparity->spages, list) {
2694                 list_del_init(&curr->list);
2695                 scrub_page_put(curr);
2696         }
2697
2698         kfree(sparity);
2699 }
2700
2701 static void scrub_parity_bio_endio_worker(struct btrfs_work *work)
2702 {
2703         struct scrub_parity *sparity = container_of(work, struct scrub_parity,
2704                                                     work);
2705         struct scrub_ctx *sctx = sparity->sctx;
2706
2707         scrub_free_parity(sparity);
2708         scrub_pending_bio_dec(sctx);
2709 }
2710
2711 static void scrub_parity_bio_endio(struct bio *bio)
2712 {
2713         struct scrub_parity *sparity = (struct scrub_parity *)bio->bi_private;
2714         struct btrfs_fs_info *fs_info = sparity->sctx->fs_info;
2715
2716         if (bio->bi_status)
2717                 bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2718                           sparity->nsectors);
2719
2720         bio_put(bio);
2721
2722         btrfs_init_work(&sparity->work, scrub_parity_bio_endio_worker, NULL,
2723                         NULL);
2724         btrfs_queue_work(fs_info->scrub_parity_workers, &sparity->work);
2725 }
2726
2727 static void scrub_parity_check_and_repair(struct scrub_parity *sparity)
2728 {
2729         struct scrub_ctx *sctx = sparity->sctx;
2730         struct btrfs_fs_info *fs_info = sctx->fs_info;
2731         struct bio *bio;
2732         struct btrfs_raid_bio *rbio;
2733         struct btrfs_bio *bbio = NULL;
2734         u64 length;
2735         int ret;
2736
2737         if (!bitmap_andnot(sparity->dbitmap, sparity->dbitmap, sparity->ebitmap,
2738                            sparity->nsectors))
2739                 goto out;
2740
2741         length = sparity->logic_end - sparity->logic_start;
2742
2743         btrfs_bio_counter_inc_blocked(fs_info);
2744         ret = btrfs_map_sblock(fs_info, BTRFS_MAP_WRITE, sparity->logic_start,
2745                                &length, &bbio);
2746         if (ret || !bbio || !bbio->raid_map)
2747                 goto bbio_out;
2748
2749         bio = btrfs_io_bio_alloc(0);
2750         bio->bi_iter.bi_sector = sparity->logic_start >> 9;
2751         bio->bi_private = sparity;
2752         bio->bi_end_io = scrub_parity_bio_endio;
2753
2754         rbio = raid56_parity_alloc_scrub_rbio(fs_info, bio, bbio,
2755                                               length, sparity->scrub_dev,
2756                                               sparity->dbitmap,
2757                                               sparity->nsectors);
2758         if (!rbio)
2759                 goto rbio_out;
2760
2761         scrub_pending_bio_inc(sctx);
2762         raid56_parity_submit_scrub_rbio(rbio);
2763         return;
2764
2765 rbio_out:
2766         bio_put(bio);
2767 bbio_out:
2768         btrfs_bio_counter_dec(fs_info);
2769         btrfs_put_bbio(bbio);
2770         bitmap_or(sparity->ebitmap, sparity->ebitmap, sparity->dbitmap,
2771                   sparity->nsectors);
2772         spin_lock(&sctx->stat_lock);
2773         sctx->stat.malloc_errors++;
2774         spin_unlock(&sctx->stat_lock);
2775 out:
2776         scrub_free_parity(sparity);
2777 }
2778
2779 static inline int scrub_calc_parity_bitmap_len(int nsectors)
2780 {
2781         return DIV_ROUND_UP(nsectors, BITS_PER_LONG) * sizeof(long);
2782 }
2783
2784 static void scrub_parity_get(struct scrub_parity *sparity)
2785 {
2786         refcount_inc(&sparity->refs);
2787 }
2788
2789 static void scrub_parity_put(struct scrub_parity *sparity)
2790 {
2791         if (!refcount_dec_and_test(&sparity->refs))
2792                 return;
2793
2794         scrub_parity_check_and_repair(sparity);
2795 }
2796
2797 static noinline_for_stack int scrub_raid56_parity(struct scrub_ctx *sctx,
2798                                                   struct map_lookup *map,
2799                                                   struct btrfs_device *sdev,
2800                                                   struct btrfs_path *path,
2801                                                   u64 logic_start,
2802                                                   u64 logic_end)
2803 {
2804         struct btrfs_fs_info *fs_info = sctx->fs_info;
2805         struct btrfs_root *root = fs_info->extent_root;
2806         struct btrfs_root *csum_root = fs_info->csum_root;
2807         struct btrfs_extent_item *extent;
2808         struct btrfs_bio *bbio = NULL;
2809         u64 flags;
2810         int ret;
2811         int slot;
2812         struct extent_buffer *l;
2813         struct btrfs_key key;
2814         u64 generation;
2815         u64 extent_logical;
2816         u64 extent_physical;
2817         /* Check the comment in scrub_stripe() for why u32 is enough here */
2818         u32 extent_len;
2819         u64 mapped_length;
2820         struct btrfs_device *extent_dev;
2821         struct scrub_parity *sparity;
2822         int nsectors;
2823         int bitmap_len;
2824         int extent_mirror_num;
2825         int stop_loop = 0;
2826
2827         ASSERT(map->stripe_len <= U32_MAX);
2828         nsectors = map->stripe_len >> fs_info->sectorsize_bits;
2829         bitmap_len = scrub_calc_parity_bitmap_len(nsectors);
2830         sparity = kzalloc(sizeof(struct scrub_parity) + 2 * bitmap_len,
2831                           GFP_NOFS);
2832         if (!sparity) {
2833                 spin_lock(&sctx->stat_lock);
2834                 sctx->stat.malloc_errors++;
2835                 spin_unlock(&sctx->stat_lock);
2836                 return -ENOMEM;
2837         }
2838
2839         ASSERT(map->stripe_len <= U32_MAX);
2840         sparity->stripe_len = map->stripe_len;
2841         sparity->nsectors = nsectors;
2842         sparity->sctx = sctx;
2843         sparity->scrub_dev = sdev;
2844         sparity->logic_start = logic_start;
2845         sparity->logic_end = logic_end;
2846         refcount_set(&sparity->refs, 1);
2847         INIT_LIST_HEAD(&sparity->spages);
2848         sparity->dbitmap = sparity->bitmap;
2849         sparity->ebitmap = (void *)sparity->bitmap + bitmap_len;
2850
2851         ret = 0;
2852         while (logic_start < logic_end) {
2853                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
2854                         key.type = BTRFS_METADATA_ITEM_KEY;
2855                 else
2856                         key.type = BTRFS_EXTENT_ITEM_KEY;
2857                 key.objectid = logic_start;
2858                 key.offset = (u64)-1;
2859
2860                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2861                 if (ret < 0)
2862                         goto out;
2863
2864                 if (ret > 0) {
2865                         ret = btrfs_previous_extent_item(root, path, 0);
2866                         if (ret < 0)
2867                                 goto out;
2868                         if (ret > 0) {
2869                                 btrfs_release_path(path);
2870                                 ret = btrfs_search_slot(NULL, root, &key,
2871                                                         path, 0, 0);
2872                                 if (ret < 0)
2873                                         goto out;
2874                         }
2875                 }
2876
2877                 stop_loop = 0;
2878                 while (1) {
2879                         u64 bytes;
2880
2881                         l = path->nodes[0];
2882                         slot = path->slots[0];
2883                         if (slot >= btrfs_header_nritems(l)) {
2884                                 ret = btrfs_next_leaf(root, path);
2885                                 if (ret == 0)
2886                                         continue;
2887                                 if (ret < 0)
2888                                         goto out;
2889
2890                                 stop_loop = 1;
2891                                 break;
2892                         }
2893                         btrfs_item_key_to_cpu(l, &key, slot);
2894
2895                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
2896                             key.type != BTRFS_METADATA_ITEM_KEY)
2897                                 goto next;
2898
2899                         if (key.type == BTRFS_METADATA_ITEM_KEY)
2900                                 bytes = fs_info->nodesize;
2901                         else
2902                                 bytes = key.offset;
2903
2904                         if (key.objectid + bytes <= logic_start)
2905                                 goto next;
2906
2907                         if (key.objectid >= logic_end) {
2908                                 stop_loop = 1;
2909                                 break;
2910                         }
2911
2912                         while (key.objectid >= logic_start + map->stripe_len)
2913                                 logic_start += map->stripe_len;
2914
2915                         extent = btrfs_item_ptr(l, slot,
2916                                                 struct btrfs_extent_item);
2917                         flags = btrfs_extent_flags(l, extent);
2918                         generation = btrfs_extent_generation(l, extent);
2919
2920                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
2921                             (key.objectid < logic_start ||
2922                              key.objectid + bytes >
2923                              logic_start + map->stripe_len)) {
2924                                 btrfs_err(fs_info,
2925                                           "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
2926                                           key.objectid, logic_start);
2927                                 spin_lock(&sctx->stat_lock);
2928                                 sctx->stat.uncorrectable_errors++;
2929                                 spin_unlock(&sctx->stat_lock);
2930                                 goto next;
2931                         }
2932 again:
2933                         extent_logical = key.objectid;
2934                         ASSERT(bytes <= U32_MAX);
2935                         extent_len = bytes;
2936
2937                         if (extent_logical < logic_start) {
2938                                 extent_len -= logic_start - extent_logical;
2939                                 extent_logical = logic_start;
2940                         }
2941
2942                         if (extent_logical + extent_len >
2943                             logic_start + map->stripe_len)
2944                                 extent_len = logic_start + map->stripe_len -
2945                                              extent_logical;
2946
2947                         scrub_parity_mark_sectors_data(sparity, extent_logical,
2948                                                        extent_len);
2949
2950                         mapped_length = extent_len;
2951                         bbio = NULL;
2952                         ret = btrfs_map_block(fs_info, BTRFS_MAP_READ,
2953                                         extent_logical, &mapped_length, &bbio,
2954                                         0);
2955                         if (!ret) {
2956                                 if (!bbio || mapped_length < extent_len)
2957                                         ret = -EIO;
2958                         }
2959                         if (ret) {
2960                                 btrfs_put_bbio(bbio);
2961                                 goto out;
2962                         }
2963                         extent_physical = bbio->stripes[0].physical;
2964                         extent_mirror_num = bbio->mirror_num;
2965                         extent_dev = bbio->stripes[0].dev;
2966                         btrfs_put_bbio(bbio);
2967
2968                         ret = btrfs_lookup_csums_range(csum_root,
2969                                                 extent_logical,
2970                                                 extent_logical + extent_len - 1,
2971                                                 &sctx->csum_list, 1);
2972                         if (ret)
2973                                 goto out;
2974
2975                         ret = scrub_extent_for_parity(sparity, extent_logical,
2976                                                       extent_len,
2977                                                       extent_physical,
2978                                                       extent_dev, flags,
2979                                                       generation,
2980                                                       extent_mirror_num);
2981
2982                         scrub_free_csums(sctx);
2983
2984                         if (ret)
2985                                 goto out;
2986
2987                         if (extent_logical + extent_len <
2988                             key.objectid + bytes) {
2989                                 logic_start += map->stripe_len;
2990
2991                                 if (logic_start >= logic_end) {
2992                                         stop_loop = 1;
2993                                         break;
2994                                 }
2995
2996                                 if (logic_start < key.objectid + bytes) {
2997                                         cond_resched();
2998                                         goto again;
2999                                 }
3000                         }
3001 next:
3002                         path->slots[0]++;
3003                 }
3004
3005                 btrfs_release_path(path);
3006
3007                 if (stop_loop)
3008                         break;
3009
3010                 logic_start += map->stripe_len;
3011         }
3012 out:
3013         if (ret < 0) {
3014                 ASSERT(logic_end - logic_start <= U32_MAX);
3015                 scrub_parity_mark_sectors_error(sparity, logic_start,
3016                                                 logic_end - logic_start);
3017         }
3018         scrub_parity_put(sparity);
3019         scrub_submit(sctx);
3020         mutex_lock(&sctx->wr_lock);
3021         scrub_wr_submit(sctx);
3022         mutex_unlock(&sctx->wr_lock);
3023
3024         btrfs_release_path(path);
3025         return ret < 0 ? ret : 0;
3026 }
3027
3028 static noinline_for_stack int scrub_stripe(struct scrub_ctx *sctx,
3029                                            struct map_lookup *map,
3030                                            struct btrfs_device *scrub_dev,
3031                                            int num, u64 base, u64 length,
3032                                            struct btrfs_block_group *cache)
3033 {
3034         struct btrfs_path *path, *ppath;
3035         struct btrfs_fs_info *fs_info = sctx->fs_info;
3036         struct btrfs_root *root = fs_info->extent_root;
3037         struct btrfs_root *csum_root = fs_info->csum_root;
3038         struct btrfs_extent_item *extent;
3039         struct blk_plug plug;
3040         u64 flags;
3041         int ret;
3042         int slot;
3043         u64 nstripes;
3044         struct extent_buffer *l;
3045         u64 physical;
3046         u64 logical;
3047         u64 logic_end;
3048         u64 physical_end;
3049         u64 generation;
3050         int mirror_num;
3051         struct reada_control *reada1;
3052         struct reada_control *reada2;
3053         struct btrfs_key key;
3054         struct btrfs_key key_end;
3055         u64 increment = map->stripe_len;
3056         u64 offset;
3057         u64 extent_logical;
3058         u64 extent_physical;
3059         /*
3060          * Unlike chunk length, extent length should never go beyond
3061          * BTRFS_MAX_EXTENT_SIZE, thus u32 is enough here.
3062          */
3063         u32 extent_len;
3064         u64 stripe_logical;
3065         u64 stripe_end;
3066         struct btrfs_device *extent_dev;
3067         int extent_mirror_num;
3068         int stop_loop = 0;
3069
3070         physical = map->stripes[num].physical;
3071         offset = 0;
3072         nstripes = div64_u64(length, map->stripe_len);
3073         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
3074                 offset = map->stripe_len * num;
3075                 increment = map->stripe_len * map->num_stripes;
3076                 mirror_num = 1;
3077         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
3078                 int factor = map->num_stripes / map->sub_stripes;
3079                 offset = map->stripe_len * (num / map->sub_stripes);
3080                 increment = map->stripe_len * factor;
3081                 mirror_num = num % map->sub_stripes + 1;
3082         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
3083                 increment = map->stripe_len;
3084                 mirror_num = num % map->num_stripes + 1;
3085         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
3086                 increment = map->stripe_len;
3087                 mirror_num = num % map->num_stripes + 1;
3088         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3089                 get_raid56_logic_offset(physical, num, map, &offset, NULL);
3090                 increment = map->stripe_len * nr_data_stripes(map);
3091                 mirror_num = 1;
3092         } else {
3093                 increment = map->stripe_len;
3094                 mirror_num = 1;
3095         }
3096
3097         path = btrfs_alloc_path();
3098         if (!path)
3099                 return -ENOMEM;
3100
3101         ppath = btrfs_alloc_path();
3102         if (!ppath) {
3103                 btrfs_free_path(path);
3104                 return -ENOMEM;
3105         }
3106
3107         /*
3108          * work on commit root. The related disk blocks are static as
3109          * long as COW is applied. This means, it is save to rewrite
3110          * them to repair disk errors without any race conditions
3111          */
3112         path->search_commit_root = 1;
3113         path->skip_locking = 1;
3114
3115         ppath->search_commit_root = 1;
3116         ppath->skip_locking = 1;
3117         /*
3118          * trigger the readahead for extent tree csum tree and wait for
3119          * completion. During readahead, the scrub is officially paused
3120          * to not hold off transaction commits
3121          */
3122         logical = base + offset;
3123         physical_end = physical + nstripes * map->stripe_len;
3124         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3125                 get_raid56_logic_offset(physical_end, num,
3126                                         map, &logic_end, NULL);
3127                 logic_end += base;
3128         } else {
3129                 logic_end = logical + increment * nstripes;
3130         }
3131         wait_event(sctx->list_wait,
3132                    atomic_read(&sctx->bios_in_flight) == 0);
3133         scrub_blocked_if_needed(fs_info);
3134
3135         /* FIXME it might be better to start readahead at commit root */
3136         key.objectid = logical;
3137         key.type = BTRFS_EXTENT_ITEM_KEY;
3138         key.offset = (u64)0;
3139         key_end.objectid = logic_end;
3140         key_end.type = BTRFS_METADATA_ITEM_KEY;
3141         key_end.offset = (u64)-1;
3142         reada1 = btrfs_reada_add(root, &key, &key_end);
3143
3144         if (cache->flags & BTRFS_BLOCK_GROUP_DATA) {
3145                 key.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3146                 key.type = BTRFS_EXTENT_CSUM_KEY;
3147                 key.offset = logical;
3148                 key_end.objectid = BTRFS_EXTENT_CSUM_OBJECTID;
3149                 key_end.type = BTRFS_EXTENT_CSUM_KEY;
3150                 key_end.offset = logic_end;
3151                 reada2 = btrfs_reada_add(csum_root, &key, &key_end);
3152         } else {
3153                 reada2 = NULL;
3154         }
3155
3156         if (!IS_ERR(reada1))
3157                 btrfs_reada_wait(reada1);
3158         if (!IS_ERR_OR_NULL(reada2))
3159                 btrfs_reada_wait(reada2);
3160
3161
3162         /*
3163          * collect all data csums for the stripe to avoid seeking during
3164          * the scrub. This might currently (crc32) end up to be about 1MB
3165          */
3166         blk_start_plug(&plug);
3167
3168         /*
3169          * now find all extents for each stripe and scrub them
3170          */
3171         ret = 0;
3172         while (physical < physical_end) {
3173                 /*
3174                  * canceled?
3175                  */
3176                 if (atomic_read(&fs_info->scrub_cancel_req) ||
3177                     atomic_read(&sctx->cancel_req)) {
3178                         ret = -ECANCELED;
3179                         goto out;
3180                 }
3181                 /*
3182                  * check to see if we have to pause
3183                  */
3184                 if (atomic_read(&fs_info->scrub_pause_req)) {
3185                         /* push queued extents */
3186                         sctx->flush_all_writes = true;
3187                         scrub_submit(sctx);
3188                         mutex_lock(&sctx->wr_lock);
3189                         scrub_wr_submit(sctx);
3190                         mutex_unlock(&sctx->wr_lock);
3191                         wait_event(sctx->list_wait,
3192                                    atomic_read(&sctx->bios_in_flight) == 0);
3193                         sctx->flush_all_writes = false;
3194                         scrub_blocked_if_needed(fs_info);
3195                 }
3196
3197                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3198                         ret = get_raid56_logic_offset(physical, num, map,
3199                                                       &logical,
3200                                                       &stripe_logical);
3201                         logical += base;
3202                         if (ret) {
3203                                 /* it is parity strip */
3204                                 stripe_logical += base;
3205                                 stripe_end = stripe_logical + increment;
3206                                 ret = scrub_raid56_parity(sctx, map, scrub_dev,
3207                                                           ppath, stripe_logical,
3208                                                           stripe_end);
3209                                 if (ret)
3210                                         goto out;
3211                                 goto skip;
3212                         }
3213                 }
3214
3215                 if (btrfs_fs_incompat(fs_info, SKINNY_METADATA))
3216                         key.type = BTRFS_METADATA_ITEM_KEY;
3217                 else
3218                         key.type = BTRFS_EXTENT_ITEM_KEY;
3219                 key.objectid = logical;
3220                 key.offset = (u64)-1;
3221
3222                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3223                 if (ret < 0)
3224                         goto out;
3225
3226                 if (ret > 0) {
3227                         ret = btrfs_previous_extent_item(root, path, 0);
3228                         if (ret < 0)
3229                                 goto out;
3230                         if (ret > 0) {
3231                                 /* there's no smaller item, so stick with the
3232                                  * larger one */
3233                                 btrfs_release_path(path);
3234                                 ret = btrfs_search_slot(NULL, root, &key,
3235                                                         path, 0, 0);
3236                                 if (ret < 0)
3237                                         goto out;
3238                         }
3239                 }
3240
3241                 stop_loop = 0;
3242                 while (1) {
3243                         u64 bytes;
3244
3245                         l = path->nodes[0];
3246                         slot = path->slots[0];
3247                         if (slot >= btrfs_header_nritems(l)) {
3248                                 ret = btrfs_next_leaf(root, path);
3249                                 if (ret == 0)
3250                                         continue;
3251                                 if (ret < 0)
3252                                         goto out;
3253
3254                                 stop_loop = 1;
3255                                 break;
3256                         }
3257                         btrfs_item_key_to_cpu(l, &key, slot);
3258
3259                         if (key.type != BTRFS_EXTENT_ITEM_KEY &&
3260                             key.type != BTRFS_METADATA_ITEM_KEY)
3261                                 goto next;
3262
3263                         if (key.type == BTRFS_METADATA_ITEM_KEY)
3264                                 bytes = fs_info->nodesize;
3265                         else
3266                                 bytes = key.offset;
3267
3268                         if (key.objectid + bytes <= logical)
3269                                 goto next;
3270
3271                         if (key.objectid >= logical + map->stripe_len) {
3272                                 /* out of this device extent */
3273                                 if (key.objectid >= logic_end)
3274                                         stop_loop = 1;
3275                                 break;
3276                         }
3277
3278                         /*
3279                          * If our block group was removed in the meanwhile, just
3280                          * stop scrubbing since there is no point in continuing.
3281                          * Continuing would prevent reusing its device extents
3282                          * for new block groups for a long time.
3283                          */
3284                         spin_lock(&cache->lock);
3285                         if (cache->removed) {
3286                                 spin_unlock(&cache->lock);
3287                                 ret = 0;
3288                                 goto out;
3289                         }
3290                         spin_unlock(&cache->lock);
3291
3292                         extent = btrfs_item_ptr(l, slot,
3293                                                 struct btrfs_extent_item);
3294                         flags = btrfs_extent_flags(l, extent);
3295                         generation = btrfs_extent_generation(l, extent);
3296
3297                         if ((flags & BTRFS_EXTENT_FLAG_TREE_BLOCK) &&
3298                             (key.objectid < logical ||
3299                              key.objectid + bytes >
3300                              logical + map->stripe_len)) {
3301                                 btrfs_err(fs_info,
3302                                            "scrub: tree block %llu spanning stripes, ignored. logical=%llu",
3303                                        key.objectid, logical);
3304                                 spin_lock(&sctx->stat_lock);
3305                                 sctx->stat.uncorrectable_errors++;
3306                                 spin_unlock(&sctx->stat_lock);
3307                                 goto next;
3308                         }
3309
3310 again:
3311                         extent_logical = key.objectid;
3312                         ASSERT(bytes <= U32_MAX);
3313                         extent_len = bytes;
3314
3315                         /*
3316                          * trim extent to this stripe
3317                          */
3318                         if (extent_logical < logical) {
3319                                 extent_len -= logical - extent_logical;
3320                                 extent_logical = logical;
3321                         }
3322                         if (extent_logical + extent_len >
3323                             logical + map->stripe_len) {
3324                                 extent_len = logical + map->stripe_len -
3325                                              extent_logical;
3326                         }
3327
3328                         extent_physical = extent_logical - logical + physical;
3329                         extent_dev = scrub_dev;
3330                         extent_mirror_num = mirror_num;
3331                         if (sctx->is_dev_replace)
3332                                 scrub_remap_extent(fs_info, extent_logical,
3333                                                    extent_len, &extent_physical,
3334                                                    &extent_dev,
3335                                                    &extent_mirror_num);
3336
3337                         if (flags & BTRFS_EXTENT_FLAG_DATA) {
3338                                 ret = btrfs_lookup_csums_range(csum_root,
3339                                                 extent_logical,
3340                                                 extent_logical + extent_len - 1,
3341                                                 &sctx->csum_list, 1);
3342                                 if (ret)
3343                                         goto out;
3344                         }
3345
3346                         ret = scrub_extent(sctx, map, extent_logical, extent_len,
3347                                            extent_physical, extent_dev, flags,
3348                                            generation, extent_mirror_num,
3349                                            extent_logical - logical + physical);
3350
3351                         scrub_free_csums(sctx);
3352
3353                         if (ret)
3354                                 goto out;
3355
3356                         if (extent_logical + extent_len <
3357                             key.objectid + bytes) {
3358                                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
3359                                         /*
3360                                          * loop until we find next data stripe
3361                                          * or we have finished all stripes.
3362                                          */
3363 loop:
3364                                         physical += map->stripe_len;
3365                                         ret = get_raid56_logic_offset(physical,
3366                                                         num, map, &logical,
3367                                                         &stripe_logical);
3368                                         logical += base;
3369
3370                                         if (ret && physical < physical_end) {
3371                                                 stripe_logical += base;
3372                                                 stripe_end = stripe_logical +
3373                                                                 increment;
3374                                                 ret = scrub_raid56_parity(sctx,
3375                                                         map, scrub_dev, ppath,
3376                                                         stripe_logical,
3377                                                         stripe_end);
3378                                                 if (ret)
3379                                                         goto out;
3380                                                 goto loop;
3381                                         }
3382                                 } else {
3383                                         physical += map->stripe_len;
3384                                         logical += increment;
3385                                 }
3386                                 if (logical < key.objectid + bytes) {
3387                                         cond_resched();
3388                                         goto again;
3389                                 }
3390
3391                                 if (physical >= physical_end) {
3392                                         stop_loop = 1;
3393                                         break;
3394                                 }
3395                         }
3396 next:
3397                         path->slots[0]++;
3398                 }
3399                 btrfs_release_path(path);
3400 skip:
3401                 logical += increment;
3402                 physical += map->stripe_len;
3403                 spin_lock(&sctx->stat_lock);
3404                 if (stop_loop)
3405                         sctx->stat.last_physical = map->stripes[num].physical +
3406                                                    length;
3407                 else
3408                         sctx->stat.last_physical = physical;
3409                 spin_unlock(&sctx->stat_lock);
3410                 if (stop_loop)
3411                         break;
3412         }
3413 out:
3414         /* push queued extents */
3415         scrub_submit(sctx);
3416         mutex_lock(&sctx->wr_lock);
3417         scrub_wr_submit(sctx);
3418         mutex_unlock(&sctx->wr_lock);
3419
3420         blk_finish_plug(&plug);
3421         btrfs_free_path(path);
3422         btrfs_free_path(ppath);
3423         return ret < 0 ? ret : 0;
3424 }
3425
3426 static noinline_for_stack int scrub_chunk(struct scrub_ctx *sctx,
3427                                           struct btrfs_device *scrub_dev,
3428                                           u64 chunk_offset, u64 length,
3429                                           u64 dev_offset,
3430                                           struct btrfs_block_group *cache)
3431 {
3432         struct btrfs_fs_info *fs_info = sctx->fs_info;
3433         struct extent_map_tree *map_tree = &fs_info->mapping_tree;
3434         struct map_lookup *map;
3435         struct extent_map *em;
3436         int i;
3437         int ret = 0;
3438
3439         read_lock(&map_tree->lock);
3440         em = lookup_extent_mapping(map_tree, chunk_offset, 1);
3441         read_unlock(&map_tree->lock);
3442
3443         if (!em) {
3444                 /*
3445                  * Might have been an unused block group deleted by the cleaner
3446                  * kthread or relocation.
3447                  */
3448                 spin_lock(&cache->lock);
3449                 if (!cache->removed)
3450                         ret = -EINVAL;
3451                 spin_unlock(&cache->lock);
3452
3453                 return ret;
3454         }
3455
3456         map = em->map_lookup;
3457         if (em->start != chunk_offset)
3458                 goto out;
3459
3460         if (em->len < length)
3461                 goto out;
3462
3463         for (i = 0; i < map->num_stripes; ++i) {
3464                 if (map->stripes[i].dev->bdev == scrub_dev->bdev &&
3465                     map->stripes[i].physical == dev_offset) {
3466                         ret = scrub_stripe(sctx, map, scrub_dev, i,
3467                                            chunk_offset, length, cache);
3468                         if (ret)
3469                                 goto out;
3470                 }
3471         }
3472 out:
3473         free_extent_map(em);
3474
3475         return ret;
3476 }
3477
3478 static noinline_for_stack
3479 int scrub_enumerate_chunks(struct scrub_ctx *sctx,
3480                            struct btrfs_device *scrub_dev, u64 start, u64 end)
3481 {
3482         struct btrfs_dev_extent *dev_extent = NULL;
3483         struct btrfs_path *path;
3484         struct btrfs_fs_info *fs_info = sctx->fs_info;
3485         struct btrfs_root *root = fs_info->dev_root;
3486         u64 length;
3487         u64 chunk_offset;
3488         int ret = 0;
3489         int ro_set;
3490         int slot;
3491         struct extent_buffer *l;
3492         struct btrfs_key key;
3493         struct btrfs_key found_key;
3494         struct btrfs_block_group *cache;
3495         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
3496
3497         path = btrfs_alloc_path();
3498         if (!path)
3499                 return -ENOMEM;
3500
3501         path->reada = READA_FORWARD;
3502         path->search_commit_root = 1;
3503         path->skip_locking = 1;
3504
3505         key.objectid = scrub_dev->devid;
3506         key.offset = 0ull;
3507         key.type = BTRFS_DEV_EXTENT_KEY;
3508
3509         while (1) {
3510                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3511                 if (ret < 0)
3512                         break;
3513                 if (ret > 0) {
3514                         if (path->slots[0] >=
3515                             btrfs_header_nritems(path->nodes[0])) {
3516                                 ret = btrfs_next_leaf(root, path);
3517                                 if (ret < 0)
3518                                         break;
3519                                 if (ret > 0) {
3520                                         ret = 0;
3521                                         break;
3522                                 }
3523                         } else {
3524                                 ret = 0;
3525                         }
3526                 }
3527
3528                 l = path->nodes[0];
3529                 slot = path->slots[0];
3530
3531                 btrfs_item_key_to_cpu(l, &found_key, slot);
3532
3533                 if (found_key.objectid != scrub_dev->devid)
3534                         break;
3535
3536                 if (found_key.type != BTRFS_DEV_EXTENT_KEY)
3537                         break;
3538
3539                 if (found_key.offset >= end)
3540                         break;
3541
3542                 if (found_key.offset < key.offset)
3543                         break;
3544
3545                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
3546                 length = btrfs_dev_extent_length(l, dev_extent);
3547
3548                 if (found_key.offset + length <= start)
3549                         goto skip;
3550
3551                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
3552
3553                 /*
3554                  * get a reference on the corresponding block group to prevent
3555                  * the chunk from going away while we scrub it
3556                  */
3557                 cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3558
3559                 /* some chunks are removed but not committed to disk yet,
3560                  * continue scrubbing */
3561                 if (!cache)
3562                         goto skip;
3563
3564                 /*
3565                  * Make sure that while we are scrubbing the corresponding block
3566                  * group doesn't get its logical address and its device extents
3567                  * reused for another block group, which can possibly be of a
3568                  * different type and different profile. We do this to prevent
3569                  * false error detections and crashes due to bogus attempts to
3570                  * repair extents.
3571                  */
3572                 spin_lock(&cache->lock);
3573                 if (cache->removed) {
3574                         spin_unlock(&cache->lock);
3575                         btrfs_put_block_group(cache);
3576                         goto skip;
3577                 }
3578                 btrfs_freeze_block_group(cache);
3579                 spin_unlock(&cache->lock);
3580
3581                 /*
3582                  * we need call btrfs_inc_block_group_ro() with scrubs_paused,
3583                  * to avoid deadlock caused by:
3584                  * btrfs_inc_block_group_ro()
3585                  * -> btrfs_wait_for_commit()
3586                  * -> btrfs_commit_transaction()
3587                  * -> btrfs_scrub_pause()
3588                  */
3589                 scrub_pause_on(fs_info);
3590
3591                 /*
3592                  * Don't do chunk preallocation for scrub.
3593                  *
3594                  * This is especially important for SYSTEM bgs, or we can hit
3595                  * -EFBIG from btrfs_finish_chunk_alloc() like:
3596                  * 1. The only SYSTEM bg is marked RO.
3597                  *    Since SYSTEM bg is small, that's pretty common.
3598                  * 2. New SYSTEM bg will be allocated
3599                  *    Due to regular version will allocate new chunk.
3600                  * 3. New SYSTEM bg is empty and will get cleaned up
3601                  *    Before cleanup really happens, it's marked RO again.
3602                  * 4. Empty SYSTEM bg get scrubbed
3603                  *    We go back to 2.
3604                  *
3605                  * This can easily boost the amount of SYSTEM chunks if cleaner
3606                  * thread can't be triggered fast enough, and use up all space
3607                  * of btrfs_super_block::sys_chunk_array
3608                  *
3609                  * While for dev replace, we need to try our best to mark block
3610                  * group RO, to prevent race between:
3611                  * - Write duplication
3612                  *   Contains latest data
3613                  * - Scrub copy
3614                  *   Contains data from commit tree
3615                  *
3616                  * If target block group is not marked RO, nocow writes can
3617                  * be overwritten by scrub copy, causing data corruption.
3618                  * So for dev-replace, it's not allowed to continue if a block
3619                  * group is not RO.
3620                  */
3621                 ret = btrfs_inc_block_group_ro(cache, sctx->is_dev_replace);
3622                 if (ret == 0) {
3623                         ro_set = 1;
3624                 } else if (ret == -ENOSPC && !sctx->is_dev_replace) {
3625                         /*
3626                          * btrfs_inc_block_group_ro return -ENOSPC when it
3627                          * failed in creating new chunk for metadata.
3628                          * It is not a problem for scrub, because
3629                          * metadata are always cowed, and our scrub paused
3630                          * commit_transactions.
3631                          */
3632                         ro_set = 0;
3633                 } else {
3634                         btrfs_warn(fs_info,
3635                                    "failed setting block group ro: %d", ret);
3636                         btrfs_unfreeze_block_group(cache);
3637                         btrfs_put_block_group(cache);
3638                         scrub_pause_off(fs_info);
3639                         break;
3640                 }
3641
3642                 /*
3643                  * Now the target block is marked RO, wait for nocow writes to
3644                  * finish before dev-replace.
3645                  * COW is fine, as COW never overwrites extents in commit tree.
3646                  */
3647                 if (sctx->is_dev_replace) {
3648                         btrfs_wait_nocow_writers(cache);
3649                         btrfs_wait_ordered_roots(fs_info, U64_MAX, cache->start,
3650                                         cache->length);
3651                 }
3652
3653                 scrub_pause_off(fs_info);
3654                 down_write(&dev_replace->rwsem);
3655                 dev_replace->cursor_right = found_key.offset + length;
3656                 dev_replace->cursor_left = found_key.offset;
3657                 dev_replace->item_needs_writeback = 1;
3658                 up_write(&dev_replace->rwsem);
3659
3660                 ret = scrub_chunk(sctx, scrub_dev, chunk_offset, length,
3661                                   found_key.offset, cache);
3662
3663                 /*
3664                  * flush, submit all pending read and write bios, afterwards
3665                  * wait for them.
3666                  * Note that in the dev replace case, a read request causes
3667                  * write requests that are submitted in the read completion
3668                  * worker. Therefore in the current situation, it is required
3669                  * that all write requests are flushed, so that all read and
3670                  * write requests are really completed when bios_in_flight
3671                  * changes to 0.
3672                  */
3673                 sctx->flush_all_writes = true;
3674                 scrub_submit(sctx);
3675                 mutex_lock(&sctx->wr_lock);
3676                 scrub_wr_submit(sctx);
3677                 mutex_unlock(&sctx->wr_lock);
3678
3679                 wait_event(sctx->list_wait,
3680                            atomic_read(&sctx->bios_in_flight) == 0);
3681
3682                 scrub_pause_on(fs_info);
3683
3684                 /*
3685                  * must be called before we decrease @scrub_paused.
3686                  * make sure we don't block transaction commit while
3687                  * we are waiting pending workers finished.
3688                  */
3689                 wait_event(sctx->list_wait,
3690                            atomic_read(&sctx->workers_pending) == 0);
3691                 sctx->flush_all_writes = false;
3692
3693                 scrub_pause_off(fs_info);
3694
3695                 down_write(&dev_replace->rwsem);
3696                 dev_replace->cursor_left = dev_replace->cursor_right;
3697                 dev_replace->item_needs_writeback = 1;
3698                 up_write(&dev_replace->rwsem);
3699
3700                 if (ro_set)
3701                         btrfs_dec_block_group_ro(cache);
3702
3703                 /*
3704                  * We might have prevented the cleaner kthread from deleting
3705                  * this block group if it was already unused because we raced
3706                  * and set it to RO mode first. So add it back to the unused
3707                  * list, otherwise it might not ever be deleted unless a manual
3708                  * balance is triggered or it becomes used and unused again.
3709                  */
3710                 spin_lock(&cache->lock);
3711                 if (!cache->removed && !cache->ro && cache->reserved == 0 &&
3712                     cache->used == 0) {
3713                         spin_unlock(&cache->lock);
3714                         if (btrfs_test_opt(fs_info, DISCARD_ASYNC))
3715                                 btrfs_discard_queue_work(&fs_info->discard_ctl,
3716                                                          cache);
3717                         else
3718                                 btrfs_mark_bg_unused(cache);
3719                 } else {
3720                         spin_unlock(&cache->lock);
3721                 }
3722
3723                 btrfs_unfreeze_block_group(cache);
3724                 btrfs_put_block_group(cache);
3725                 if (ret)
3726                         break;
3727                 if (sctx->is_dev_replace &&
3728                     atomic64_read(&dev_replace->num_write_errors) > 0) {
3729                         ret = -EIO;
3730                         break;
3731                 }
3732                 if (sctx->stat.malloc_errors > 0) {
3733                         ret = -ENOMEM;
3734                         break;
3735                 }
3736 skip:
3737                 key.offset = found_key.offset + length;
3738                 btrfs_release_path(path);
3739         }
3740
3741         btrfs_free_path(path);
3742
3743         return ret;
3744 }
3745
3746 static noinline_for_stack int scrub_supers(struct scrub_ctx *sctx,
3747                                            struct btrfs_device *scrub_dev)
3748 {
3749         int     i;
3750         u64     bytenr;
3751         u64     gen;
3752         int     ret;
3753         struct btrfs_fs_info *fs_info = sctx->fs_info;
3754
3755         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
3756                 return -EROFS;
3757
3758         /* Seed devices of a new filesystem has their own generation. */
3759         if (scrub_dev->fs_devices != fs_info->fs_devices)
3760                 gen = scrub_dev->generation;
3761         else
3762                 gen = fs_info->last_trans_committed;
3763
3764         for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
3765                 bytenr = btrfs_sb_offset(i);
3766                 if (bytenr + BTRFS_SUPER_INFO_SIZE >
3767                     scrub_dev->commit_total_bytes)
3768                         break;
3769                 if (!btrfs_check_super_location(scrub_dev, bytenr))
3770                         continue;
3771
3772                 ret = scrub_pages(sctx, bytenr, BTRFS_SUPER_INFO_SIZE, bytenr,
3773                                   scrub_dev, BTRFS_EXTENT_FLAG_SUPER, gen, i,
3774                                   NULL, bytenr);
3775                 if (ret)
3776                         return ret;
3777         }
3778         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3779
3780         return 0;
3781 }
3782
3783 static void scrub_workers_put(struct btrfs_fs_info *fs_info)
3784 {
3785         if (refcount_dec_and_mutex_lock(&fs_info->scrub_workers_refcnt,
3786                                         &fs_info->scrub_lock)) {
3787                 struct btrfs_workqueue *scrub_workers = NULL;
3788                 struct btrfs_workqueue *scrub_wr_comp = NULL;
3789                 struct btrfs_workqueue *scrub_parity = NULL;
3790
3791                 scrub_workers = fs_info->scrub_workers;
3792                 scrub_wr_comp = fs_info->scrub_wr_completion_workers;
3793                 scrub_parity = fs_info->scrub_parity_workers;
3794
3795                 fs_info->scrub_workers = NULL;
3796                 fs_info->scrub_wr_completion_workers = NULL;
3797                 fs_info->scrub_parity_workers = NULL;
3798                 mutex_unlock(&fs_info->scrub_lock);
3799
3800                 btrfs_destroy_workqueue(scrub_workers);
3801                 btrfs_destroy_workqueue(scrub_wr_comp);
3802                 btrfs_destroy_workqueue(scrub_parity);
3803         }
3804 }
3805
3806 /*
3807  * get a reference count on fs_info->scrub_workers. start worker if necessary
3808  */
3809 static noinline_for_stack int scrub_workers_get(struct btrfs_fs_info *fs_info,
3810                                                 int is_dev_replace)
3811 {
3812         struct btrfs_workqueue *scrub_workers = NULL;
3813         struct btrfs_workqueue *scrub_wr_comp = NULL;
3814         struct btrfs_workqueue *scrub_parity = NULL;
3815         unsigned int flags = WQ_FREEZABLE | WQ_UNBOUND;
3816         int max_active = fs_info->thread_pool_size;
3817         int ret = -ENOMEM;
3818
3819         if (refcount_inc_not_zero(&fs_info->scrub_workers_refcnt))
3820                 return 0;
3821
3822         scrub_workers = btrfs_alloc_workqueue(fs_info, "scrub", flags,
3823                                               is_dev_replace ? 1 : max_active, 4);
3824         if (!scrub_workers)
3825                 goto fail_scrub_workers;
3826
3827         scrub_wr_comp = btrfs_alloc_workqueue(fs_info, "scrubwrc", flags,
3828                                               max_active, 2);
3829         if (!scrub_wr_comp)
3830                 goto fail_scrub_wr_completion_workers;
3831
3832         scrub_parity = btrfs_alloc_workqueue(fs_info, "scrubparity", flags,
3833                                              max_active, 2);
3834         if (!scrub_parity)
3835                 goto fail_scrub_parity_workers;
3836
3837         mutex_lock(&fs_info->scrub_lock);
3838         if (refcount_read(&fs_info->scrub_workers_refcnt) == 0) {
3839                 ASSERT(fs_info->scrub_workers == NULL &&
3840                        fs_info->scrub_wr_completion_workers == NULL &&
3841                        fs_info->scrub_parity_workers == NULL);
3842                 fs_info->scrub_workers = scrub_workers;
3843                 fs_info->scrub_wr_completion_workers = scrub_wr_comp;
3844                 fs_info->scrub_parity_workers = scrub_parity;
3845                 refcount_set(&fs_info->scrub_workers_refcnt, 1);
3846                 mutex_unlock(&fs_info->scrub_lock);
3847                 return 0;
3848         }
3849         /* Other thread raced in and created the workers for us */
3850         refcount_inc(&fs_info->scrub_workers_refcnt);
3851         mutex_unlock(&fs_info->scrub_lock);
3852
3853         ret = 0;
3854         btrfs_destroy_workqueue(scrub_parity);
3855 fail_scrub_parity_workers:
3856         btrfs_destroy_workqueue(scrub_wr_comp);
3857 fail_scrub_wr_completion_workers:
3858         btrfs_destroy_workqueue(scrub_workers);
3859 fail_scrub_workers:
3860         return ret;
3861 }
3862
3863 int btrfs_scrub_dev(struct btrfs_fs_info *fs_info, u64 devid, u64 start,
3864                     u64 end, struct btrfs_scrub_progress *progress,
3865                     int readonly, int is_dev_replace)
3866 {
3867         struct scrub_ctx *sctx;
3868         int ret;
3869         struct btrfs_device *dev;
3870         unsigned int nofs_flag;
3871
3872         if (btrfs_fs_closing(fs_info))
3873                 return -EAGAIN;
3874
3875         if (fs_info->nodesize > BTRFS_STRIPE_LEN) {
3876                 /*
3877                  * in this case scrub is unable to calculate the checksum
3878                  * the way scrub is implemented. Do not handle this
3879                  * situation at all because it won't ever happen.
3880                  */
3881                 btrfs_err(fs_info,
3882                            "scrub: size assumption nodesize <= BTRFS_STRIPE_LEN (%d <= %d) fails",
3883                        fs_info->nodesize,
3884                        BTRFS_STRIPE_LEN);
3885                 return -EINVAL;
3886         }
3887
3888         if (fs_info->nodesize >
3889             PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK ||
3890             fs_info->sectorsize > PAGE_SIZE * SCRUB_MAX_PAGES_PER_BLOCK) {
3891                 /*
3892                  * would exhaust the array bounds of pagev member in
3893                  * struct scrub_block
3894                  */
3895                 btrfs_err(fs_info,
3896                           "scrub: size assumption nodesize and sectorsize <= SCRUB_MAX_PAGES_PER_BLOCK (%d <= %d && %d <= %d) fails",
3897                        fs_info->nodesize,
3898                        SCRUB_MAX_PAGES_PER_BLOCK,
3899                        fs_info->sectorsize,
3900                        SCRUB_MAX_PAGES_PER_BLOCK);
3901                 return -EINVAL;
3902         }
3903
3904         /* Allocate outside of device_list_mutex */
3905         sctx = scrub_setup_ctx(fs_info, is_dev_replace);
3906         if (IS_ERR(sctx))
3907                 return PTR_ERR(sctx);
3908
3909         ret = scrub_workers_get(fs_info, is_dev_replace);
3910         if (ret)
3911                 goto out_free_ctx;
3912
3913         mutex_lock(&fs_info->fs_devices->device_list_mutex);
3914         dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL);
3915         if (!dev || (test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) &&
3916                      !is_dev_replace)) {
3917                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3918                 ret = -ENODEV;
3919                 goto out;
3920         }
3921
3922         if (!is_dev_replace && !readonly &&
3923             !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state)) {
3924                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3925                 btrfs_err_in_rcu(fs_info,
3926                         "scrub on devid %llu: filesystem on %s is not writable",
3927                                  devid, rcu_str_deref(dev->name));
3928                 ret = -EROFS;
3929                 goto out;
3930         }
3931
3932         mutex_lock(&fs_info->scrub_lock);
3933         if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &dev->dev_state) ||
3934             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &dev->dev_state)) {
3935                 mutex_unlock(&fs_info->scrub_lock);
3936                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3937                 ret = -EIO;
3938                 goto out;
3939         }
3940
3941         down_read(&fs_info->dev_replace.rwsem);
3942         if (dev->scrub_ctx ||
3943             (!is_dev_replace &&
3944              btrfs_dev_replace_is_ongoing(&fs_info->dev_replace))) {
3945                 up_read(&fs_info->dev_replace.rwsem);
3946                 mutex_unlock(&fs_info->scrub_lock);
3947                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3948                 ret = -EINPROGRESS;
3949                 goto out;
3950         }
3951         up_read(&fs_info->dev_replace.rwsem);
3952
3953         sctx->readonly = readonly;
3954         dev->scrub_ctx = sctx;
3955         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3956
3957         /*
3958          * checking @scrub_pause_req here, we can avoid
3959          * race between committing transaction and scrubbing.
3960          */
3961         __scrub_blocked_if_needed(fs_info);
3962         atomic_inc(&fs_info->scrubs_running);
3963         mutex_unlock(&fs_info->scrub_lock);
3964
3965         /*
3966          * In order to avoid deadlock with reclaim when there is a transaction
3967          * trying to pause scrub, make sure we use GFP_NOFS for all the
3968          * allocations done at btrfs_scrub_pages() and scrub_pages_for_parity()
3969          * invoked by our callees. The pausing request is done when the
3970          * transaction commit starts, and it blocks the transaction until scrub
3971          * is paused (done at specific points at scrub_stripe() or right above
3972          * before incrementing fs_info->scrubs_running).
3973          */
3974         nofs_flag = memalloc_nofs_save();
3975         if (!is_dev_replace) {
3976                 btrfs_info(fs_info, "scrub: started on devid %llu", devid);
3977                 /*
3978                  * by holding device list mutex, we can
3979                  * kick off writing super in log tree sync.
3980                  */
3981                 mutex_lock(&fs_info->fs_devices->device_list_mutex);
3982                 ret = scrub_supers(sctx, dev);
3983                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
3984         }
3985
3986         if (!ret)
3987                 ret = scrub_enumerate_chunks(sctx, dev, start, end);
3988         memalloc_nofs_restore(nofs_flag);
3989
3990         wait_event(sctx->list_wait, atomic_read(&sctx->bios_in_flight) == 0);
3991         atomic_dec(&fs_info->scrubs_running);
3992         wake_up(&fs_info->scrub_pause_wait);
3993
3994         wait_event(sctx->list_wait, atomic_read(&sctx->workers_pending) == 0);
3995
3996         if (progress)
3997                 memcpy(progress, &sctx->stat, sizeof(*progress));
3998
3999         if (!is_dev_replace)
4000                 btrfs_info(fs_info, "scrub: %s on devid %llu with status: %d",
4001                         ret ? "not finished" : "finished", devid, ret);
4002
4003         mutex_lock(&fs_info->scrub_lock);
4004         dev->scrub_ctx = NULL;
4005         mutex_unlock(&fs_info->scrub_lock);
4006
4007         scrub_workers_put(fs_info);
4008         scrub_put_ctx(sctx);
4009
4010         return ret;
4011 out:
4012         scrub_workers_put(fs_info);
4013 out_free_ctx:
4014         scrub_free_ctx(sctx);
4015
4016         return ret;
4017 }
4018
4019 void btrfs_scrub_pause(struct btrfs_fs_info *fs_info)
4020 {
4021         mutex_lock(&fs_info->scrub_lock);
4022         atomic_inc(&fs_info->scrub_pause_req);
4023         while (atomic_read(&fs_info->scrubs_paused) !=
4024                atomic_read(&fs_info->scrubs_running)) {
4025                 mutex_unlock(&fs_info->scrub_lock);
4026                 wait_event(fs_info->scrub_pause_wait,
4027                            atomic_read(&fs_info->scrubs_paused) ==
4028                            atomic_read(&fs_info->scrubs_running));
4029                 mutex_lock(&fs_info->scrub_lock);
4030         }
4031         mutex_unlock(&fs_info->scrub_lock);
4032 }
4033
4034 void btrfs_scrub_continue(struct btrfs_fs_info *fs_info)
4035 {
4036         atomic_dec(&fs_info->scrub_pause_req);
4037         wake_up(&fs_info->scrub_pause_wait);
4038 }
4039
4040 int btrfs_scrub_cancel(struct btrfs_fs_info *fs_info)
4041 {
4042         mutex_lock(&fs_info->scrub_lock);
4043         if (!atomic_read(&fs_info->scrubs_running)) {
4044                 mutex_unlock(&fs_info->scrub_lock);
4045                 return -ENOTCONN;
4046         }
4047
4048         atomic_inc(&fs_info->scrub_cancel_req);
4049         while (atomic_read(&fs_info->scrubs_running)) {
4050                 mutex_unlock(&fs_info->scrub_lock);
4051                 wait_event(fs_info->scrub_pause_wait,
4052                            atomic_read(&fs_info->scrubs_running) == 0);
4053                 mutex_lock(&fs_info->scrub_lock);
4054         }
4055         atomic_dec(&fs_info->scrub_cancel_req);
4056         mutex_unlock(&fs_info->scrub_lock);
4057
4058         return 0;
4059 }
4060
4061 int btrfs_scrub_cancel_dev(struct btrfs_device *dev)
4062 {
4063         struct btrfs_fs_info *fs_info = dev->fs_info;
4064         struct scrub_ctx *sctx;
4065
4066         mutex_lock(&fs_info->scrub_lock);
4067         sctx = dev->scrub_ctx;
4068         if (!sctx) {
4069                 mutex_unlock(&fs_info->scrub_lock);
4070                 return -ENOTCONN;
4071         }
4072         atomic_inc(&sctx->cancel_req);
4073         while (dev->scrub_ctx) {
4074                 mutex_unlock(&fs_info->scrub_lock);
4075                 wait_event(fs_info->scrub_pause_wait,
4076                            dev->scrub_ctx == NULL);
4077                 mutex_lock(&fs_info->scrub_lock);
4078         }
4079         mutex_unlock(&fs_info->scrub_lock);
4080
4081         return 0;
4082 }
4083
4084 int btrfs_scrub_progress(struct btrfs_fs_info *fs_info, u64 devid,
4085                          struct btrfs_scrub_progress *progress)
4086 {
4087         struct btrfs_device *dev;
4088         struct scrub_ctx *sctx = NULL;
4089
4090         mutex_lock(&fs_info->fs_devices->device_list_mutex);
4091         dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL);
4092         if (dev)
4093                 sctx = dev->scrub_ctx;
4094         if (sctx)
4095                 memcpy(progress, &sctx->stat, sizeof(*progress));
4096         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
4097
4098         return dev ? (sctx ? 0 : -ENOTCONN) : -ENODEV;
4099 }
4100
4101 static void scrub_remap_extent(struct btrfs_fs_info *fs_info,
4102                                u64 extent_logical, u32 extent_len,
4103                                u64 *extent_physical,
4104                                struct btrfs_device **extent_dev,
4105                                int *extent_mirror_num)
4106 {
4107         u64 mapped_length;
4108         struct btrfs_bio *bbio = NULL;
4109         int ret;
4110
4111         mapped_length = extent_len;
4112         ret = btrfs_map_block(fs_info, BTRFS_MAP_READ, extent_logical,
4113                               &mapped_length, &bbio, 0);
4114         if (ret || !bbio || mapped_length < extent_len ||
4115             !bbio->stripes[0].dev->bdev) {
4116                 btrfs_put_bbio(bbio);
4117                 return;
4118         }
4119
4120         *extent_physical = bbio->stripes[0].physical;
4121         *extent_mirror_num = bbio->mirror_num;
4122         *extent_dev = bbio->stripes[0].dev;
4123         btrfs_put_bbio(bbio);
4124 }