fs/btrfs/volumes.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/sched.h>
   7 #include <linux/sched/mm.h>
   8 #include <linux/bio.h>
   9 #include <linux/slab.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/ratelimit.h>
  12 #include <linux/kthread.h>
  13 #include <linux/raid/pq.h>
  14 #include <linux/semaphore.h>
  15 #include <linux/uuid.h>
  16 #include <linux/list_sort.h>
  17 #include "misc.h"
  18 #include "ctree.h"
  19 #include "extent_map.h"
  20 #include "disk-io.h"
  21 #include "transaction.h"
  22 #include "print-tree.h"
  23 #include "volumes.h"
  24 #include "raid56.h"
  25 #include "async-thread.h"
  26 #include "check-integrity.h"
  27 #include "rcu-string.h"
  28 #include "dev-replace.h"
  29 #include "sysfs.h"
  30 #include "tree-checker.h"
  31 #include "space-info.h"
  32 #include "block-group.h"
  33 #include "discard.h"
  34
  35 const struct btrfs_raid_attr btrfs_raid_array[BTRFS_NR_RAID_TYPES] = {
  36         [BTRFS_RAID_RAID10] = {
  37                 .sub_stripes    = 2,
  38                 .dev_stripes    = 1,
  39                 .devs_max       = 0,    /* 0 == as many as possible */
  40                 .devs_min       = 4,
  41                 .tolerated_failures = 1,
  42                 .devs_increment = 2,
  43                 .ncopies        = 2,
  44                 .nparity        = 0,
  45                 .raid_name      = "raid10",
  46                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID10,
  47                 .mindev_error   = BTRFS_ERROR_DEV_RAID10_MIN_NOT_MET,
  48         },
  49         [BTRFS_RAID_RAID1] = {
  50                 .sub_stripes    = 1,
  51                 .dev_stripes    = 1,
  52                 .devs_max       = 2,
  53                 .devs_min       = 2,
  54                 .tolerated_failures = 1,
  55                 .devs_increment = 2,
  56                 .ncopies        = 2,
  57                 .nparity        = 0,
  58                 .raid_name      = "raid1",
  59                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1,
  60                 .mindev_error   = BTRFS_ERROR_DEV_RAID1_MIN_NOT_MET,
  61         },
  62         [BTRFS_RAID_RAID1C3] = {
  63                 .sub_stripes    = 1,
  64                 .dev_stripes    = 1,
  65                 .devs_max       = 3,
  66                 .devs_min       = 3,
  67                 .tolerated_failures = 2,
  68                 .devs_increment = 3,
  69                 .ncopies        = 3,
  70                 .nparity        = 0,
  71                 .raid_name      = "raid1c3",
  72                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1C3,
  73                 .mindev_error   = BTRFS_ERROR_DEV_RAID1C3_MIN_NOT_MET,
  74         },
  75         [BTRFS_RAID_RAID1C4] = {
  76                 .sub_stripes    = 1,
  77                 .dev_stripes    = 1,
  78                 .devs_max       = 4,
  79                 .devs_min       = 4,
  80                 .tolerated_failures = 3,
  81                 .devs_increment = 4,
  82                 .ncopies        = 4,
  83                 .nparity        = 0,
  84                 .raid_name      = "raid1c4",
  85                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID1C4,
  86                 .mindev_error   = BTRFS_ERROR_DEV_RAID1C4_MIN_NOT_MET,
  87         },
  88         [BTRFS_RAID_DUP] = {
  89                 .sub_stripes    = 1,
  90                 .dev_stripes    = 2,
  91                 .devs_max       = 1,
  92                 .devs_min       = 1,
  93                 .tolerated_failures = 0,
  94                 .devs_increment = 1,
  95                 .ncopies        = 2,
  96                 .nparity        = 0,
  97                 .raid_name      = "dup",
  98                 .bg_flag        = BTRFS_BLOCK_GROUP_DUP,
  99                 .mindev_error   = 0,
 100         },
 101         [BTRFS_RAID_RAID0] = {
 102                 .sub_stripes    = 1,
 103                 .dev_stripes    = 1,
 104                 .devs_max       = 0,
 105                 .devs_min       = 2,
 106                 .tolerated_failures = 0,
 107                 .devs_increment = 1,
 108                 .ncopies        = 1,
 109                 .nparity        = 0,
 110                 .raid_name      = "raid0",
 111                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID0,
 112                 .mindev_error   = 0,
 113         },
 114         [BTRFS_RAID_SINGLE] = {
 115                 .sub_stripes    = 1,
 116                 .dev_stripes    = 1,
 117                 .devs_max       = 1,
 118                 .devs_min       = 1,
 119                 .tolerated_failures = 0,
 120                 .devs_increment = 1,
 121                 .ncopies        = 1,
 122                 .nparity        = 0,
 123                 .raid_name      = "single",
 124                 .bg_flag        = 0,
 125                 .mindev_error   = 0,
 126         },
 127         [BTRFS_RAID_RAID5] = {
 128                 .sub_stripes    = 1,
 129                 .dev_stripes    = 1,
 130                 .devs_max       = 0,
 131                 .devs_min       = 2,
 132                 .tolerated_failures = 1,
 133                 .devs_increment = 1,
 134                 .ncopies        = 1,
 135                 .nparity        = 1,
 136                 .raid_name      = "raid5",
 137                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID5,
 138                 .mindev_error   = BTRFS_ERROR_DEV_RAID5_MIN_NOT_MET,
 139         },
 140         [BTRFS_RAID_RAID6] = {
 141                 .sub_stripes    = 1,
 142                 .dev_stripes    = 1,
 143                 .devs_max       = 0,
 144                 .devs_min       = 3,
 145                 .tolerated_failures = 2,
 146                 .devs_increment = 1,
 147                 .ncopies        = 1,
 148                 .nparity        = 2,
 149                 .raid_name      = "raid6",
 150                 .bg_flag        = BTRFS_BLOCK_GROUP_RAID6,
 151                 .mindev_error   = BTRFS_ERROR_DEV_RAID6_MIN_NOT_MET,
 152         },
 153 };
 154
 155 const char *btrfs_bg_type_to_raid_name(u64 flags)
 156 {
 157         const int index = btrfs_bg_flags_to_raid_index(flags);
 158
 159         if (index >= BTRFS_NR_RAID_TYPES)
 160                 return NULL;
 161
 162         return btrfs_raid_array[index].raid_name;
 163 }
 164
 165 /*
 166  * Fill @buf with textual description of @bg_flags, no more than @size_buf
 167  * bytes including terminating null byte.
 168  */
 169 void btrfs_describe_block_groups(u64 bg_flags, char *buf, u32 size_buf)
 170 {
 171         int i;
 172         int ret;
 173         char *bp = buf;
 174         u64 flags = bg_flags;
 175         u32 size_bp = size_buf;
 176
 177         if (!flags) {
 178                 strcpy(bp, "NONE");
 179                 return;
 180         }
 181
 182 #define DESCRIBE_FLAG(flag, desc)                                               \
 183         do {                                                            \
 184                 if (flags & (flag)) {                                   \
 185                         ret = snprintf(bp, size_bp, "%s|", (desc));     \
 186                         if (ret < 0 || ret >= size_bp)                  \
 187                                 goto out_overflow;                      \
 188                         size_bp -= ret;                                 \
 189                         bp += ret;                                      \
 190                         flags &= ~(flag);                               \
 191                 }                                                       \
 192         } while (0)
 193
 194         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_DATA, "data");
 195         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_SYSTEM, "system");
 196         DESCRIBE_FLAG(BTRFS_BLOCK_GROUP_METADATA, "metadata");
 197
 198         DESCRIBE_FLAG(BTRFS_AVAIL_ALLOC_BIT_SINGLE, "single");
 199         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++)
 200                 DESCRIBE_FLAG(btrfs_raid_array[i].bg_flag,
 201                               btrfs_raid_array[i].raid_name);
 202 #undef DESCRIBE_FLAG
 203
 204         if (flags) {
 205                 ret = snprintf(bp, size_bp, "0x%llx|", flags);
 206                 size_bp -= ret;
 207         }
 208
 209         if (size_bp < size_buf)
 210                 buf[size_buf - size_bp - 1] = '\0'; /* remove last | */
 211
 212         /*
 213          * The text is trimmed, it's up to the caller to provide sufficiently
 214          * large buffer
 215          */
 216 out_overflow:;
 217 }
 218
 219 static int init_first_rw_device(struct btrfs_trans_handle *trans);
 220 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info);
 221 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev);
 222 static void btrfs_dev_stat_print_on_load(struct btrfs_device *device);
 223 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
 224                              enum btrfs_map_op op,
 225                              u64 logical, u64 *length,
 226                              struct btrfs_bio **bbio_ret,
 227                              int mirror_num, int need_raid_map);
 228
 229 /*
 230  * Device locking
 231  * ==============
 232  *
 233  * There are several mutexes that protect manipulation of devices and low-level
 234  * structures like chunks but not block groups, extents or files
 235  *
 236  * uuid_mutex (global lock)
 237  * ------------------------
 238  * protects the fs_uuids list that tracks all per-fs fs_devices, resulting from
 239  * the SCAN_DEV ioctl registration or from mount either implicitly (the first
 240  * device) or requested by the device= mount option
 241  *
 242  * the mutex can be very coarse and can cover long-running operations
 243  *
 244  * protects: updates to fs_devices counters like missing devices, rw devices,
 245  * seeding, structure cloning, opening/closing devices at mount/umount time
 246  *
 247  * global::fs_devs - add, remove, updates to the global list
 248  *
 249  * does not protect: manipulation of the fs_devices::devices list in general
 250  * but in mount context it could be used to exclude list modifications by eg.
 251  * scan ioctl
 252  *
 253  * btrfs_device::name - renames (write side), read is RCU
 254  *
 255  * fs_devices::device_list_mutex (per-fs, with RCU)
 256  * ------------------------------------------------
 257  * protects updates to fs_devices::devices, ie. adding and deleting
 258  *
 259  * simple list traversal with read-only actions can be done with RCU protection
 260  *
 261  * may be used to exclude some operations from running concurrently without any
 262  * modifications to the list (see write_all_supers)
 263  *
 264  * Is not required at mount and close times, because our device list is
 265  * protected by the uuid_mutex at that point.
 266  *
 267  * balance_mutex
 268  * -------------
 269  * protects balance structures (status, state) and context accessed from
 270  * several places (internally, ioctl)
 271  *
 272  * chunk_mutex
 273  * -----------
 274  * protects chunks, adding or removing during allocation, trim or when a new
 275  * device is added/removed. Additionally it also protects post_commit_list of
 276  * individual devices, since they can be added to the transaction's
 277  * post_commit_list only with chunk_mutex held.
 278  *
 279  * cleaner_mutex
 280  * -------------
 281  * a big lock that is held by the cleaner thread and prevents running subvolume
 282  * cleaning together with relocation or delayed iputs
 283  *
 284  *
 285  * Lock nesting
 286  * ============
 287  *
 288  * uuid_mutex
 289  *   device_list_mutex
 290  *     chunk_mutex
 291  *   balance_mutex
 292  *
 293  *
 294  * Exclusive operations
 295  * ====================
 296  *
 297  * Maintains the exclusivity of the following operations that apply to the
 298  * whole filesystem and cannot run in parallel.
 299  *
 300  * - Balance (*)
 301  * - Device add
 302  * - Device remove
 303  * - Device replace (*)
 304  * - Resize
 305  *
 306  * The device operations (as above) can be in one of the following states:
 307  *
 308  * - Running state
 309  * - Paused state
 310  * - Completed state
 311  *
 312  * Only device operations marked with (*) can go into the Paused state for the
 313  * following reasons:
 314  *
 315  * - ioctl (only Balance can be Paused through ioctl)
 316  * - filesystem remounted as read-only
 317  * - filesystem unmounted and mounted as read-only
 318  * - system power-cycle and filesystem mounted as read-only
 319  * - filesystem or device errors leading to forced read-only
 320  *
 321  * The status of exclusive operation is set and cleared atomically.
 322  * During the course of Paused state, fs_info::exclusive_operation remains set.
 323  * A device operation in Paused or Running state can be canceled or resumed
 324  * either by ioctl (Balance only) or when remounted as read-write.
 325  * The exclusive status is cleared when the device operation is canceled or
 326  * completed.
 327  */
 328
 329 DEFINE_MUTEX(uuid_mutex);
 330 static LIST_HEAD(fs_uuids);
 331 struct list_head * __attribute_const__ btrfs_get_fs_uuids(void)
 332 {
 333         return &fs_uuids;
 334 }
 335
 336 /*
 337  * alloc_fs_devices - allocate struct btrfs_fs_devices
 338  * @fsid:               if not NULL, copy the UUID to fs_devices::fsid
 339  * @metadata_fsid:      if not NULL, copy the UUID to fs_devices::metadata_fsid
 340  *
 341  * Return a pointer to a new struct btrfs_fs_devices on success, or ERR_PTR().
 342  * The returned struct is not linked onto any lists and can be destroyed with
 343  * kfree() right away.
 344  */
 345 static struct btrfs_fs_devices *alloc_fs_devices(const u8 *fsid,
 346                                                  const u8 *metadata_fsid)
 347 {
 348         struct btrfs_fs_devices *fs_devs;
 349
 350         fs_devs = kzalloc(sizeof(*fs_devs), GFP_KERNEL);
 351         if (!fs_devs)
 352                 return ERR_PTR(-ENOMEM);
 353
 354         mutex_init(&fs_devs->device_list_mutex);
 355
 356         INIT_LIST_HEAD(&fs_devs->devices);
 357         INIT_LIST_HEAD(&fs_devs->alloc_list);
 358         INIT_LIST_HEAD(&fs_devs->fs_list);
 359         INIT_LIST_HEAD(&fs_devs->seed_list);
 360         if (fsid)
 361                 memcpy(fs_devs->fsid, fsid, BTRFS_FSID_SIZE);
 362
 363         if (metadata_fsid)
 364                 memcpy(fs_devs->metadata_uuid, metadata_fsid, BTRFS_FSID_SIZE);
 365         else if (fsid)
 366                 memcpy(fs_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE);
 367
 368         return fs_devs;
 369 }
 370
 371 void btrfs_free_device(struct btrfs_device *device)
 372 {
 373         WARN_ON(!list_empty(&device->post_commit_list));
 374         rcu_string_free(device->name);
 375         extent_io_tree_release(&device->alloc_state);
 376         bio_put(device->flush_bio);
 377         kfree(device);
 378 }
 379
 380 static void free_fs_devices(struct btrfs_fs_devices *fs_devices)
 381 {
 382         struct btrfs_device *device;
 383         WARN_ON(fs_devices->opened);
 384         while (!list_empty(&fs_devices->devices)) {
 385                 device = list_entry(fs_devices->devices.next,
 386                                     struct btrfs_device, dev_list);
 387                 list_del(&device->dev_list);
 388                 btrfs_free_device(device);
 389         }
 390         kfree(fs_devices);
 391 }
 392
 393 void __exit btrfs_cleanup_fs_uuids(void)
 394 {
 395         struct btrfs_fs_devices *fs_devices;
 396
 397         while (!list_empty(&fs_uuids)) {
 398                 fs_devices = list_entry(fs_uuids.next,
 399                                         struct btrfs_fs_devices, fs_list);
 400                 list_del(&fs_devices->fs_list);
 401                 free_fs_devices(fs_devices);
 402         }
 403 }
 404
 405 /*
 406  * Returns a pointer to a new btrfs_device on success; ERR_PTR() on error.
 407  * Returned struct is not linked onto any lists and must be destroyed using
 408  * btrfs_free_device.
 409  */
 410 static struct btrfs_device *__alloc_device(struct btrfs_fs_info *fs_info)
 411 {
 412         struct btrfs_device *dev;
 413
 414         dev = kzalloc(sizeof(*dev), GFP_KERNEL);
 415         if (!dev)
 416                 return ERR_PTR(-ENOMEM);
 417
 418         /*
 419          * Preallocate a bio that's always going to be used for flushing device
 420          * barriers and matches the device lifespan
 421          */
 422         dev->flush_bio = bio_alloc_bioset(GFP_KERNEL, 0, NULL);
 423         if (!dev->flush_bio) {
 424                 kfree(dev);
 425                 return ERR_PTR(-ENOMEM);
 426         }
 427
 428         INIT_LIST_HEAD(&dev->dev_list);
 429         INIT_LIST_HEAD(&dev->dev_alloc_list);
 430         INIT_LIST_HEAD(&dev->post_commit_list);
 431
 432         atomic_set(&dev->reada_in_flight, 0);
 433         atomic_set(&dev->dev_stats_ccnt, 0);
 434         btrfs_device_data_ordered_init(dev, fs_info);
 435         INIT_RADIX_TREE(&dev->reada_zones, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 436         INIT_RADIX_TREE(&dev->reada_extents, GFP_NOFS & ~__GFP_DIRECT_RECLAIM);
 437         extent_io_tree_init(fs_info, &dev->alloc_state,
 438                             IO_TREE_DEVICE_ALLOC_STATE, NULL);
 439
 440         return dev;
 441 }
 442
 443 static noinline struct btrfs_fs_devices *find_fsid(
 444                 const u8 *fsid, const u8 *metadata_fsid)
 445 {
 446         struct btrfs_fs_devices *fs_devices;
 447
 448         ASSERT(fsid);
 449
 450         /* Handle non-split brain cases */
 451         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 452                 if (metadata_fsid) {
 453                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0
 454                             && memcmp(metadata_fsid, fs_devices->metadata_uuid,
 455                                       BTRFS_FSID_SIZE) == 0)
 456                                 return fs_devices;
 457                 } else {
 458                         if (memcmp(fsid, fs_devices->fsid, BTRFS_FSID_SIZE) == 0)
 459                                 return fs_devices;
 460                 }
 461         }
 462         return NULL;
 463 }
 464
 465 static struct btrfs_fs_devices *find_fsid_with_metadata_uuid(
 466                                 struct btrfs_super_block *disk_super)
 467 {
 468
 469         struct btrfs_fs_devices *fs_devices;
 470
 471         /*
 472          * Handle scanned device having completed its fsid change but
 473          * belonging to a fs_devices that was created by first scanning
 474          * a device which didn't have its fsid/metadata_uuid changed
 475          * at all and the CHANGING_FSID_V2 flag set.
 476          */
 477         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 478                 if (fs_devices->fsid_change &&
 479                     memcmp(disk_super->metadata_uuid, fs_devices->fsid,
 480                            BTRFS_FSID_SIZE) == 0 &&
 481                     memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
 482                            BTRFS_FSID_SIZE) == 0) {
 483                         return fs_devices;
 484                 }
 485         }
 486         /*
 487          * Handle scanned device having completed its fsid change but
 488          * belonging to a fs_devices that was created by a device that
 489          * has an outdated pair of fsid/metadata_uuid and
 490          * CHANGING_FSID_V2 flag set.
 491          */
 492         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 493                 if (fs_devices->fsid_change &&
 494                     memcmp(fs_devices->metadata_uuid,
 495                            fs_devices->fsid, BTRFS_FSID_SIZE) != 0 &&
 496                     memcmp(disk_super->metadata_uuid, fs_devices->metadata_uuid,
 497                            BTRFS_FSID_SIZE) == 0) {
 498                         return fs_devices;
 499                 }
 500         }
 501
 502         return find_fsid(disk_super->fsid, disk_super->metadata_uuid);
 503 }
 504
 505
 506 static int
 507 btrfs_get_bdev_and_sb(const char *device_path, fmode_t flags, void *holder,
 508                       int flush, struct block_device **bdev,
 509                       struct btrfs_super_block **disk_super)
 510 {
 511         int ret;
 512
 513         *bdev = blkdev_get_by_path(device_path, flags, holder);
 514
 515         if (IS_ERR(*bdev)) {
 516                 ret = PTR_ERR(*bdev);
 517                 goto error;
 518         }
 519
 520         if (flush)
 521                 filemap_write_and_wait((*bdev)->bd_inode->i_mapping);
 522         ret = set_blocksize(*bdev, BTRFS_BDEV_BLOCKSIZE);
 523         if (ret) {
 524                 blkdev_put(*bdev, flags);
 525                 goto error;
 526         }
 527         invalidate_bdev(*bdev);
 528         *disk_super = btrfs_read_dev_super(*bdev);
 529         if (IS_ERR(*disk_super)) {
 530                 ret = PTR_ERR(*disk_super);
 531                 blkdev_put(*bdev, flags);
 532                 goto error;
 533         }
 534
 535         return 0;
 536
 537 error:
 538         *bdev = NULL;
 539         return ret;
 540 }
 541
 542 static bool device_path_matched(const char *path, struct btrfs_device *device)
 543 {
 544         int found;
 545
 546         rcu_read_lock();
 547         found = strcmp(rcu_str_deref(device->name), path);
 548         rcu_read_unlock();
 549
 550         return found == 0;
 551 }
 552
 553 /*
 554  *  Search and remove all stale (devices which are not mounted) devices.
 555  *  When both inputs are NULL, it will search and release all stale devices.
 556  *  path:       Optional. When provided will it release all unmounted devices
 557  *              matching this path only.
 558  *  skip_dev:   Optional. Will skip this device when searching for the stale
 559  *              devices.
 560  *  Return:     0 for success or if @path is NULL.
 561  *              -EBUSY if @path is a mounted device.
 562  *              -ENOENT if @path does not match any device in the list.
 563  */
 564 static int btrfs_free_stale_devices(const char *path,
 565                                      struct btrfs_device *skip_device)
 566 {
 567         struct btrfs_fs_devices *fs_devices, *tmp_fs_devices;
 568         struct btrfs_device *device, *tmp_device;
 569         int ret = 0;
 570
 571         if (path)
 572                 ret = -ENOENT;
 573
 574         list_for_each_entry_safe(fs_devices, tmp_fs_devices, &fs_uuids, fs_list) {
 575
 576                 mutex_lock(&fs_devices->device_list_mutex);
 577                 list_for_each_entry_safe(device, tmp_device,
 578                                          &fs_devices->devices, dev_list) {
 579                         if (skip_device && skip_device == device)
 580                                 continue;
 581                         if (path && !device->name)
 582                                 continue;
 583                         if (path && !device_path_matched(path, device))
 584                                 continue;
 585                         if (fs_devices->opened) {
 586                                 /* for an already deleted device return 0 */
 587                                 if (path && ret != 0)
 588                                         ret = -EBUSY;
 589                                 break;
 590                         }
 591
 592                         /* delete the stale device */
 593                         fs_devices->num_devices--;
 594                         list_del(&device->dev_list);
 595                         btrfs_free_device(device);
 596
 597                         ret = 0;
 598                 }
 599                 mutex_unlock(&fs_devices->device_list_mutex);
 600
 601                 if (fs_devices->num_devices == 0) {
 602                         btrfs_sysfs_remove_fsid(fs_devices);
 603                         list_del(&fs_devices->fs_list);
 604                         free_fs_devices(fs_devices);
 605                 }
 606         }
 607
 608         return ret;
 609 }
 610
 611 /*
 612  * This is only used on mount, and we are protected from competing things
 613  * messing with our fs_devices by the uuid_mutex, thus we do not need the
 614  * fs_devices->device_list_mutex here.
 615  */
 616 static int btrfs_open_one_device(struct btrfs_fs_devices *fs_devices,
 617                         struct btrfs_device *device, fmode_t flags,
 618                         void *holder)
 619 {
 620         struct request_queue *q;
 621         struct block_device *bdev;
 622         struct btrfs_super_block *disk_super;
 623         u64 devid;
 624         int ret;
 625
 626         if (device->bdev)
 627                 return -EINVAL;
 628         if (!device->name)
 629                 return -EINVAL;
 630
 631         ret = btrfs_get_bdev_and_sb(device->name->str, flags, holder, 1,
 632                                     &bdev, &disk_super);
 633         if (ret)
 634                 return ret;
 635
 636         devid = btrfs_stack_device_id(&disk_super->dev_item);
 637         if (devid != device->devid)
 638                 goto error_free_page;
 639
 640         if (memcmp(device->uuid, disk_super->dev_item.uuid, BTRFS_UUID_SIZE))
 641                 goto error_free_page;
 642
 643         device->generation = btrfs_super_generation(disk_super);
 644
 645         if (btrfs_super_flags(disk_super) & BTRFS_SUPER_FLAG_SEEDING) {
 646                 if (btrfs_super_incompat_flags(disk_super) &
 647                     BTRFS_FEATURE_INCOMPAT_METADATA_UUID) {
 648                         pr_err(
 649                 "BTRFS: Invalid seeding and uuid-changed device detected\n");
 650                         goto error_free_page;
 651                 }
 652
 653                 clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
 654                 fs_devices->seeding = true;
 655         } else {
 656                 if (bdev_read_only(bdev))
 657                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
 658                 else
 659                         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
 660         }
 661
 662         q = bdev_get_queue(bdev);
 663         if (!blk_queue_nonrot(q))
 664                 fs_devices->rotating = true;
 665
 666         device->bdev = bdev;
 667         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
 668         device->mode = flags;
 669
 670         fs_devices->open_devices++;
 671         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
 672             device->devid != BTRFS_DEV_REPLACE_DEVID) {
 673                 fs_devices->rw_devices++;
 674                 list_add_tail(&device->dev_alloc_list, &fs_devices->alloc_list);
 675         }
 676         btrfs_release_disk_super(disk_super);
 677
 678         return 0;
 679
 680 error_free_page:
 681         btrfs_release_disk_super(disk_super);
 682         blkdev_put(bdev, flags);
 683
 684         return -EINVAL;
 685 }
 686
 687 /*
 688  * Handle scanned device having its CHANGING_FSID_V2 flag set and the fs_devices
 689  * being created with a disk that has already completed its fsid change. Such
 690  * disk can belong to an fs which has its FSID changed or to one which doesn't.
 691  * Handle both cases here.
 692  */
 693 static struct btrfs_fs_devices *find_fsid_inprogress(
 694                                         struct btrfs_super_block *disk_super)
 695 {
 696         struct btrfs_fs_devices *fs_devices;
 697
 698         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 699                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
 700                            BTRFS_FSID_SIZE) != 0 &&
 701                     memcmp(fs_devices->metadata_uuid, disk_super->fsid,
 702                            BTRFS_FSID_SIZE) == 0 && !fs_devices->fsid_change) {
 703                         return fs_devices;
 704                 }
 705         }
 706
 707         return find_fsid(disk_super->fsid, NULL);
 708 }
 709
 710
 711 static struct btrfs_fs_devices *find_fsid_changed(
 712                                         struct btrfs_super_block *disk_super)
 713 {
 714         struct btrfs_fs_devices *fs_devices;
 715
 716         /*
 717          * Handles the case where scanned device is part of an fs that had
 718          * multiple successful changes of FSID but curently device didn't
 719          * observe it. Meaning our fsid will be different than theirs. We need
 720          * to handle two subcases :
 721          *  1 - The fs still continues to have different METADATA/FSID uuids.
 722          *  2 - The fs is switched back to its original FSID (METADATA/FSID
 723          *  are equal).
 724          */
 725         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 726                 /* Changed UUIDs */
 727                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
 728                            BTRFS_FSID_SIZE) != 0 &&
 729                     memcmp(fs_devices->metadata_uuid, disk_super->metadata_uuid,
 730                            BTRFS_FSID_SIZE) == 0 &&
 731                     memcmp(fs_devices->fsid, disk_super->fsid,
 732                            BTRFS_FSID_SIZE) != 0)
 733                         return fs_devices;
 734
 735                 /* Unchanged UUIDs */
 736                 if (memcmp(fs_devices->metadata_uuid, fs_devices->fsid,
 737                            BTRFS_FSID_SIZE) == 0 &&
 738                     memcmp(fs_devices->fsid, disk_super->metadata_uuid,
 739                            BTRFS_FSID_SIZE) == 0)
 740                         return fs_devices;
 741         }
 742
 743         return NULL;
 744 }
 745
 746 static struct btrfs_fs_devices *find_fsid_reverted_metadata(
 747                                 struct btrfs_super_block *disk_super)
 748 {
 749         struct btrfs_fs_devices *fs_devices;
 750
 751         /*
 752          * Handle the case where the scanned device is part of an fs whose last
 753          * metadata UUID change reverted it to the original FSID. At the same
 754          * time * fs_devices was first created by another constitutent device
 755          * which didn't fully observe the operation. This results in an
 756          * btrfs_fs_devices created with metadata/fsid different AND
 757          * btrfs_fs_devices::fsid_change set AND the metadata_uuid of the
 758          * fs_devices equal to the FSID of the disk.
 759          */
 760         list_for_each_entry(fs_devices, &fs_uuids, fs_list) {
 761                 if (memcmp(fs_devices->fsid, fs_devices->metadata_uuid,
 762                            BTRFS_FSID_SIZE) != 0 &&
 763                     memcmp(fs_devices->metadata_uuid, disk_super->fsid,
 764                            BTRFS_FSID_SIZE) == 0 &&
 765                     fs_devices->fsid_change)
 766                         return fs_devices;
 767         }
 768
 769         return NULL;
 770 }
 771 /*
 772  * Add new device to list of registered devices
 773  *
 774  * Returns:
 775  * device pointer which was just added or updated when successful
 776  * error pointer when failed
 777  */
 778 static noinline struct btrfs_device *device_list_add(const char *path,
 779                            struct btrfs_super_block *disk_super,
 780                            bool *new_device_added)
 781 {
 782         struct btrfs_device *device;
 783         struct btrfs_fs_devices *fs_devices = NULL;
 784         struct rcu_string *name;
 785         u64 found_transid = btrfs_super_generation(disk_super);
 786         u64 devid = btrfs_stack_device_id(&disk_super->dev_item);
 787         bool has_metadata_uuid = (btrfs_super_incompat_flags(disk_super) &
 788                 BTRFS_FEATURE_INCOMPAT_METADATA_UUID);
 789         bool fsid_change_in_progress = (btrfs_super_flags(disk_super) &
 790                                         BTRFS_SUPER_FLAG_CHANGING_FSID_V2);
 791
 792         if (fsid_change_in_progress) {
 793                 if (!has_metadata_uuid)
 794                         fs_devices = find_fsid_inprogress(disk_super);
 795                 else
 796                         fs_devices = find_fsid_changed(disk_super);
 797         } else if (has_metadata_uuid) {
 798                 fs_devices = find_fsid_with_metadata_uuid(disk_super);
 799         } else {
 800                 fs_devices = find_fsid_reverted_metadata(disk_super);
 801                 if (!fs_devices)
 802                         fs_devices = find_fsid(disk_super->fsid, NULL);
 803         }
 804
 805
 806         if (!fs_devices) {
 807                 if (has_metadata_uuid)
 808                         fs_devices = alloc_fs_devices(disk_super->fsid,
 809                                                       disk_super->metadata_uuid);
 810                 else
 811                         fs_devices = alloc_fs_devices(disk_super->fsid, NULL);
 812
 813                 if (IS_ERR(fs_devices))
 814                         return ERR_CAST(fs_devices);
 815
 816                 fs_devices->fsid_change = fsid_change_in_progress;
 817
 818                 mutex_lock(&fs_devices->device_list_mutex);
 819                 list_add(&fs_devices->fs_list, &fs_uuids);
 820
 821                 device = NULL;
 822         } else {
 823                 mutex_lock(&fs_devices->device_list_mutex);
 824                 device = btrfs_find_device(fs_devices, devid,
 825                                 disk_super->dev_item.uuid, NULL, false);
 826
 827                 /*
 828                  * If this disk has been pulled into an fs devices created by
 829                  * a device which had the CHANGING_FSID_V2 flag then replace the
 830                  * metadata_uuid/fsid values of the fs_devices.
 831                  */
 832                 if (fs_devices->fsid_change &&
 833                     found_transid > fs_devices->latest_generation) {
 834                         memcpy(fs_devices->fsid, disk_super->fsid,
 835                                         BTRFS_FSID_SIZE);
 836
 837                         if (has_metadata_uuid)
 838                                 memcpy(fs_devices->metadata_uuid,
 839                                        disk_super->metadata_uuid,
 840                                        BTRFS_FSID_SIZE);
 841                         else
 842                                 memcpy(fs_devices->metadata_uuid,
 843                                        disk_super->fsid, BTRFS_FSID_SIZE);
 844
 845                         fs_devices->fsid_change = false;
 846                 }
 847         }
 848
 849         if (!device) {
 850                 if (fs_devices->opened) {
 851                         mutex_unlock(&fs_devices->device_list_mutex);
 852                         return ERR_PTR(-EBUSY);
 853                 }
 854
 855                 device = btrfs_alloc_device(NULL, &devid,
 856                                             disk_super->dev_item.uuid);
 857                 if (IS_ERR(device)) {
 858                         mutex_unlock(&fs_devices->device_list_mutex);
 859                         /* we can safely leave the fs_devices entry around */
 860                         return device;
 861                 }
 862
 863                 name = rcu_string_strdup(path, GFP_NOFS);
 864                 if (!name) {
 865                         btrfs_free_device(device);
 866                         mutex_unlock(&fs_devices->device_list_mutex);
 867                         return ERR_PTR(-ENOMEM);
 868                 }
 869                 rcu_assign_pointer(device->name, name);
 870
 871                 list_add_rcu(&device->dev_list, &fs_devices->devices);
 872                 fs_devices->num_devices++;
 873
 874                 device->fs_devices = fs_devices;
 875                 *new_device_added = true;
 876
 877                 if (disk_super->label[0])
 878                         pr_info(
 879         "BTRFS: device label %s devid %llu transid %llu %s scanned by %s (%d)\n",
 880                                 disk_super->label, devid, found_transid, path,
 881                                 current->comm, task_pid_nr(current));
 882                 else
 883                         pr_info(
 884         "BTRFS: device fsid %pU devid %llu transid %llu %s scanned by %s (%d)\n",
 885                                 disk_super->fsid, devid, found_transid, path,
 886                                 current->comm, task_pid_nr(current));
 887
 888         } else if (!device->name || strcmp(device->name->str, path)) {
 889                 /*
 890                  * When FS is already mounted.
 891                  * 1. If you are here and if the device->name is NULL that
 892                  *    means this device was missing at time of FS mount.
 893                  * 2. If you are here and if the device->name is different
 894                  *    from 'path' that means either
 895                  *      a. The same device disappeared and reappeared with
 896                  *         different name. or
 897                  *      b. The missing-disk-which-was-replaced, has
 898                  *         reappeared now.
 899                  *
 900                  * We must allow 1 and 2a above. But 2b would be a spurious
 901                  * and unintentional.
 902                  *
 903                  * Further in case of 1 and 2a above, the disk at 'path'
 904                  * would have missed some transaction when it was away and
 905                  * in case of 2a the stale bdev has to be updated as well.
 906                  * 2b must not be allowed at all time.
 907                  */
 908
 909                 /*
 910                  * For now, we do allow update to btrfs_fs_device through the
 911                  * btrfs dev scan cli after FS has been mounted.  We're still
 912                  * tracking a problem where systems fail mount by subvolume id
 913                  * when we reject replacement on a mounted FS.
 914                  */
 915                 if (!fs_devices->opened && found_transid < device->generation) {
 916                         /*
 917                          * That is if the FS is _not_ mounted and if you
 918                          * are here, that means there is more than one
 919                          * disk with same uuid and devid.We keep the one
 920                          * with larger generation number or the last-in if
 921                          * generation are equal.
 922                          */
 923                         mutex_unlock(&fs_devices->device_list_mutex);
 924                         return ERR_PTR(-EEXIST);
 925                 }
 926
 927                 /*
 928                  * We are going to replace the device path for a given devid,
 929                  * make sure it's the same device if the device is mounted
 930                  */
 931                 if (device->bdev) {
 932                         struct block_device *path_bdev;
 933
 934                         path_bdev = lookup_bdev(path);
 935                         if (IS_ERR(path_bdev)) {
 936                                 mutex_unlock(&fs_devices->device_list_mutex);
 937                                 return ERR_CAST(path_bdev);
 938                         }
 939
 940                         if (device->bdev != path_bdev) {
 941                                 bdput(path_bdev);
 942                                 mutex_unlock(&fs_devices->device_list_mutex);
 943                                 btrfs_warn_in_rcu(device->fs_info,
 944         "duplicate device %s devid %llu generation %llu scanned by %s (%d)",
 945                                                   path, devid, found_transid,
 946                                                   current->comm,
 947                                                   task_pid_nr(current));
 948                                 return ERR_PTR(-EEXIST);
 949                         }
 950                         bdput(path_bdev);
 951                         btrfs_info_in_rcu(device->fs_info,
 952         "devid %llu device path %s changed to %s scanned by %s (%d)",
 953                                           devid, rcu_str_deref(device->name),
 954                                           path, current->comm,
 955                                           task_pid_nr(current));
 956                 }
 957
 958                 name = rcu_string_strdup(path, GFP_NOFS);
 959                 if (!name) {
 960                         mutex_unlock(&fs_devices->device_list_mutex);
 961                         return ERR_PTR(-ENOMEM);
 962                 }
 963                 rcu_string_free(device->name);
 964                 rcu_assign_pointer(device->name, name);
 965                 if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
 966                         fs_devices->missing_devices--;
 967                         clear_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
 968                 }
 969         }
 970
 971         /*
 972          * Unmount does not free the btrfs_device struct but would zero
 973          * generation along with most of the other members. So just update
 974          * it back. We need it to pick the disk with largest generation
 975          * (as above).
 976          */
 977         if (!fs_devices->opened) {
 978                 device->generation = found_transid;
 979                 fs_devices->latest_generation = max_t(u64, found_transid,
 980                                                 fs_devices->latest_generation);
 981         }
 982
 983         fs_devices->total_devices = btrfs_super_num_devices(disk_super);
 984
 985         mutex_unlock(&fs_devices->device_list_mutex);
 986         return device;
 987 }
 988
 989 static struct btrfs_fs_devices *clone_fs_devices(struct btrfs_fs_devices *orig)
 990 {
 991         struct btrfs_fs_devices *fs_devices;
 992         struct btrfs_device *device;
 993         struct btrfs_device *orig_dev;
 994         int ret = 0;
 995
 996         fs_devices = alloc_fs_devices(orig->fsid, NULL);
 997         if (IS_ERR(fs_devices))
 998                 return fs_devices;
 999
1000         mutex_lock(&orig->device_list_mutex);
1001         fs_devices->total_devices = orig->total_devices;
1002
1003         list_for_each_entry(orig_dev, &orig->devices, dev_list) {
1004                 struct rcu_string *name;
1005
1006                 device = btrfs_alloc_device(NULL, &orig_dev->devid,
1007                                             orig_dev->uuid);
1008                 if (IS_ERR(device)) {
1009                         ret = PTR_ERR(device);
1010                         goto error;
1011                 }
1012
1013                 /*
1014                  * This is ok to do without rcu read locked because we hold the
1015                  * uuid mutex so nothing we touch in here is going to disappear.
1016                  */
1017                 if (orig_dev->name) {
1018                         name = rcu_string_strdup(orig_dev->name->str,
1019                                         GFP_KERNEL);
1020                         if (!name) {
1021                                 btrfs_free_device(device);
1022                                 ret = -ENOMEM;
1023                                 goto error;
1024                         }
1025                         rcu_assign_pointer(device->name, name);
1026                 }
1027
1028                 list_add(&device->dev_list, &fs_devices->devices);
1029                 device->fs_devices = fs_devices;
1030                 fs_devices->num_devices++;
1031         }
1032         mutex_unlock(&orig->device_list_mutex);
1033         return fs_devices;
1034 error:
1035         mutex_unlock(&orig->device_list_mutex);
1036         free_fs_devices(fs_devices);
1037         return ERR_PTR(ret);
1038 }
1039
1040 static void __btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices,
1041                                       int step, struct btrfs_device **latest_dev)
1042 {
1043         struct btrfs_device *device, *next;
1044
1045         /* This is the initialized path, it is safe to release the devices. */
1046         list_for_each_entry_safe(device, next, &fs_devices->devices, dev_list) {
1047                 if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state)) {
1048                         if (!test_bit(BTRFS_DEV_STATE_REPLACE_TGT,
1049                                       &device->dev_state) &&
1050                             !test_bit(BTRFS_DEV_STATE_MISSING,
1051                                       &device->dev_state) &&
1052                             (!*latest_dev ||
1053                              device->generation > (*latest_dev)->generation)) {
1054                                 *latest_dev = device;
1055                         }
1056                         continue;
1057                 }
1058
1059                 /*
1060                  * We have already validated the presence of BTRFS_DEV_REPLACE_DEVID,
1061                  * in btrfs_init_dev_replace() so just continue.
1062                  */
1063                 if (device->devid == BTRFS_DEV_REPLACE_DEVID)
1064                         continue;
1065
1066                 if (device->bdev) {
1067                         blkdev_put(device->bdev, device->mode);
1068                         device->bdev = NULL;
1069                         fs_devices->open_devices--;
1070                 }
1071                 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1072                         list_del_init(&device->dev_alloc_list);
1073                         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1074                 }
1075                 list_del_init(&device->dev_list);
1076                 fs_devices->num_devices--;
1077                 btrfs_free_device(device);
1078         }
1079
1080 }
1081
1082 /*
1083  * After we have read the system tree and know devids belonging to this
1084  * filesystem, remove the device which does not belong there.
1085  */
1086 void btrfs_free_extra_devids(struct btrfs_fs_devices *fs_devices, int step)
1087 {
1088         struct btrfs_device *latest_dev = NULL;
1089         struct btrfs_fs_devices *seed_dev;
1090
1091         mutex_lock(&uuid_mutex);
1092         __btrfs_free_extra_devids(fs_devices, step, &latest_dev);
1093
1094         list_for_each_entry(seed_dev, &fs_devices->seed_list, seed_list)
1095                 __btrfs_free_extra_devids(seed_dev, step, &latest_dev);
1096
1097         fs_devices->latest_bdev = latest_dev->bdev;
1098
1099         mutex_unlock(&uuid_mutex);
1100 }
1101
1102 static void btrfs_close_bdev(struct btrfs_device *device)
1103 {
1104         if (!device->bdev)
1105                 return;
1106
1107         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
1108                 sync_blockdev(device->bdev);
1109                 invalidate_bdev(device->bdev);
1110         }
1111
1112         blkdev_put(device->bdev, device->mode);
1113 }
1114
1115 static void btrfs_close_one_device(struct btrfs_device *device)
1116 {
1117         struct btrfs_fs_devices *fs_devices = device->fs_devices;
1118
1119         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
1120             device->devid != BTRFS_DEV_REPLACE_DEVID) {
1121                 list_del_init(&device->dev_alloc_list);
1122                 fs_devices->rw_devices--;
1123         }
1124
1125         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
1126                 fs_devices->missing_devices--;
1127
1128         btrfs_close_bdev(device);
1129         if (device->bdev) {
1130                 fs_devices->open_devices--;
1131                 device->bdev = NULL;
1132         }
1133         clear_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
1134
1135         device->fs_info = NULL;
1136         atomic_set(&device->dev_stats_ccnt, 0);
1137         extent_io_tree_release(&device->alloc_state);
1138
1139         /* Verify the device is back in a pristine state  */
1140         ASSERT(!test_bit(BTRFS_DEV_STATE_FLUSH_SENT, &device->dev_state));
1141         ASSERT(!test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1142         ASSERT(list_empty(&device->dev_alloc_list));
1143         ASSERT(list_empty(&device->post_commit_list));
1144         ASSERT(atomic_read(&device->reada_in_flight) == 0);
1145 }
1146
1147 static void close_fs_devices(struct btrfs_fs_devices *fs_devices)
1148 {
1149         struct btrfs_device *device, *tmp;
1150
1151         lockdep_assert_held(&uuid_mutex);
1152
1153         if (--fs_devices->opened > 0)
1154                 return;
1155
1156         list_for_each_entry_safe(device, tmp, &fs_devices->devices, dev_list)
1157                 btrfs_close_one_device(device);
1158
1159         WARN_ON(fs_devices->open_devices);
1160         WARN_ON(fs_devices->rw_devices);
1161         fs_devices->opened = 0;
1162         fs_devices->seeding = false;
1163         fs_devices->fs_info = NULL;
1164 }
1165
1166 void btrfs_close_devices(struct btrfs_fs_devices *fs_devices)
1167 {
1168         LIST_HEAD(list);
1169         struct btrfs_fs_devices *tmp;
1170
1171         mutex_lock(&uuid_mutex);
1172         close_fs_devices(fs_devices);
1173         if (!fs_devices->opened)
1174                 list_splice_init(&fs_devices->seed_list, &list);
1175
1176         list_for_each_entry_safe(fs_devices, tmp, &list, seed_list) {
1177                 close_fs_devices(fs_devices);
1178                 list_del(&fs_devices->seed_list);
1179                 free_fs_devices(fs_devices);
1180         }
1181         mutex_unlock(&uuid_mutex);
1182 }
1183
1184 static int open_fs_devices(struct btrfs_fs_devices *fs_devices,
1185                                 fmode_t flags, void *holder)
1186 {
1187         struct btrfs_device *device;
1188         struct btrfs_device *latest_dev = NULL;
1189         struct btrfs_device *tmp_device;
1190
1191         flags |= FMODE_EXCL;
1192
1193         list_for_each_entry_safe(device, tmp_device, &fs_devices->devices,
1194                                  dev_list) {
1195                 int ret;
1196
1197                 ret = btrfs_open_one_device(fs_devices, device, flags, holder);
1198                 if (ret == 0 &&
1199                     (!latest_dev || device->generation > latest_dev->generation)) {
1200                         latest_dev = device;
1201                 } else if (ret == -ENODATA) {
1202                         fs_devices->num_devices--;
1203                         list_del(&device->dev_list);
1204                         btrfs_free_device(device);
1205                 }
1206         }
1207         if (fs_devices->open_devices == 0)
1208                 return -EINVAL;
1209
1210         fs_devices->opened = 1;
1211         fs_devices->latest_bdev = latest_dev->bdev;
1212         fs_devices->total_rw_bytes = 0;
1213         fs_devices->chunk_alloc_policy = BTRFS_CHUNK_ALLOC_REGULAR;
1214
1215         return 0;
1216 }
1217
1218 static int devid_cmp(void *priv, struct list_head *a, struct list_head *b)
1219 {
1220         struct btrfs_device *dev1, *dev2;
1221
1222         dev1 = list_entry(a, struct btrfs_device, dev_list);
1223         dev2 = list_entry(b, struct btrfs_device, dev_list);
1224
1225         if (dev1->devid < dev2->devid)
1226                 return -1;
1227         else if (dev1->devid > dev2->devid)
1228                 return 1;
1229         return 0;
1230 }
1231
1232 int btrfs_open_devices(struct btrfs_fs_devices *fs_devices,
1233                        fmode_t flags, void *holder)
1234 {
1235         int ret;
1236
1237         lockdep_assert_held(&uuid_mutex);
1238         /*
1239          * The device_list_mutex cannot be taken here in case opening the
1240          * underlying device takes further locks like bd_mutex.
1241          *
1242          * We also don't need the lock here as this is called during mount and
1243          * exclusion is provided by uuid_mutex
1244          */
1245
1246         if (fs_devices->opened) {
1247                 fs_devices->opened++;
1248                 ret = 0;
1249         } else {
1250                 list_sort(NULL, &fs_devices->devices, devid_cmp);
1251                 ret = open_fs_devices(fs_devices, flags, holder);
1252         }
1253
1254         return ret;
1255 }
1256
1257 void btrfs_release_disk_super(struct btrfs_super_block *super)
1258 {
1259         struct page *page = virt_to_page(super);
1260
1261         put_page(page);
1262 }
1263
1264 static struct btrfs_super_block *btrfs_read_disk_super(struct block_device *bdev,
1265                                                        u64 bytenr)
1266 {
1267         struct btrfs_super_block *disk_super;
1268         struct page *page;
1269         void *p;
1270         pgoff_t index;
1271
1272         /* make sure our super fits in the device */
1273         if (bytenr + PAGE_SIZE >= i_size_read(bdev->bd_inode))
1274                 return ERR_PTR(-EINVAL);
1275
1276         /* make sure our super fits in the page */
1277         if (sizeof(*disk_super) > PAGE_SIZE)
1278                 return ERR_PTR(-EINVAL);
1279
1280         /* make sure our super doesn't straddle pages on disk */
1281         index = bytenr >> PAGE_SHIFT;
1282         if ((bytenr + sizeof(*disk_super) - 1) >> PAGE_SHIFT != index)
1283                 return ERR_PTR(-EINVAL);
1284
1285         /* pull in the page with our super */
1286         page = read_cache_page_gfp(bdev->bd_inode->i_mapping, index, GFP_KERNEL);
1287
1288         if (IS_ERR(page))
1289                 return ERR_CAST(page);
1290
1291         p = page_address(page);
1292
1293         /* align our pointer to the offset of the super block */
1294         disk_super = p + offset_in_page(bytenr);
1295
1296         if (btrfs_super_bytenr(disk_super) != bytenr ||
1297             btrfs_super_magic(disk_super) != BTRFS_MAGIC) {
1298                 btrfs_release_disk_super(p);
1299                 return ERR_PTR(-EINVAL);
1300         }
1301
1302         if (disk_super->label[0] && disk_super->label[BTRFS_LABEL_SIZE - 1])
1303                 disk_super->label[BTRFS_LABEL_SIZE - 1] = 0;
1304
1305         return disk_super;
1306 }
1307
1308 int btrfs_forget_devices(const char *path)
1309 {
1310         int ret;
1311
1312         mutex_lock(&uuid_mutex);
1313         ret = btrfs_free_stale_devices(strlen(path) ? path : NULL, NULL);
1314         mutex_unlock(&uuid_mutex);
1315
1316         return ret;
1317 }
1318
1319 /*
1320  * Look for a btrfs signature on a device. This may be called out of the mount path
1321  * and we are not allowed to call set_blocksize during the scan. The superblock
1322  * is read via pagecache
1323  */
1324 struct btrfs_device *btrfs_scan_one_device(const char *path, fmode_t flags,
1325                                            void *holder)
1326 {
1327         struct btrfs_super_block *disk_super;
1328         bool new_device_added = false;
1329         struct btrfs_device *device = NULL;
1330         struct block_device *bdev;
1331         u64 bytenr;
1332
1333         lockdep_assert_held(&uuid_mutex);
1334
1335         /*
1336          * we would like to check all the supers, but that would make
1337          * a btrfs mount succeed after a mkfs from a different FS.
1338          * So, we need to add a special mount option to scan for
1339          * later supers, using BTRFS_SUPER_MIRROR_MAX instead
1340          */
1341         bytenr = btrfs_sb_offset(0);
1342         flags |= FMODE_EXCL;
1343
1344         bdev = blkdev_get_by_path(path, flags, holder);
1345         if (IS_ERR(bdev))
1346                 return ERR_CAST(bdev);
1347
1348         disk_super = btrfs_read_disk_super(bdev, bytenr);
1349         if (IS_ERR(disk_super)) {
1350                 device = ERR_CAST(disk_super);
1351                 goto error_bdev_put;
1352         }
1353
1354         device = device_list_add(path, disk_super, &new_device_added);
1355         if (!IS_ERR(device)) {
1356                 if (new_device_added)
1357                         btrfs_free_stale_devices(path, device);
1358         }
1359
1360         btrfs_release_disk_super(disk_super);
1361
1362 error_bdev_put:
1363         blkdev_put(bdev, flags);
1364
1365         return device;
1366 }
1367
1368 /*
1369  * Try to find a chunk that intersects [start, start + len] range and when one
1370  * such is found, record the end of it in *start
1371  */
1372 static bool contains_pending_extent(struct btrfs_device *device, u64 *start,
1373                                     u64 len)
1374 {
1375         u64 physical_start, physical_end;
1376
1377         lockdep_assert_held(&device->fs_info->chunk_mutex);
1378
1379         if (!find_first_extent_bit(&device->alloc_state, *start,
1380                                    &physical_start, &physical_end,
1381                                    CHUNK_ALLOCATED, NULL)) {
1382
1383                 if (in_range(physical_start, *start, len) ||
1384                     in_range(*start, physical_start,
1385                              physical_end - physical_start)) {
1386                         *start = physical_end + 1;
1387                         return true;
1388                 }
1389         }
1390         return false;
1391 }
1392
1393 static u64 dev_extent_search_start(struct btrfs_device *device, u64 start)
1394 {
1395         switch (device->fs_devices->chunk_alloc_policy) {
1396         case BTRFS_CHUNK_ALLOC_REGULAR:
1397                 /*
1398                  * We don't want to overwrite the superblock on the drive nor
1399                  * any area used by the boot loader (grub for example), so we
1400                  * make sure to start at an offset of at least 1MB.
1401                  */
1402                 return max_t(u64, start, SZ_1M);
1403         default:
1404                 BUG();
1405         }
1406 }
1407
1408 /**
1409  * dev_extent_hole_check - check if specified hole is suitable for allocation
1410  * @device:     the device which we have the hole
1411  * @hole_start: starting position of the hole
1412  * @hole_size:  the size of the hole
1413  * @num_bytes:  the size of the free space that we need
1414  *
1415  * This function may modify @hole_start and @hole_end to reflect the suitable
1416  * position for allocation. Returns 1 if hole position is updated, 0 otherwise.
1417  */
1418 static bool dev_extent_hole_check(struct btrfs_device *device, u64 *hole_start,
1419                                   u64 *hole_size, u64 num_bytes)
1420 {
1421         bool changed = false;
1422         u64 hole_end = *hole_start + *hole_size;
1423
1424         /*
1425          * Check before we set max_hole_start, otherwise we could end up
1426          * sending back this offset anyway.
1427          */
1428         if (contains_pending_extent(device, hole_start, *hole_size)) {
1429                 if (hole_end >= *hole_start)
1430                         *hole_size = hole_end - *hole_start;
1431                 else
1432                         *hole_size = 0;
1433                 changed = true;
1434         }
1435
1436         switch (device->fs_devices->chunk_alloc_policy) {
1437         case BTRFS_CHUNK_ALLOC_REGULAR:
1438                 /* No extra check */
1439                 break;
1440         default:
1441                 BUG();
1442         }
1443
1444         return changed;
1445 }
1446
1447 /*
1448  * find_free_dev_extent_start - find free space in the specified device
1449  * @device:       the device which we search the free space in
1450  * @num_bytes:    the size of the free space that we need
1451  * @search_start: the position from which to begin the search
1452  * @start:        store the start of the free space.
1453  * @len:          the size of the free space. that we find, or the size
1454  *                of the max free space if we don't find suitable free space
1455  *
1456  * this uses a pretty simple search, the expectation is that it is
1457  * called very infrequently and that a given device has a small number
1458  * of extents
1459  *
1460  * @start is used to store the start of the free space if we find. But if we
1461  * don't find suitable free space, it will be used to store the start position
1462  * of the max free space.
1463  *
1464  * @len is used to store the size of the free space that we find.
1465  * But if we don't find suitable free space, it is used to store the size of
1466  * the max free space.
1467  *
1468  * NOTE: This function will search *commit* root of device tree, and does extra
1469  * check to ensure dev extents are not double allocated.
1470  * This makes the function safe to allocate dev extents but may not report
1471  * correct usable device space, as device extent freed in current transaction
1472  * is not reported as avaiable.
1473  */
1474 static int find_free_dev_extent_start(struct btrfs_device *device,
1475                                 u64 num_bytes, u64 search_start, u64 *start,
1476                                 u64 *len)
1477 {
1478         struct btrfs_fs_info *fs_info = device->fs_info;
1479         struct btrfs_root *root = fs_info->dev_root;
1480         struct btrfs_key key;
1481         struct btrfs_dev_extent *dev_extent;
1482         struct btrfs_path *path;
1483         u64 hole_size;
1484         u64 max_hole_start;
1485         u64 max_hole_size;
1486         u64 extent_end;
1487         u64 search_end = device->total_bytes;
1488         int ret;
1489         int slot;
1490         struct extent_buffer *l;
1491
1492         search_start = dev_extent_search_start(device, search_start);
1493
1494         path = btrfs_alloc_path();
1495         if (!path)
1496                 return -ENOMEM;
1497
1498         max_hole_start = search_start;
1499         max_hole_size = 0;
1500
1501 again:
1502         if (search_start >= search_end ||
1503                 test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
1504                 ret = -ENOSPC;
1505                 goto out;
1506         }
1507
1508         path->reada = READA_FORWARD;
1509         path->search_commit_root = 1;
1510         path->skip_locking = 1;
1511
1512         key.objectid = device->devid;
1513         key.offset = search_start;
1514         key.type = BTRFS_DEV_EXTENT_KEY;
1515
1516         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
1517         if (ret < 0)
1518                 goto out;
1519         if (ret > 0) {
1520                 ret = btrfs_previous_item(root, path, key.objectid, key.type);
1521                 if (ret < 0)
1522                         goto out;
1523         }
1524
1525         while (1) {
1526                 l = path->nodes[0];
1527                 slot = path->slots[0];
1528                 if (slot >= btrfs_header_nritems(l)) {
1529                         ret = btrfs_next_leaf(root, path);
1530                         if (ret == 0)
1531                                 continue;
1532                         if (ret < 0)
1533                                 goto out;
1534
1535                         break;
1536                 }
1537                 btrfs_item_key_to_cpu(l, &key, slot);
1538
1539                 if (key.objectid < device->devid)
1540                         goto next;
1541
1542                 if (key.objectid > device->devid)
1543                         break;
1544
1545                 if (key.type != BTRFS_DEV_EXTENT_KEY)
1546                         goto next;
1547
1548                 if (key.offset > search_start) {
1549                         hole_size = key.offset - search_start;
1550                         dev_extent_hole_check(device, &search_start, &hole_size,
1551                                               num_bytes);
1552
1553                         if (hole_size > max_hole_size) {
1554                                 max_hole_start = search_start;
1555                                 max_hole_size = hole_size;
1556                         }
1557
1558                         /*
1559                          * If this free space is greater than which we need,
1560                          * it must be the max free space that we have found
1561                          * until now, so max_hole_start must point to the start
1562                          * of this free space and the length of this free space
1563                          * is stored in max_hole_size. Thus, we return
1564                          * max_hole_start and max_hole_size and go back to the
1565                          * caller.
1566                          */
1567                         if (hole_size >= num_bytes) {
1568                                 ret = 0;
1569                                 goto out;
1570                         }
1571                 }
1572
1573                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
1574                 extent_end = key.offset + btrfs_dev_extent_length(l,
1575                                                                   dev_extent);
1576                 if (extent_end > search_start)
1577                         search_start = extent_end;
1578 next:
1579                 path->slots[0]++;
1580                 cond_resched();
1581         }
1582
1583         /*
1584          * At this point, search_start should be the end of
1585          * allocated dev extents, and when shrinking the device,
1586          * search_end may be smaller than search_start.
1587          */
1588         if (search_end > search_start) {
1589                 hole_size = search_end - search_start;
1590                 if (dev_extent_hole_check(device, &search_start, &hole_size,
1591                                           num_bytes)) {
1592                         btrfs_release_path(path);
1593                         goto again;
1594                 }
1595
1596                 if (hole_size > max_hole_size) {
1597                         max_hole_start = search_start;
1598                         max_hole_size = hole_size;
1599                 }
1600         }
1601
1602         /* See above. */
1603         if (max_hole_size < num_bytes)
1604                 ret = -ENOSPC;
1605         else
1606                 ret = 0;
1607
1608 out:
1609         btrfs_free_path(path);
1610         *start = max_hole_start;
1611         if (len)
1612                 *len = max_hole_size;
1613         return ret;
1614 }
1615
1616 int find_free_dev_extent(struct btrfs_device *device, u64 num_bytes,
1617                          u64 *start, u64 *len)
1618 {
1619         /* FIXME use last free of some kind */
1620         return find_free_dev_extent_start(device, num_bytes, 0, start, len);
1621 }
1622
1623 static int btrfs_free_dev_extent(struct btrfs_trans_handle *trans,
1624                           struct btrfs_device *device,
1625                           u64 start, u64 *dev_extent_len)
1626 {
1627         struct btrfs_fs_info *fs_info = device->fs_info;
1628         struct btrfs_root *root = fs_info->dev_root;
1629         int ret;
1630         struct btrfs_path *path;
1631         struct btrfs_key key;
1632         struct btrfs_key found_key;
1633         struct extent_buffer *leaf = NULL;
1634         struct btrfs_dev_extent *extent = NULL;
1635
1636         path = btrfs_alloc_path();
1637         if (!path)
1638                 return -ENOMEM;
1639
1640         key.objectid = device->devid;
1641         key.offset = start;
1642         key.type = BTRFS_DEV_EXTENT_KEY;
1643 again:
1644         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1645         if (ret > 0) {
1646                 ret = btrfs_previous_item(root, path, key.objectid,
1647                                           BTRFS_DEV_EXTENT_KEY);
1648                 if (ret)
1649                         goto out;
1650                 leaf = path->nodes[0];
1651                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1652                 extent = btrfs_item_ptr(leaf, path->slots[0],
1653                                         struct btrfs_dev_extent);
1654                 BUG_ON(found_key.offset > start || found_key.offset +
1655                        btrfs_dev_extent_length(leaf, extent) < start);
1656                 key = found_key;
1657                 btrfs_release_path(path);
1658                 goto again;
1659         } else if (ret == 0) {
1660                 leaf = path->nodes[0];
1661                 extent = btrfs_item_ptr(leaf, path->slots[0],
1662                                         struct btrfs_dev_extent);
1663         } else {
1664                 btrfs_handle_fs_error(fs_info, ret, "Slot search failed");
1665                 goto out;
1666         }
1667
1668         *dev_extent_len = btrfs_dev_extent_length(leaf, extent);
1669
1670         ret = btrfs_del_item(trans, root, path);
1671         if (ret) {
1672                 btrfs_handle_fs_error(fs_info, ret,
1673                                       "Failed to remove dev extent item");
1674         } else {
1675                 set_bit(BTRFS_TRANS_HAVE_FREE_BGS, &trans->transaction->flags);
1676         }
1677 out:
1678         btrfs_free_path(path);
1679         return ret;
1680 }
1681
1682 static int btrfs_alloc_dev_extent(struct btrfs_trans_handle *trans,
1683                                   struct btrfs_device *device,
1684                                   u64 chunk_offset, u64 start, u64 num_bytes)
1685 {
1686         int ret;
1687         struct btrfs_path *path;
1688         struct btrfs_fs_info *fs_info = device->fs_info;
1689         struct btrfs_root *root = fs_info->dev_root;
1690         struct btrfs_dev_extent *extent;
1691         struct extent_buffer *leaf;
1692         struct btrfs_key key;
1693
1694         WARN_ON(!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state));
1695         WARN_ON(test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state));
1696         path = btrfs_alloc_path();
1697         if (!path)
1698                 return -ENOMEM;
1699
1700         key.objectid = device->devid;
1701         key.offset = start;
1702         key.type = BTRFS_DEV_EXTENT_KEY;
1703         ret = btrfs_insert_empty_item(trans, root, path, &key,
1704                                       sizeof(*extent));
1705         if (ret)
1706                 goto out;
1707
1708         leaf = path->nodes[0];
1709         extent = btrfs_item_ptr(leaf, path->slots[0],
1710                                 struct btrfs_dev_extent);
1711         btrfs_set_dev_extent_chunk_tree(leaf, extent,
1712                                         BTRFS_CHUNK_TREE_OBJECTID);
1713         btrfs_set_dev_extent_chunk_objectid(leaf, extent,
1714                                             BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1715         btrfs_set_dev_extent_chunk_offset(leaf, extent, chunk_offset);
1716
1717         btrfs_set_dev_extent_length(leaf, extent, num_bytes);
1718         btrfs_mark_buffer_dirty(leaf);
1719 out:
1720         btrfs_free_path(path);
1721         return ret;
1722 }
1723
1724 static u64 find_next_chunk(struct btrfs_fs_info *fs_info)
1725 {
1726         struct extent_map_tree *em_tree;
1727         struct extent_map *em;
1728         struct rb_node *n;
1729         u64 ret = 0;
1730
1731         em_tree = &fs_info->mapping_tree;
1732         read_lock(&em_tree->lock);
1733         n = rb_last(&em_tree->map.rb_root);
1734         if (n) {
1735                 em = rb_entry(n, struct extent_map, rb_node);
1736                 ret = em->start + em->len;
1737         }
1738         read_unlock(&em_tree->lock);
1739
1740         return ret;
1741 }
1742
1743 static noinline int find_next_devid(struct btrfs_fs_info *fs_info,
1744                                     u64 *devid_ret)
1745 {
1746         int ret;
1747         struct btrfs_key key;
1748         struct btrfs_key found_key;
1749         struct btrfs_path *path;
1750
1751         path = btrfs_alloc_path();
1752         if (!path)
1753                 return -ENOMEM;
1754
1755         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1756         key.type = BTRFS_DEV_ITEM_KEY;
1757         key.offset = (u64)-1;
1758
1759         ret = btrfs_search_slot(NULL, fs_info->chunk_root, &key, path, 0, 0);
1760         if (ret < 0)
1761                 goto error;
1762
1763         if (ret == 0) {
1764                 /* Corruption */
1765                 btrfs_err(fs_info, "corrupted chunk tree devid -1 matched");
1766                 ret = -EUCLEAN;
1767                 goto error;
1768         }
1769
1770         ret = btrfs_previous_item(fs_info->chunk_root, path,
1771                                   BTRFS_DEV_ITEMS_OBJECTID,
1772                                   BTRFS_DEV_ITEM_KEY);
1773         if (ret) {
1774                 *devid_ret = 1;
1775         } else {
1776                 btrfs_item_key_to_cpu(path->nodes[0], &found_key,
1777                                       path->slots[0]);
1778                 *devid_ret = found_key.offset + 1;
1779         }
1780         ret = 0;
1781 error:
1782         btrfs_free_path(path);
1783         return ret;
1784 }
1785
1786 /*
1787  * the device information is stored in the chunk root
1788  * the btrfs_device struct should be fully filled in
1789  */
1790 static int btrfs_add_dev_item(struct btrfs_trans_handle *trans,
1791                             struct btrfs_device *device)
1792 {
1793         int ret;
1794         struct btrfs_path *path;
1795         struct btrfs_dev_item *dev_item;
1796         struct extent_buffer *leaf;
1797         struct btrfs_key key;
1798         unsigned long ptr;
1799
1800         path = btrfs_alloc_path();
1801         if (!path)
1802                 return -ENOMEM;
1803
1804         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1805         key.type = BTRFS_DEV_ITEM_KEY;
1806         key.offset = device->devid;
1807
1808         ret = btrfs_insert_empty_item(trans, trans->fs_info->chunk_root, path,
1809                                       &key, sizeof(*dev_item));
1810         if (ret)
1811                 goto out;
1812
1813         leaf = path->nodes[0];
1814         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
1815
1816         btrfs_set_device_id(leaf, dev_item, device->devid);
1817         btrfs_set_device_generation(leaf, dev_item, 0);
1818         btrfs_set_device_type(leaf, dev_item, device->type);
1819         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
1820         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
1821         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
1822         btrfs_set_device_total_bytes(leaf, dev_item,
1823                                      btrfs_device_get_disk_total_bytes(device));
1824         btrfs_set_device_bytes_used(leaf, dev_item,
1825                                     btrfs_device_get_bytes_used(device));
1826         btrfs_set_device_group(leaf, dev_item, 0);
1827         btrfs_set_device_seek_speed(leaf, dev_item, 0);
1828         btrfs_set_device_bandwidth(leaf, dev_item, 0);
1829         btrfs_set_device_start_offset(leaf, dev_item, 0);
1830
1831         ptr = btrfs_device_uuid(dev_item);
1832         write_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
1833         ptr = btrfs_device_fsid(dev_item);
1834         write_extent_buffer(leaf, trans->fs_info->fs_devices->metadata_uuid,
1835                             ptr, BTRFS_FSID_SIZE);
1836         btrfs_mark_buffer_dirty(leaf);
1837
1838         ret = 0;
1839 out:
1840         btrfs_free_path(path);
1841         return ret;
1842 }
1843
1844 /*
1845  * Function to update ctime/mtime for a given device path.
1846  * Mainly used for ctime/mtime based probe like libblkid.
1847  */
1848 static void update_dev_time(const char *path_name)
1849 {
1850         struct file *filp;
1851
1852         filp = filp_open(path_name, O_RDWR, 0);
1853         if (IS_ERR(filp))
1854                 return;
1855         file_update_time(filp);
1856         filp_close(filp, NULL);
1857 }
1858
1859 static int btrfs_rm_dev_item(struct btrfs_device *device)
1860 {
1861         struct btrfs_root *root = device->fs_info->chunk_root;
1862         int ret;
1863         struct btrfs_path *path;
1864         struct btrfs_key key;
1865         struct btrfs_trans_handle *trans;
1866
1867         path = btrfs_alloc_path();
1868         if (!path)
1869                 return -ENOMEM;
1870
1871         trans = btrfs_start_transaction(root, 0);
1872         if (IS_ERR(trans)) {
1873                 btrfs_free_path(path);
1874                 return PTR_ERR(trans);
1875         }
1876         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
1877         key.type = BTRFS_DEV_ITEM_KEY;
1878         key.offset = device->devid;
1879
1880         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1881         if (ret) {
1882                 if (ret > 0)
1883                         ret = -ENOENT;
1884                 btrfs_abort_transaction(trans, ret);
1885                 btrfs_end_transaction(trans);
1886                 goto out;
1887         }
1888
1889         ret = btrfs_del_item(trans, root, path);
1890         if (ret) {
1891                 btrfs_abort_transaction(trans, ret);
1892                 btrfs_end_transaction(trans);
1893         }
1894
1895 out:
1896         btrfs_free_path(path);
1897         if (!ret)
1898                 ret = btrfs_commit_transaction(trans);
1899         return ret;
1900 }
1901
1902 /*
1903  * Verify that @num_devices satisfies the RAID profile constraints in the whole
1904  * filesystem. It's up to the caller to adjust that number regarding eg. device
1905  * replace.
1906  */
1907 static int btrfs_check_raid_min_devices(struct btrfs_fs_info *fs_info,
1908                 u64 num_devices)
1909 {
1910         u64 all_avail;
1911         unsigned seq;
1912         int i;
1913
1914         do {
1915                 seq = read_seqbegin(&fs_info->profiles_lock);
1916
1917                 all_avail = fs_info->avail_data_alloc_bits |
1918                             fs_info->avail_system_alloc_bits |
1919                             fs_info->avail_metadata_alloc_bits;
1920         } while (read_seqretry(&fs_info->profiles_lock, seq));
1921
1922         for (i = 0; i < BTRFS_NR_RAID_TYPES; i++) {
1923                 if (!(all_avail & btrfs_raid_array[i].bg_flag))
1924                         continue;
1925
1926                 if (num_devices < btrfs_raid_array[i].devs_min) {
1927                         int ret = btrfs_raid_array[i].mindev_error;
1928
1929                         if (ret)
1930                                 return ret;
1931                 }
1932         }
1933
1934         return 0;
1935 }
1936
1937 static struct btrfs_device * btrfs_find_next_active_device(
1938                 struct btrfs_fs_devices *fs_devs, struct btrfs_device *device)
1939 {
1940         struct btrfs_device *next_device;
1941
1942         list_for_each_entry(next_device, &fs_devs->devices, dev_list) {
1943                 if (next_device != device &&
1944                     !test_bit(BTRFS_DEV_STATE_MISSING, &next_device->dev_state)
1945                     && next_device->bdev)
1946                         return next_device;
1947         }
1948
1949         return NULL;
1950 }
1951
1952 /*
1953  * Helper function to check if the given device is part of s_bdev / latest_bdev
1954  * and replace it with the provided or the next active device, in the context
1955  * where this function called, there should be always be another device (or
1956  * this_dev) which is active.
1957  */
1958 void __cold btrfs_assign_next_active_device(struct btrfs_device *device,
1959                                             struct btrfs_device *next_device)
1960 {
1961         struct btrfs_fs_info *fs_info = device->fs_info;
1962
1963         if (!next_device)
1964                 next_device = btrfs_find_next_active_device(fs_info->fs_devices,
1965                                                             device);
1966         ASSERT(next_device);
1967
1968         if (fs_info->sb->s_bdev &&
1969                         (fs_info->sb->s_bdev == device->bdev))
1970                 fs_info->sb->s_bdev = next_device->bdev;
1971
1972         if (fs_info->fs_devices->latest_bdev == device->bdev)
1973                 fs_info->fs_devices->latest_bdev = next_device->bdev;
1974 }
1975
1976 /*
1977  * Return btrfs_fs_devices::num_devices excluding the device that's being
1978  * currently replaced.
1979  */
1980 static u64 btrfs_num_devices(struct btrfs_fs_info *fs_info)
1981 {
1982         u64 num_devices = fs_info->fs_devices->num_devices;
1983
1984         down_read(&fs_info->dev_replace.rwsem);
1985         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace)) {
1986                 ASSERT(num_devices > 1);
1987                 num_devices--;
1988         }
1989         up_read(&fs_info->dev_replace.rwsem);
1990
1991         return num_devices;
1992 }
1993
1994 void btrfs_scratch_superblocks(struct btrfs_fs_info *fs_info,
1995                                struct block_device *bdev,
1996                                const char *device_path)
1997 {
1998         struct btrfs_super_block *disk_super;
1999         int copy_num;
2000
2001         if (!bdev)
2002                 return;
2003
2004         for (copy_num = 0; copy_num < BTRFS_SUPER_MIRROR_MAX; copy_num++) {
2005                 struct page *page;
2006                 int ret;
2007
2008                 disk_super = btrfs_read_dev_one_super(bdev, copy_num);
2009                 if (IS_ERR(disk_super))
2010                         continue;
2011
2012                 memset(&disk_super->magic, 0, sizeof(disk_super->magic));
2013
2014                 page = virt_to_page(disk_super);
2015                 set_page_dirty(page);
2016                 lock_page(page);
2017                 /* write_on_page() unlocks the page */
2018                 ret = write_one_page(page);
2019                 if (ret)
2020                         btrfs_warn(fs_info,
2021                                 "error clearing superblock number %d (%d)",
2022                                 copy_num, ret);
2023                 btrfs_release_disk_super(disk_super);
2024
2025         }
2026
2027         /* Notify udev that device has changed */
2028         btrfs_kobject_uevent(bdev, KOBJ_CHANGE);
2029
2030         /* Update ctime/mtime for device path for libblkid */
2031         update_dev_time(device_path);
2032 }
2033
2034 int btrfs_rm_device(struct btrfs_fs_info *fs_info, const char *device_path,
2035                     u64 devid)
2036 {
2037         struct btrfs_device *device;
2038         struct btrfs_fs_devices *cur_devices;
2039         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2040         u64 num_devices;
2041         int ret = 0;
2042
2043         mutex_lock(&uuid_mutex);
2044
2045         num_devices = btrfs_num_devices(fs_info);
2046
2047         ret = btrfs_check_raid_min_devices(fs_info, num_devices - 1);
2048         if (ret)
2049                 goto out;
2050
2051         device = btrfs_find_device_by_devspec(fs_info, devid, device_path);
2052
2053         if (IS_ERR(device)) {
2054                 if (PTR_ERR(device) == -ENOENT &&
2055                     strcmp(device_path, "missing") == 0)
2056                         ret = BTRFS_ERROR_DEV_MISSING_NOT_FOUND;
2057                 else
2058                         ret = PTR_ERR(device);
2059                 goto out;
2060         }
2061
2062         if (btrfs_pinned_by_swapfile(fs_info, device)) {
2063                 btrfs_warn_in_rcu(fs_info,
2064                   "cannot remove device %s (devid %llu) due to active swapfile",
2065                                   rcu_str_deref(device->name), device->devid);
2066                 ret = -ETXTBSY;
2067                 goto out;
2068         }
2069
2070         if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2071                 ret = BTRFS_ERROR_DEV_TGT_REPLACE;
2072                 goto out;
2073         }
2074
2075         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
2076             fs_info->fs_devices->rw_devices == 1) {
2077                 ret = BTRFS_ERROR_DEV_ONLY_WRITABLE;
2078                 goto out;
2079         }
2080
2081         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2082                 mutex_lock(&fs_info->chunk_mutex);
2083                 list_del_init(&device->dev_alloc_list);
2084                 device->fs_devices->rw_devices--;
2085                 mutex_unlock(&fs_info->chunk_mutex);
2086         }
2087
2088         mutex_unlock(&uuid_mutex);
2089         ret = btrfs_shrink_device(device, 0);
2090         if (!ret)
2091                 btrfs_reada_remove_dev(device);
2092         mutex_lock(&uuid_mutex);
2093         if (ret)
2094                 goto error_undo;
2095
2096         /*
2097          * TODO: the superblock still includes this device in its num_devices
2098          * counter although write_all_supers() is not locked out. This
2099          * could give a filesystem state which requires a degraded mount.
2100          */
2101         ret = btrfs_rm_dev_item(device);
2102         if (ret)
2103                 goto error_undo;
2104
2105         clear_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2106         btrfs_scrub_cancel_dev(device);
2107
2108         /*
2109          * the device list mutex makes sure that we don't change
2110          * the device list while someone else is writing out all
2111          * the device supers. Whoever is writing all supers, should
2112          * lock the device list mutex before getting the number of
2113          * devices in the super block (super_copy). Conversely,
2114          * whoever updates the number of devices in the super block
2115          * (super_copy) should hold the device list mutex.
2116          */
2117
2118         /*
2119          * In normal cases the cur_devices == fs_devices. But in case
2120          * of deleting a seed device, the cur_devices should point to
2121          * its own fs_devices listed under the fs_devices->seed.
2122          */
2123         cur_devices = device->fs_devices;
2124         mutex_lock(&fs_devices->device_list_mutex);
2125         list_del_rcu(&device->dev_list);
2126
2127         cur_devices->num_devices--;
2128         cur_devices->total_devices--;
2129         /* Update total_devices of the parent fs_devices if it's seed */
2130         if (cur_devices != fs_devices)
2131                 fs_devices->total_devices--;
2132
2133         if (test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state))
2134                 cur_devices->missing_devices--;
2135
2136         btrfs_assign_next_active_device(device, NULL);
2137
2138         if (device->bdev) {
2139                 cur_devices->open_devices--;
2140                 /* remove sysfs entry */
2141                 btrfs_sysfs_remove_device(device);
2142         }
2143
2144         num_devices = btrfs_super_num_devices(fs_info->super_copy) - 1;
2145         btrfs_set_super_num_devices(fs_info->super_copy, num_devices);
2146         mutex_unlock(&fs_devices->device_list_mutex);
2147
2148         /*
2149          * at this point, the device is zero sized and detached from
2150          * the devices list.  All that's left is to zero out the old
2151          * supers and free the device.
2152          */
2153         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2154                 btrfs_scratch_superblocks(fs_info, device->bdev,
2155                                           device->name->str);
2156
2157         btrfs_close_bdev(device);
2158         synchronize_rcu();
2159         btrfs_free_device(device);
2160
2161         if (cur_devices->open_devices == 0) {
2162                 list_del_init(&cur_devices->seed_list);
2163                 close_fs_devices(cur_devices);
2164                 free_fs_devices(cur_devices);
2165         }
2166
2167 out:
2168         mutex_unlock(&uuid_mutex);
2169         return ret;
2170
2171 error_undo:
2172         btrfs_reada_undo_remove_dev(device);
2173         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
2174                 mutex_lock(&fs_info->chunk_mutex);
2175                 list_add(&device->dev_alloc_list,
2176                          &fs_devices->alloc_list);
2177                 device->fs_devices->rw_devices++;
2178                 mutex_unlock(&fs_info->chunk_mutex);
2179         }
2180         goto out;
2181 }
2182
2183 void btrfs_rm_dev_replace_remove_srcdev(struct btrfs_device *srcdev)
2184 {
2185         struct btrfs_fs_devices *fs_devices;
2186
2187         lockdep_assert_held(&srcdev->fs_info->fs_devices->device_list_mutex);
2188
2189         /*
2190          * in case of fs with no seed, srcdev->fs_devices will point
2191          * to fs_devices of fs_info. However when the dev being replaced is
2192          * a seed dev it will point to the seed's local fs_devices. In short
2193          * srcdev will have its correct fs_devices in both the cases.
2194          */
2195         fs_devices = srcdev->fs_devices;
2196
2197         list_del_rcu(&srcdev->dev_list);
2198         list_del(&srcdev->dev_alloc_list);
2199         fs_devices->num_devices--;
2200         if (test_bit(BTRFS_DEV_STATE_MISSING, &srcdev->dev_state))
2201                 fs_devices->missing_devices--;
2202
2203         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &srcdev->dev_state))
2204                 fs_devices->rw_devices--;
2205
2206         if (srcdev->bdev)
2207                 fs_devices->open_devices--;
2208 }
2209
2210 void btrfs_rm_dev_replace_free_srcdev(struct btrfs_device *srcdev)
2211 {
2212         struct btrfs_fs_devices *fs_devices = srcdev->fs_devices;
2213
2214         mutex_lock(&uuid_mutex);
2215
2216         btrfs_close_bdev(srcdev);
2217         synchronize_rcu();
2218         btrfs_free_device(srcdev);
2219
2220         /* if this is no devs we rather delete the fs_devices */
2221         if (!fs_devices->num_devices) {
2222                 /*
2223                  * On a mounted FS, num_devices can't be zero unless it's a
2224                  * seed. In case of a seed device being replaced, the replace
2225                  * target added to the sprout FS, so there will be no more
2226                  * device left under the seed FS.
2227                  */
2228                 ASSERT(fs_devices->seeding);
2229
2230                 list_del_init(&fs_devices->seed_list);
2231                 close_fs_devices(fs_devices);
2232                 free_fs_devices(fs_devices);
2233         }
2234         mutex_unlock(&uuid_mutex);
2235 }
2236
2237 void btrfs_destroy_dev_replace_tgtdev(struct btrfs_device *tgtdev)
2238 {
2239         struct btrfs_fs_devices *fs_devices = tgtdev->fs_info->fs_devices;
2240
2241         mutex_lock(&fs_devices->device_list_mutex);
2242
2243         btrfs_sysfs_remove_device(tgtdev);
2244
2245         if (tgtdev->bdev)
2246                 fs_devices->open_devices--;
2247
2248         fs_devices->num_devices--;
2249
2250         btrfs_assign_next_active_device(tgtdev, NULL);
2251
2252         list_del_rcu(&tgtdev->dev_list);
2253
2254         mutex_unlock(&fs_devices->device_list_mutex);
2255
2256         /*
2257          * The update_dev_time() with in btrfs_scratch_superblocks()
2258          * may lead to a call to btrfs_show_devname() which will try
2259          * to hold device_list_mutex. And here this device
2260          * is already out of device list, so we don't have to hold
2261          * the device_list_mutex lock.
2262          */
2263         btrfs_scratch_superblocks(tgtdev->fs_info, tgtdev->bdev,
2264                                   tgtdev->name->str);
2265
2266         btrfs_close_bdev(tgtdev);
2267         synchronize_rcu();
2268         btrfs_free_device(tgtdev);
2269 }
2270
2271 static struct btrfs_device *btrfs_find_device_by_path(
2272                 struct btrfs_fs_info *fs_info, const char *device_path)
2273 {
2274         int ret = 0;
2275         struct btrfs_super_block *disk_super;
2276         u64 devid;
2277         u8 *dev_uuid;
2278         struct block_device *bdev;
2279         struct btrfs_device *device;
2280
2281         ret = btrfs_get_bdev_and_sb(device_path, FMODE_READ,
2282                                     fs_info->bdev_holder, 0, &bdev, &disk_super);
2283         if (ret)
2284                 return ERR_PTR(ret);
2285
2286         devid = btrfs_stack_device_id(&disk_super->dev_item);
2287         dev_uuid = disk_super->dev_item.uuid;
2288         if (btrfs_fs_incompat(fs_info, METADATA_UUID))
2289                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2290                                            disk_super->metadata_uuid, true);
2291         else
2292                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2293                                            disk_super->fsid, true);
2294
2295         btrfs_release_disk_super(disk_super);
2296         if (!device)
2297                 device = ERR_PTR(-ENOENT);
2298         blkdev_put(bdev, FMODE_READ);
2299         return device;
2300 }
2301
2302 /*
2303  * Lookup a device given by device id, or the path if the id is 0.
2304  */
2305 struct btrfs_device *btrfs_find_device_by_devspec(
2306                 struct btrfs_fs_info *fs_info, u64 devid,
2307                 const char *device_path)
2308 {
2309         struct btrfs_device *device;
2310
2311         if (devid) {
2312                 device = btrfs_find_device(fs_info->fs_devices, devid, NULL,
2313                                            NULL, true);
2314                 if (!device)
2315                         return ERR_PTR(-ENOENT);
2316                 return device;
2317         }
2318
2319         if (!device_path || !device_path[0])
2320                 return ERR_PTR(-EINVAL);
2321
2322         if (strcmp(device_path, "missing") == 0) {
2323                 /* Find first missing device */
2324                 list_for_each_entry(device, &fs_info->fs_devices->devices,
2325                                     dev_list) {
2326                         if (test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
2327                                      &device->dev_state) && !device->bdev)
2328                                 return device;
2329                 }
2330                 return ERR_PTR(-ENOENT);
2331         }
2332
2333         return btrfs_find_device_by_path(fs_info, device_path);
2334 }
2335
2336 /*
2337  * does all the dirty work required for changing file system's UUID.
2338  */
2339 static int btrfs_prepare_sprout(struct btrfs_fs_info *fs_info)
2340 {
2341         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2342         struct btrfs_fs_devices *old_devices;
2343         struct btrfs_fs_devices *seed_devices;
2344         struct btrfs_super_block *disk_super = fs_info->super_copy;
2345         struct btrfs_device *device;
2346         u64 super_flags;
2347
2348         lockdep_assert_held(&uuid_mutex);
2349         if (!fs_devices->seeding)
2350                 return -EINVAL;
2351
2352         /*
2353          * Private copy of the seed devices, anchored at
2354          * fs_info->fs_devices->seed_list
2355          */
2356         seed_devices = alloc_fs_devices(NULL, NULL);
2357         if (IS_ERR(seed_devices))
2358                 return PTR_ERR(seed_devices);
2359
2360         /*
2361          * It's necessary to retain a copy of the original seed fs_devices in
2362          * fs_uuids so that filesystems which have been seeded can successfully
2363          * reference the seed device from open_seed_devices. This also supports
2364          * multiple fs seed.
2365          */
2366         old_devices = clone_fs_devices(fs_devices);
2367         if (IS_ERR(old_devices)) {
2368                 kfree(seed_devices);
2369                 return PTR_ERR(old_devices);
2370         }
2371
2372         list_add(&old_devices->fs_list, &fs_uuids);
2373
2374         memcpy(seed_devices, fs_devices, sizeof(*seed_devices));
2375         seed_devices->opened = 1;
2376         INIT_LIST_HEAD(&seed_devices->devices);
2377         INIT_LIST_HEAD(&seed_devices->alloc_list);
2378         mutex_init(&seed_devices->device_list_mutex);
2379
2380         mutex_lock(&fs_devices->device_list_mutex);
2381         list_splice_init_rcu(&fs_devices->devices, &seed_devices->devices,
2382                               synchronize_rcu);
2383         list_for_each_entry(device, &seed_devices->devices, dev_list)
2384                 device->fs_devices = seed_devices;
2385
2386         fs_devices->seeding = false;
2387         fs_devices->num_devices = 0;
2388         fs_devices->open_devices = 0;
2389         fs_devices->missing_devices = 0;
2390         fs_devices->rotating = false;
2391         list_add(&seed_devices->seed_list, &fs_devices->seed_list);
2392
2393         generate_random_uuid(fs_devices->fsid);
2394         memcpy(fs_devices->metadata_uuid, fs_devices->fsid, BTRFS_FSID_SIZE);
2395         memcpy(disk_super->fsid, fs_devices->fsid, BTRFS_FSID_SIZE);
2396         mutex_unlock(&fs_devices->device_list_mutex);
2397
2398         super_flags = btrfs_super_flags(disk_super) &
2399                       ~BTRFS_SUPER_FLAG_SEEDING;
2400         btrfs_set_super_flags(disk_super, super_flags);
2401
2402         return 0;
2403 }
2404
2405 /*
2406  * Store the expected generation for seed devices in device items.
2407  */
2408 static int btrfs_finish_sprout(struct btrfs_trans_handle *trans)
2409 {
2410         struct btrfs_fs_info *fs_info = trans->fs_info;
2411         struct btrfs_root *root = fs_info->chunk_root;
2412         struct btrfs_path *path;
2413         struct extent_buffer *leaf;
2414         struct btrfs_dev_item *dev_item;
2415         struct btrfs_device *device;
2416         struct btrfs_key key;
2417         u8 fs_uuid[BTRFS_FSID_SIZE];
2418         u8 dev_uuid[BTRFS_UUID_SIZE];
2419         u64 devid;
2420         int ret;
2421
2422         path = btrfs_alloc_path();
2423         if (!path)
2424                 return -ENOMEM;
2425
2426         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2427         key.offset = 0;
2428         key.type = BTRFS_DEV_ITEM_KEY;
2429
2430         while (1) {
2431                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2432                 if (ret < 0)
2433                         goto error;
2434
2435                 leaf = path->nodes[0];
2436 next_slot:
2437                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
2438                         ret = btrfs_next_leaf(root, path);
2439                         if (ret > 0)
2440                                 break;
2441                         if (ret < 0)
2442                                 goto error;
2443                         leaf = path->nodes[0];
2444                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2445                         btrfs_release_path(path);
2446                         continue;
2447                 }
2448
2449                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
2450                 if (key.objectid != BTRFS_DEV_ITEMS_OBJECTID ||
2451                     key.type != BTRFS_DEV_ITEM_KEY)
2452                         break;
2453
2454                 dev_item = btrfs_item_ptr(leaf, path->slots[0],
2455                                           struct btrfs_dev_item);
2456                 devid = btrfs_device_id(leaf, dev_item);
2457                 read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
2458                                    BTRFS_UUID_SIZE);
2459                 read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
2460                                    BTRFS_FSID_SIZE);
2461                 device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
2462                                            fs_uuid, true);
2463                 BUG_ON(!device); /* Logic error */
2464
2465                 if (device->fs_devices->seeding) {
2466                         btrfs_set_device_generation(leaf, dev_item,
2467                                                     device->generation);
2468                         btrfs_mark_buffer_dirty(leaf);
2469                 }
2470
2471                 path->slots[0]++;
2472                 goto next_slot;
2473         }
2474         ret = 0;
2475 error:
2476         btrfs_free_path(path);
2477         return ret;
2478 }
2479
2480 int btrfs_init_new_device(struct btrfs_fs_info *fs_info, const char *device_path)
2481 {
2482         struct btrfs_root *root = fs_info->dev_root;
2483         struct request_queue *q;
2484         struct btrfs_trans_handle *trans;
2485         struct btrfs_device *device;
2486         struct block_device *bdev;
2487         struct super_block *sb = fs_info->sb;
2488         struct rcu_string *name;
2489         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2490         u64 orig_super_total_bytes;
2491         u64 orig_super_num_devices;
2492         int seeding_dev = 0;
2493         int ret = 0;
2494         bool locked = false;
2495
2496         if (sb_rdonly(sb) && !fs_devices->seeding)
2497                 return -EROFS;
2498
2499         bdev = blkdev_get_by_path(device_path, FMODE_WRITE | FMODE_EXCL,
2500                                   fs_info->bdev_holder);
2501         if (IS_ERR(bdev))
2502                 return PTR_ERR(bdev);
2503
2504         if (fs_devices->seeding) {
2505                 seeding_dev = 1;
2506                 down_write(&sb->s_umount);
2507                 mutex_lock(&uuid_mutex);
2508                 locked = true;
2509         }
2510
2511         sync_blockdev(bdev);
2512
2513         rcu_read_lock();
2514         list_for_each_entry_rcu(device, &fs_devices->devices, dev_list) {
2515                 if (device->bdev == bdev) {
2516                         ret = -EEXIST;
2517                         rcu_read_unlock();
2518                         goto error;
2519                 }
2520         }
2521         rcu_read_unlock();
2522
2523         device = btrfs_alloc_device(fs_info, NULL, NULL);
2524         if (IS_ERR(device)) {
2525                 /* we can safely leave the fs_devices entry around */
2526                 ret = PTR_ERR(device);
2527                 goto error;
2528         }
2529
2530         name = rcu_string_strdup(device_path, GFP_KERNEL);
2531         if (!name) {
2532                 ret = -ENOMEM;
2533                 goto error_free_device;
2534         }
2535         rcu_assign_pointer(device->name, name);
2536
2537         trans = btrfs_start_transaction(root, 0);
2538         if (IS_ERR(trans)) {
2539                 ret = PTR_ERR(trans);
2540                 goto error_free_device;
2541         }
2542
2543         q = bdev_get_queue(bdev);
2544         set_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state);
2545         device->generation = trans->transid;
2546         device->io_width = fs_info->sectorsize;
2547         device->io_align = fs_info->sectorsize;
2548         device->sector_size = fs_info->sectorsize;
2549         device->total_bytes = round_down(i_size_read(bdev->bd_inode),
2550                                          fs_info->sectorsize);
2551         device->disk_total_bytes = device->total_bytes;
2552         device->commit_total_bytes = device->total_bytes;
2553         device->fs_info = fs_info;
2554         device->bdev = bdev;
2555         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
2556         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
2557         device->mode = FMODE_EXCL;
2558         device->dev_stats_valid = 1;
2559         set_blocksize(device->bdev, BTRFS_BDEV_BLOCKSIZE);
2560
2561         if (seeding_dev) {
2562                 sb->s_flags &= ~SB_RDONLY;
2563                 ret = btrfs_prepare_sprout(fs_info);
2564                 if (ret) {
2565                         btrfs_abort_transaction(trans, ret);
2566                         goto error_trans;
2567                 }
2568         }
2569
2570         device->fs_devices = fs_devices;
2571
2572         mutex_lock(&fs_devices->device_list_mutex);
2573         mutex_lock(&fs_info->chunk_mutex);
2574         list_add_rcu(&device->dev_list, &fs_devices->devices);
2575         list_add(&device->dev_alloc_list, &fs_devices->alloc_list);
2576         fs_devices->num_devices++;
2577         fs_devices->open_devices++;
2578         fs_devices->rw_devices++;
2579         fs_devices->total_devices++;
2580         fs_devices->total_rw_bytes += device->total_bytes;
2581
2582         atomic64_add(device->total_bytes, &fs_info->free_chunk_space);
2583
2584         if (!blk_queue_nonrot(q))
2585                 fs_devices->rotating = true;
2586
2587         orig_super_total_bytes = btrfs_super_total_bytes(fs_info->super_copy);
2588         btrfs_set_super_total_bytes(fs_info->super_copy,
2589                 round_down(orig_super_total_bytes + device->total_bytes,
2590                            fs_info->sectorsize));
2591
2592         orig_super_num_devices = btrfs_super_num_devices(fs_info->super_copy);
2593         btrfs_set_super_num_devices(fs_info->super_copy,
2594                                     orig_super_num_devices + 1);
2595
2596         /*
2597          * we've got more storage, clear any full flags on the space
2598          * infos
2599          */
2600         btrfs_clear_space_info_full(fs_info);
2601
2602         mutex_unlock(&fs_info->chunk_mutex);
2603
2604         /* Add sysfs device entry */
2605         btrfs_sysfs_add_device(device);
2606
2607         mutex_unlock(&fs_devices->device_list_mutex);
2608
2609         if (seeding_dev) {
2610                 mutex_lock(&fs_info->chunk_mutex);
2611                 ret = init_first_rw_device(trans);
2612                 mutex_unlock(&fs_info->chunk_mutex);
2613                 if (ret) {
2614                         btrfs_abort_transaction(trans, ret);
2615                         goto error_sysfs;
2616                 }
2617         }
2618
2619         ret = btrfs_add_dev_item(trans, device);
2620         if (ret) {
2621                 btrfs_abort_transaction(trans, ret);
2622                 goto error_sysfs;
2623         }
2624
2625         if (seeding_dev) {
2626                 ret = btrfs_finish_sprout(trans);
2627                 if (ret) {
2628                         btrfs_abort_transaction(trans, ret);
2629                         goto error_sysfs;
2630                 }
2631
2632                 /*
2633                  * fs_devices now represents the newly sprouted filesystem and
2634                  * its fsid has been changed by btrfs_prepare_sprout
2635                  */
2636                 btrfs_sysfs_update_sprout_fsid(fs_devices);
2637         }
2638
2639         ret = btrfs_commit_transaction(trans);
2640
2641         if (seeding_dev) {
2642                 mutex_unlock(&uuid_mutex);
2643                 up_write(&sb->s_umount);
2644                 locked = false;
2645
2646                 if (ret) /* transaction commit */
2647                         return ret;
2648
2649                 ret = btrfs_relocate_sys_chunks(fs_info);
2650                 if (ret < 0)
2651                         btrfs_handle_fs_error(fs_info, ret,
2652                                     "Failed to relocate sys chunks after device initialization. This can be fixed using the \"btrfs balance\" command.");
2653                 trans = btrfs_attach_transaction(root);
2654                 if (IS_ERR(trans)) {
2655                         if (PTR_ERR(trans) == -ENOENT)
2656                                 return 0;
2657                         ret = PTR_ERR(trans);
2658                         trans = NULL;
2659                         goto error_sysfs;
2660                 }
2661                 ret = btrfs_commit_transaction(trans);
2662         }
2663
2664         /*
2665          * Now that we have written a new super block to this device, check all
2666          * other fs_devices list if device_path alienates any other scanned
2667          * device.
2668          * We can ignore the return value as it typically returns -EINVAL and
2669          * only succeeds if the device was an alien.
2670          */
2671         btrfs_forget_devices(device_path);
2672
2673         /* Update ctime/mtime for blkid or udev */
2674         update_dev_time(device_path);
2675
2676         return ret;
2677
2678 error_sysfs:
2679         btrfs_sysfs_remove_device(device);
2680         mutex_lock(&fs_info->fs_devices->device_list_mutex);
2681         mutex_lock(&fs_info->chunk_mutex);
2682         list_del_rcu(&device->dev_list);
2683         list_del(&device->dev_alloc_list);
2684         fs_info->fs_devices->num_devices--;
2685         fs_info->fs_devices->open_devices--;
2686         fs_info->fs_devices->rw_devices--;
2687         fs_info->fs_devices->total_devices--;
2688         fs_info->fs_devices->total_rw_bytes -= device->total_bytes;
2689         atomic64_sub(device->total_bytes, &fs_info->free_chunk_space);
2690         btrfs_set_super_total_bytes(fs_info->super_copy,
2691                                     orig_super_total_bytes);
2692         btrfs_set_super_num_devices(fs_info->super_copy,
2693                                     orig_super_num_devices);
2694         mutex_unlock(&fs_info->chunk_mutex);
2695         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
2696 error_trans:
2697         if (seeding_dev)
2698                 sb->s_flags |= SB_RDONLY;
2699         if (trans)
2700                 btrfs_end_transaction(trans);
2701 error_free_device:
2702         btrfs_free_device(device);
2703 error:
2704         blkdev_put(bdev, FMODE_EXCL);
2705         if (locked) {
2706                 mutex_unlock(&uuid_mutex);
2707                 up_write(&sb->s_umount);
2708         }
2709         return ret;
2710 }
2711
2712 static noinline int btrfs_update_device(struct btrfs_trans_handle *trans,
2713                                         struct btrfs_device *device)
2714 {
2715         int ret;
2716         struct btrfs_path *path;
2717         struct btrfs_root *root = device->fs_info->chunk_root;
2718         struct btrfs_dev_item *dev_item;
2719         struct extent_buffer *leaf;
2720         struct btrfs_key key;
2721
2722         path = btrfs_alloc_path();
2723         if (!path)
2724                 return -ENOMEM;
2725
2726         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
2727         key.type = BTRFS_DEV_ITEM_KEY;
2728         key.offset = device->devid;
2729
2730         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2731         if (ret < 0)
2732                 goto out;
2733
2734         if (ret > 0) {
2735                 ret = -ENOENT;
2736                 goto out;
2737         }
2738
2739         leaf = path->nodes[0];
2740         dev_item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_dev_item);
2741
2742         btrfs_set_device_id(leaf, dev_item, device->devid);
2743         btrfs_set_device_type(leaf, dev_item, device->type);
2744         btrfs_set_device_io_align(leaf, dev_item, device->io_align);
2745         btrfs_set_device_io_width(leaf, dev_item, device->io_width);
2746         btrfs_set_device_sector_size(leaf, dev_item, device->sector_size);
2747         btrfs_set_device_total_bytes(leaf, dev_item,
2748                                      btrfs_device_get_disk_total_bytes(device));
2749         btrfs_set_device_bytes_used(leaf, dev_item,
2750                                     btrfs_device_get_bytes_used(device));
2751         btrfs_mark_buffer_dirty(leaf);
2752
2753 out:
2754         btrfs_free_path(path);
2755         return ret;
2756 }
2757
2758 int btrfs_grow_device(struct btrfs_trans_handle *trans,
2759                       struct btrfs_device *device, u64 new_size)
2760 {
2761         struct btrfs_fs_info *fs_info = device->fs_info;
2762         struct btrfs_super_block *super_copy = fs_info->super_copy;
2763         u64 old_total;
2764         u64 diff;
2765
2766         if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
2767                 return -EACCES;
2768
2769         new_size = round_down(new_size, fs_info->sectorsize);
2770
2771         mutex_lock(&fs_info->chunk_mutex);
2772         old_total = btrfs_super_total_bytes(super_copy);
2773         diff = round_down(new_size - device->total_bytes, fs_info->sectorsize);
2774
2775         if (new_size <= device->total_bytes ||
2776             test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
2777                 mutex_unlock(&fs_info->chunk_mutex);
2778                 return -EINVAL;
2779         }
2780
2781         btrfs_set_super_total_bytes(super_copy,
2782                         round_down(old_total + diff, fs_info->sectorsize));
2783         device->fs_devices->total_rw_bytes += diff;
2784
2785         btrfs_device_set_total_bytes(device, new_size);
2786         btrfs_device_set_disk_total_bytes(device, new_size);
2787         btrfs_clear_space_info_full(device->fs_info);
2788         if (list_empty(&device->post_commit_list))
2789                 list_add_tail(&device->post_commit_list,
2790                               &trans->transaction->dev_update_list);
2791         mutex_unlock(&fs_info->chunk_mutex);
2792
2793         return btrfs_update_device(trans, device);
2794 }
2795
2796 static int btrfs_free_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2797 {
2798         struct btrfs_fs_info *fs_info = trans->fs_info;
2799         struct btrfs_root *root = fs_info->chunk_root;
2800         int ret;
2801         struct btrfs_path *path;
2802         struct btrfs_key key;
2803
2804         path = btrfs_alloc_path();
2805         if (!path)
2806                 return -ENOMEM;
2807
2808         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
2809         key.offset = chunk_offset;
2810         key.type = BTRFS_CHUNK_ITEM_KEY;
2811
2812         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
2813         if (ret < 0)
2814                 goto out;
2815         else if (ret > 0) { /* Logic error or corruption */
2816                 btrfs_handle_fs_error(fs_info, -ENOENT,
2817                                       "Failed lookup while freeing chunk.");
2818                 ret = -ENOENT;
2819                 goto out;
2820         }
2821
2822         ret = btrfs_del_item(trans, root, path);
2823         if (ret < 0)
2824                 btrfs_handle_fs_error(fs_info, ret,
2825                                       "Failed to delete chunk item.");
2826 out:
2827         btrfs_free_path(path);
2828         return ret;
2829 }
2830
2831 static int btrfs_del_sys_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
2832 {
2833         struct btrfs_super_block *super_copy = fs_info->super_copy;
2834         struct btrfs_disk_key *disk_key;
2835         struct btrfs_chunk *chunk;
2836         u8 *ptr;
2837         int ret = 0;
2838         u32 num_stripes;
2839         u32 array_size;
2840         u32 len = 0;
2841         u32 cur;
2842         struct btrfs_key key;
2843
2844         mutex_lock(&fs_info->chunk_mutex);
2845         array_size = btrfs_super_sys_array_size(super_copy);
2846
2847         ptr = super_copy->sys_chunk_array;
2848         cur = 0;
2849
2850         while (cur < array_size) {
2851                 disk_key = (struct btrfs_disk_key *)ptr;
2852                 btrfs_disk_key_to_cpu(&key, disk_key);
2853
2854                 len = sizeof(*disk_key);
2855
2856                 if (key.type == BTRFS_CHUNK_ITEM_KEY) {
2857                         chunk = (struct btrfs_chunk *)(ptr + len);
2858                         num_stripes = btrfs_stack_chunk_num_stripes(chunk);
2859                         len += btrfs_chunk_item_size(num_stripes);
2860                 } else {
2861                         ret = -EIO;
2862                         break;
2863                 }
2864                 if (key.objectid == BTRFS_FIRST_CHUNK_TREE_OBJECTID &&
2865                     key.offset == chunk_offset) {
2866                         memmove(ptr, ptr + len, array_size - (cur + len));
2867                         array_size -= len;
2868                         btrfs_set_super_sys_array_size(super_copy, array_size);
2869                 } else {
2870                         ptr += len;
2871                         cur += len;
2872                 }
2873         }
2874         mutex_unlock(&fs_info->chunk_mutex);
2875         return ret;
2876 }
2877
2878 /*
2879  * btrfs_get_chunk_map() - Find the mapping containing the given logical extent.
2880  * @logical: Logical block offset in bytes.
2881  * @length: Length of extent in bytes.
2882  *
2883  * Return: Chunk mapping or ERR_PTR.
2884  */
2885 struct extent_map *btrfs_get_chunk_map(struct btrfs_fs_info *fs_info,
2886                                        u64 logical, u64 length)
2887 {
2888         struct extent_map_tree *em_tree;
2889         struct extent_map *em;
2890
2891         em_tree = &fs_info->mapping_tree;
2892         read_lock(&em_tree->lock);
2893         em = lookup_extent_mapping(em_tree, logical, length);
2894         read_unlock(&em_tree->lock);
2895
2896         if (!em) {
2897                 btrfs_crit(fs_info, "unable to find logical %llu length %llu",
2898                            logical, length);
2899                 return ERR_PTR(-EINVAL);
2900         }
2901
2902         if (em->start > logical || em->start + em->len < logical) {
2903                 btrfs_crit(fs_info,
2904                            "found a bad mapping, wanted %llu-%llu, found %llu-%llu",
2905                            logical, length, em->start, em->start + em->len);
2906                 free_extent_map(em);
2907                 return ERR_PTR(-EINVAL);
2908         }
2909
2910         /* callers are responsible for dropping em's ref. */
2911         return em;
2912 }
2913
2914 int btrfs_remove_chunk(struct btrfs_trans_handle *trans, u64 chunk_offset)
2915 {
2916         struct btrfs_fs_info *fs_info = trans->fs_info;
2917         struct extent_map *em;
2918         struct map_lookup *map;
2919         u64 dev_extent_len = 0;
2920         int i, ret = 0;
2921         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
2922
2923         em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
2924         if (IS_ERR(em)) {
2925                 /*
2926                  * This is a logic error, but we don't want to just rely on the
2927                  * user having built with ASSERT enabled, so if ASSERT doesn't
2928                  * do anything we still error out.
2929                  */
2930                 ASSERT(0);
2931                 return PTR_ERR(em);
2932         }
2933         map = em->map_lookup;
2934         mutex_lock(&fs_info->chunk_mutex);
2935         check_system_chunk(trans, map->type);
2936         mutex_unlock(&fs_info->chunk_mutex);
2937
2938         /*
2939          * Take the device list mutex to prevent races with the final phase of
2940          * a device replace operation that replaces the device object associated
2941          * with map stripes (dev-replace.c:btrfs_dev_replace_finishing()).
2942          */
2943         mutex_lock(&fs_devices->device_list_mutex);
2944         for (i = 0; i < map->num_stripes; i++) {
2945                 struct btrfs_device *device = map->stripes[i].dev;
2946                 ret = btrfs_free_dev_extent(trans, device,
2947                                             map->stripes[i].physical,
2948                                             &dev_extent_len);
2949                 if (ret) {
2950                         mutex_unlock(&fs_devices->device_list_mutex);
2951                         btrfs_abort_transaction(trans, ret);
2952                         goto out;
2953                 }
2954
2955                 if (device->bytes_used > 0) {
2956                         mutex_lock(&fs_info->chunk_mutex);
2957                         btrfs_device_set_bytes_used(device,
2958                                         device->bytes_used - dev_extent_len);
2959                         atomic64_add(dev_extent_len, &fs_info->free_chunk_space);
2960                         btrfs_clear_space_info_full(fs_info);
2961                         mutex_unlock(&fs_info->chunk_mutex);
2962                 }
2963
2964                 ret = btrfs_update_device(trans, device);
2965                 if (ret) {
2966                         mutex_unlock(&fs_devices->device_list_mutex);
2967                         btrfs_abort_transaction(trans, ret);
2968                         goto out;
2969                 }
2970         }
2971         mutex_unlock(&fs_devices->device_list_mutex);
2972
2973         ret = btrfs_free_chunk(trans, chunk_offset);
2974         if (ret) {
2975                 btrfs_abort_transaction(trans, ret);
2976                 goto out;
2977         }
2978
2979         trace_btrfs_chunk_free(fs_info, map, chunk_offset, em->len);
2980
2981         if (map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
2982                 ret = btrfs_del_sys_chunk(fs_info, chunk_offset);
2983                 if (ret) {
2984                         btrfs_abort_transaction(trans, ret);
2985                         goto out;
2986                 }
2987         }
2988
2989         ret = btrfs_remove_block_group(trans, chunk_offset, em);
2990         if (ret) {
2991                 btrfs_abort_transaction(trans, ret);
2992                 goto out;
2993         }
2994
2995 out:
2996         /* once for us */
2997         free_extent_map(em);
2998         return ret;
2999 }
3000
3001 static int btrfs_relocate_chunk(struct btrfs_fs_info *fs_info, u64 chunk_offset)
3002 {
3003         struct btrfs_root *root = fs_info->chunk_root;
3004         struct btrfs_trans_handle *trans;
3005         struct btrfs_block_group *block_group;
3006         int ret;
3007
3008         /*
3009          * Prevent races with automatic removal of unused block groups.
3010          * After we relocate and before we remove the chunk with offset
3011          * chunk_offset, automatic removal of the block group can kick in,
3012          * resulting in a failure when calling btrfs_remove_chunk() below.
3013          *
3014          * Make sure to acquire this mutex before doing a tree search (dev
3015          * or chunk trees) to find chunks. Otherwise the cleaner kthread might
3016          * call btrfs_remove_chunk() (through btrfs_delete_unused_bgs()) after
3017          * we release the path used to search the chunk/dev tree and before
3018          * the current task acquires this mutex and calls us.
3019          */
3020         lockdep_assert_held(&fs_info->delete_unused_bgs_mutex);
3021
3022         /* step one, relocate all the extents inside this chunk */
3023         btrfs_scrub_pause(fs_info);
3024         ret = btrfs_relocate_block_group(fs_info, chunk_offset);
3025         btrfs_scrub_continue(fs_info);
3026         if (ret)
3027                 return ret;
3028
3029         block_group = btrfs_lookup_block_group(fs_info, chunk_offset);
3030         if (!block_group)
3031                 return -ENOENT;
3032         btrfs_discard_cancel_work(&fs_info->discard_ctl, block_group);
3033         btrfs_put_block_group(block_group);
3034
3035         trans = btrfs_start_trans_remove_block_group(root->fs_info,
3036                                                      chunk_offset);
3037         if (IS_ERR(trans)) {
3038                 ret = PTR_ERR(trans);
3039                 btrfs_handle_fs_error(root->fs_info, ret, NULL);
3040                 return ret;
3041         }
3042
3043         /*
3044          * step two, delete the device extents and the
3045          * chunk tree entries
3046          */
3047         ret = btrfs_remove_chunk(trans, chunk_offset);
3048         btrfs_end_transaction(trans);
3049         return ret;
3050 }
3051
3052 static int btrfs_relocate_sys_chunks(struct btrfs_fs_info *fs_info)
3053 {
3054         struct btrfs_root *chunk_root = fs_info->chunk_root;
3055         struct btrfs_path *path;
3056         struct extent_buffer *leaf;
3057         struct btrfs_chunk *chunk;
3058         struct btrfs_key key;
3059         struct btrfs_key found_key;
3060         u64 chunk_type;
3061         bool retried = false;
3062         int failed = 0;
3063         int ret;
3064
3065         path = btrfs_alloc_path();
3066         if (!path)
3067                 return -ENOMEM;
3068
3069 again:
3070         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3071         key.offset = (u64)-1;
3072         key.type = BTRFS_CHUNK_ITEM_KEY;
3073
3074         while (1) {
3075                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
3076                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3077                 if (ret < 0) {
3078                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3079                         goto error;
3080                 }
3081                 BUG_ON(ret == 0); /* Corruption */
3082
3083                 ret = btrfs_previous_item(chunk_root, path, key.objectid,
3084                                           key.type);
3085                 if (ret)
3086                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3087                 if (ret < 0)
3088                         goto error;
3089                 if (ret > 0)
3090                         break;
3091
3092                 leaf = path->nodes[0];
3093                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3094
3095                 chunk = btrfs_item_ptr(leaf, path->slots[0],
3096                                        struct btrfs_chunk);
3097                 chunk_type = btrfs_chunk_type(leaf, chunk);
3098                 btrfs_release_path(path);
3099
3100                 if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) {
3101                         ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3102                         if (ret == -ENOSPC)
3103                                 failed++;
3104                         else
3105                                 BUG_ON(ret);
3106                 }
3107                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3108
3109                 if (found_key.offset == 0)
3110                         break;
3111                 key.offset = found_key.offset - 1;
3112         }
3113         ret = 0;
3114         if (failed && !retried) {
3115                 failed = 0;
3116                 retried = true;
3117                 goto again;
3118         } else if (WARN_ON(failed && retried)) {
3119                 ret = -ENOSPC;
3120         }
3121 error:
3122         btrfs_free_path(path);
3123         return ret;
3124 }
3125
3126 /*
3127  * return 1 : allocate a data chunk successfully,
3128  * return <0: errors during allocating a data chunk,
3129  * return 0 : no need to allocate a data chunk.
3130  */
3131 static int btrfs_may_alloc_data_chunk(struct btrfs_fs_info *fs_info,
3132                                       u64 chunk_offset)
3133 {
3134         struct btrfs_block_group *cache;
3135         u64 bytes_used;
3136         u64 chunk_type;
3137
3138         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3139         ASSERT(cache);
3140         chunk_type = cache->flags;
3141         btrfs_put_block_group(cache);
3142
3143         if (!(chunk_type & BTRFS_BLOCK_GROUP_DATA))
3144                 return 0;
3145
3146         spin_lock(&fs_info->data_sinfo->lock);
3147         bytes_used = fs_info->data_sinfo->bytes_used;
3148         spin_unlock(&fs_info->data_sinfo->lock);
3149
3150         if (!bytes_used) {
3151                 struct btrfs_trans_handle *trans;
3152                 int ret;
3153
3154                 trans = btrfs_join_transaction(fs_info->tree_root);
3155                 if (IS_ERR(trans))
3156                         return PTR_ERR(trans);
3157
3158                 ret = btrfs_force_chunk_alloc(trans, BTRFS_BLOCK_GROUP_DATA);
3159                 btrfs_end_transaction(trans);
3160                 if (ret < 0)
3161                         return ret;
3162                 return 1;
3163         }
3164
3165         return 0;
3166 }
3167
3168 static int insert_balance_item(struct btrfs_fs_info *fs_info,
3169                                struct btrfs_balance_control *bctl)
3170 {
3171         struct btrfs_root *root = fs_info->tree_root;
3172         struct btrfs_trans_handle *trans;
3173         struct btrfs_balance_item *item;
3174         struct btrfs_disk_balance_args disk_bargs;
3175         struct btrfs_path *path;
3176         struct extent_buffer *leaf;
3177         struct btrfs_key key;
3178         int ret, err;
3179
3180         path = btrfs_alloc_path();
3181         if (!path)
3182                 return -ENOMEM;
3183
3184         trans = btrfs_start_transaction(root, 0);
3185         if (IS_ERR(trans)) {
3186                 btrfs_free_path(path);
3187                 return PTR_ERR(trans);
3188         }
3189
3190         key.objectid = BTRFS_BALANCE_OBJECTID;
3191         key.type = BTRFS_TEMPORARY_ITEM_KEY;
3192         key.offset = 0;
3193
3194         ret = btrfs_insert_empty_item(trans, root, path, &key,
3195                                       sizeof(*item));
3196         if (ret)
3197                 goto out;
3198
3199         leaf = path->nodes[0];
3200         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
3201
3202         memzero_extent_buffer(leaf, (unsigned long)item, sizeof(*item));
3203
3204         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->data);
3205         btrfs_set_balance_data(leaf, item, &disk_bargs);
3206         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->meta);
3207         btrfs_set_balance_meta(leaf, item, &disk_bargs);
3208         btrfs_cpu_balance_args_to_disk(&disk_bargs, &bctl->sys);
3209         btrfs_set_balance_sys(leaf, item, &disk_bargs);
3210
3211         btrfs_set_balance_flags(leaf, item, bctl->flags);
3212
3213         btrfs_mark_buffer_dirty(leaf);
3214 out:
3215         btrfs_free_path(path);
3216         err = btrfs_commit_transaction(trans);
3217         if (err && !ret)
3218                 ret = err;
3219         return ret;
3220 }
3221
3222 static int del_balance_item(struct btrfs_fs_info *fs_info)
3223 {
3224         struct btrfs_root *root = fs_info->tree_root;
3225         struct btrfs_trans_handle *trans;
3226         struct btrfs_path *path;
3227         struct btrfs_key key;
3228         int ret, err;
3229
3230         path = btrfs_alloc_path();
3231         if (!path)
3232                 return -ENOMEM;
3233
3234         trans = btrfs_start_transaction_fallback_global_rsv(root, 0);
3235         if (IS_ERR(trans)) {
3236                 btrfs_free_path(path);
3237                 return PTR_ERR(trans);
3238         }
3239
3240         key.objectid = BTRFS_BALANCE_OBJECTID;
3241         key.type = BTRFS_TEMPORARY_ITEM_KEY;
3242         key.offset = 0;
3243
3244         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
3245         if (ret < 0)
3246                 goto out;
3247         if (ret > 0) {
3248                 ret = -ENOENT;
3249                 goto out;
3250         }
3251
3252         ret = btrfs_del_item(trans, root, path);
3253 out:
3254         btrfs_free_path(path);
3255         err = btrfs_commit_transaction(trans);
3256         if (err && !ret)
3257                 ret = err;
3258         return ret;
3259 }
3260
3261 /*
3262  * This is a heuristic used to reduce the number of chunks balanced on
3263  * resume after balance was interrupted.
3264  */
3265 static void update_balance_args(struct btrfs_balance_control *bctl)
3266 {
3267         /*
3268          * Turn on soft mode for chunk types that were being converted.
3269          */
3270         if (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)
3271                 bctl->data.flags |= BTRFS_BALANCE_ARGS_SOFT;
3272         if (bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)
3273                 bctl->sys.flags |= BTRFS_BALANCE_ARGS_SOFT;
3274         if (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)
3275                 bctl->meta.flags |= BTRFS_BALANCE_ARGS_SOFT;
3276
3277         /*
3278          * Turn on usage filter if is not already used.  The idea is
3279          * that chunks that we have already balanced should be
3280          * reasonably full.  Don't do it for chunks that are being
3281          * converted - that will keep us from relocating unconverted
3282          * (albeit full) chunks.
3283          */
3284         if (!(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3285             !(bctl->data.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3286             !(bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3287                 bctl->data.flags |= BTRFS_BALANCE_ARGS_USAGE;
3288                 bctl->data.usage = 90;
3289         }
3290         if (!(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3291             !(bctl->sys.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3292             !(bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3293                 bctl->sys.flags |= BTRFS_BALANCE_ARGS_USAGE;
3294                 bctl->sys.usage = 90;
3295         }
3296         if (!(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE) &&
3297             !(bctl->meta.flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3298             !(bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT)) {
3299                 bctl->meta.flags |= BTRFS_BALANCE_ARGS_USAGE;
3300                 bctl->meta.usage = 90;
3301         }
3302 }
3303
3304 /*
3305  * Clear the balance status in fs_info and delete the balance item from disk.
3306  */
3307 static void reset_balance_state(struct btrfs_fs_info *fs_info)
3308 {
3309         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3310         int ret;
3311
3312         BUG_ON(!fs_info->balance_ctl);
3313
3314         spin_lock(&fs_info->balance_lock);
3315         fs_info->balance_ctl = NULL;
3316         spin_unlock(&fs_info->balance_lock);
3317
3318         kfree(bctl);
3319         ret = del_balance_item(fs_info);
3320         if (ret)
3321                 btrfs_handle_fs_error(fs_info, ret, NULL);
3322 }
3323
3324 /*
3325  * Balance filters.  Return 1 if chunk should be filtered out
3326  * (should not be balanced).
3327  */
3328 static int chunk_profiles_filter(u64 chunk_type,
3329                                  struct btrfs_balance_args *bargs)
3330 {
3331         chunk_type = chunk_to_extended(chunk_type) &
3332                                 BTRFS_EXTENDED_PROFILE_MASK;
3333
3334         if (bargs->profiles & chunk_type)
3335                 return 0;
3336
3337         return 1;
3338 }
3339
3340 static int chunk_usage_range_filter(struct btrfs_fs_info *fs_info, u64 chunk_offset,
3341                               struct btrfs_balance_args *bargs)
3342 {
3343         struct btrfs_block_group *cache;
3344         u64 chunk_used;
3345         u64 user_thresh_min;
3346         u64 user_thresh_max;
3347         int ret = 1;
3348
3349         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3350         chunk_used = cache->used;
3351
3352         if (bargs->usage_min == 0)
3353                 user_thresh_min = 0;
3354         else
3355                 user_thresh_min = div_factor_fine(cache->length,
3356                                                   bargs->usage_min);
3357
3358         if (bargs->usage_max == 0)
3359                 user_thresh_max = 1;
3360         else if (bargs->usage_max > 100)
3361                 user_thresh_max = cache->length;
3362         else
3363                 user_thresh_max = div_factor_fine(cache->length,
3364                                                   bargs->usage_max);
3365
3366         if (user_thresh_min <= chunk_used && chunk_used < user_thresh_max)
3367                 ret = 0;
3368
3369         btrfs_put_block_group(cache);
3370         return ret;
3371 }
3372
3373 static int chunk_usage_filter(struct btrfs_fs_info *fs_info,
3374                 u64 chunk_offset, struct btrfs_balance_args *bargs)
3375 {
3376         struct btrfs_block_group *cache;
3377         u64 chunk_used, user_thresh;
3378         int ret = 1;
3379
3380         cache = btrfs_lookup_block_group(fs_info, chunk_offset);
3381         chunk_used = cache->used;
3382
3383         if (bargs->usage_min == 0)
3384                 user_thresh = 1;
3385         else if (bargs->usage > 100)
3386                 user_thresh = cache->length;
3387         else
3388                 user_thresh = div_factor_fine(cache->length, bargs->usage);
3389
3390         if (chunk_used < user_thresh)
3391                 ret = 0;
3392
3393         btrfs_put_block_group(cache);
3394         return ret;
3395 }
3396
3397 static int chunk_devid_filter(struct extent_buffer *leaf,
3398                               struct btrfs_chunk *chunk,
3399                               struct btrfs_balance_args *bargs)
3400 {
3401         struct btrfs_stripe *stripe;
3402         int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3403         int i;
3404
3405         for (i = 0; i < num_stripes; i++) {
3406                 stripe = btrfs_stripe_nr(chunk, i);
3407                 if (btrfs_stripe_devid(leaf, stripe) == bargs->devid)
3408                         return 0;
3409         }
3410
3411         return 1;
3412 }
3413
3414 static u64 calc_data_stripes(u64 type, int num_stripes)
3415 {
3416         const int index = btrfs_bg_flags_to_raid_index(type);
3417         const int ncopies = btrfs_raid_array[index].ncopies;
3418         const int nparity = btrfs_raid_array[index].nparity;
3419
3420         if (nparity)
3421                 return num_stripes - nparity;
3422         else
3423                 return num_stripes / ncopies;
3424 }
3425
3426 /* [pstart, pend) */
3427 static int chunk_drange_filter(struct extent_buffer *leaf,
3428                                struct btrfs_chunk *chunk,
3429                                struct btrfs_balance_args *bargs)
3430 {
3431         struct btrfs_stripe *stripe;
3432         int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3433         u64 stripe_offset;
3434         u64 stripe_length;
3435         u64 type;
3436         int factor;
3437         int i;
3438
3439         if (!(bargs->flags & BTRFS_BALANCE_ARGS_DEVID))
3440                 return 0;
3441
3442         type = btrfs_chunk_type(leaf, chunk);
3443         factor = calc_data_stripes(type, num_stripes);
3444
3445         for (i = 0; i < num_stripes; i++) {
3446                 stripe = btrfs_stripe_nr(chunk, i);
3447                 if (btrfs_stripe_devid(leaf, stripe) != bargs->devid)
3448                         continue;
3449
3450                 stripe_offset = btrfs_stripe_offset(leaf, stripe);
3451                 stripe_length = btrfs_chunk_length(leaf, chunk);
3452                 stripe_length = div_u64(stripe_length, factor);
3453
3454                 if (stripe_offset < bargs->pend &&
3455                     stripe_offset + stripe_length > bargs->pstart)
3456                         return 0;
3457         }
3458
3459         return 1;
3460 }
3461
3462 /* [vstart, vend) */
3463 static int chunk_vrange_filter(struct extent_buffer *leaf,
3464                                struct btrfs_chunk *chunk,
3465                                u64 chunk_offset,
3466                                struct btrfs_balance_args *bargs)
3467 {
3468         if (chunk_offset < bargs->vend &&
3469             chunk_offset + btrfs_chunk_length(leaf, chunk) > bargs->vstart)
3470                 /* at least part of the chunk is inside this vrange */
3471                 return 0;
3472
3473         return 1;
3474 }
3475
3476 static int chunk_stripes_range_filter(struct extent_buffer *leaf,
3477                                struct btrfs_chunk *chunk,
3478                                struct btrfs_balance_args *bargs)
3479 {
3480         int num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
3481
3482         if (bargs->stripes_min <= num_stripes
3483                         && num_stripes <= bargs->stripes_max)
3484                 return 0;
3485
3486         return 1;
3487 }
3488
3489 static int chunk_soft_convert_filter(u64 chunk_type,
3490                                      struct btrfs_balance_args *bargs)
3491 {
3492         if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3493                 return 0;
3494
3495         chunk_type = chunk_to_extended(chunk_type) &
3496                                 BTRFS_EXTENDED_PROFILE_MASK;
3497
3498         if (bargs->target == chunk_type)
3499                 return 1;
3500
3501         return 0;
3502 }
3503
3504 static int should_balance_chunk(struct extent_buffer *leaf,
3505                                 struct btrfs_chunk *chunk, u64 chunk_offset)
3506 {
3507         struct btrfs_fs_info *fs_info = leaf->fs_info;
3508         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3509         struct btrfs_balance_args *bargs = NULL;
3510         u64 chunk_type = btrfs_chunk_type(leaf, chunk);
3511
3512         /* type filter */
3513         if (!((chunk_type & BTRFS_BLOCK_GROUP_TYPE_MASK) &
3514               (bctl->flags & BTRFS_BALANCE_TYPE_MASK))) {
3515                 return 0;
3516         }
3517
3518         if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3519                 bargs = &bctl->data;
3520         else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3521                 bargs = &bctl->sys;
3522         else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3523                 bargs = &bctl->meta;
3524
3525         /* profiles filter */
3526         if ((bargs->flags & BTRFS_BALANCE_ARGS_PROFILES) &&
3527             chunk_profiles_filter(chunk_type, bargs)) {
3528                 return 0;
3529         }
3530
3531         /* usage filter */
3532         if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE) &&
3533             chunk_usage_filter(fs_info, chunk_offset, bargs)) {
3534                 return 0;
3535         } else if ((bargs->flags & BTRFS_BALANCE_ARGS_USAGE_RANGE) &&
3536             chunk_usage_range_filter(fs_info, chunk_offset, bargs)) {
3537                 return 0;
3538         }
3539
3540         /* devid filter */
3541         if ((bargs->flags & BTRFS_BALANCE_ARGS_DEVID) &&
3542             chunk_devid_filter(leaf, chunk, bargs)) {
3543                 return 0;
3544         }
3545
3546         /* drange filter, makes sense only with devid filter */
3547         if ((bargs->flags & BTRFS_BALANCE_ARGS_DRANGE) &&
3548             chunk_drange_filter(leaf, chunk, bargs)) {
3549                 return 0;
3550         }
3551
3552         /* vrange filter */
3553         if ((bargs->flags & BTRFS_BALANCE_ARGS_VRANGE) &&
3554             chunk_vrange_filter(leaf, chunk, chunk_offset, bargs)) {
3555                 return 0;
3556         }
3557
3558         /* stripes filter */
3559         if ((bargs->flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE) &&
3560             chunk_stripes_range_filter(leaf, chunk, bargs)) {
3561                 return 0;
3562         }
3563
3564         /* soft profile changing mode */
3565         if ((bargs->flags & BTRFS_BALANCE_ARGS_SOFT) &&
3566             chunk_soft_convert_filter(chunk_type, bargs)) {
3567                 return 0;
3568         }
3569
3570         /*
3571          * limited by count, must be the last filter
3572          */
3573         if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT)) {
3574                 if (bargs->limit == 0)
3575                         return 0;
3576                 else
3577                         bargs->limit--;
3578         } else if ((bargs->flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)) {
3579                 /*
3580                  * Same logic as the 'limit' filter; the minimum cannot be
3581                  * determined here because we do not have the global information
3582                  * about the count of all chunks that satisfy the filters.
3583                  */
3584                 if (bargs->limit_max == 0)
3585                         return 0;
3586                 else
3587                         bargs->limit_max--;
3588         }
3589
3590         return 1;
3591 }
3592
3593 static int __btrfs_balance(struct btrfs_fs_info *fs_info)
3594 {
3595         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3596         struct btrfs_root *chunk_root = fs_info->chunk_root;
3597         u64 chunk_type;
3598         struct btrfs_chunk *chunk;
3599         struct btrfs_path *path = NULL;
3600         struct btrfs_key key;
3601         struct btrfs_key found_key;
3602         struct extent_buffer *leaf;
3603         int slot;
3604         int ret;
3605         int enospc_errors = 0;
3606         bool counting = true;
3607         /* The single value limit and min/max limits use the same bytes in the */
3608         u64 limit_data = bctl->data.limit;
3609         u64 limit_meta = bctl->meta.limit;
3610         u64 limit_sys = bctl->sys.limit;
3611         u32 count_data = 0;
3612         u32 count_meta = 0;
3613         u32 count_sys = 0;
3614         int chunk_reserved = 0;
3615
3616         path = btrfs_alloc_path();
3617         if (!path) {
3618                 ret = -ENOMEM;
3619                 goto error;
3620         }
3621
3622         /* zero out stat counters */
3623         spin_lock(&fs_info->balance_lock);
3624         memset(&bctl->stat, 0, sizeof(bctl->stat));
3625         spin_unlock(&fs_info->balance_lock);
3626 again:
3627         if (!counting) {
3628                 /*
3629                  * The single value limit and min/max limits use the same bytes
3630                  * in the
3631                  */
3632                 bctl->data.limit = limit_data;
3633                 bctl->meta.limit = limit_meta;
3634                 bctl->sys.limit = limit_sys;
3635         }
3636         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
3637         key.offset = (u64)-1;
3638         key.type = BTRFS_CHUNK_ITEM_KEY;
3639
3640         while (1) {
3641                 if ((!counting && atomic_read(&fs_info->balance_pause_req)) ||
3642                     atomic_read(&fs_info->balance_cancel_req)) {
3643                         ret = -ECANCELED;
3644                         goto error;
3645                 }
3646
3647                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
3648                 ret = btrfs_search_slot(NULL, chunk_root, &key, path, 0, 0);
3649                 if (ret < 0) {
3650                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3651                         goto error;
3652                 }
3653
3654                 /*
3655                  * this shouldn't happen, it means the last relocate
3656                  * failed
3657                  */
3658                 if (ret == 0)
3659                         BUG(); /* FIXME break ? */
3660
3661                 ret = btrfs_previous_item(chunk_root, path, 0,
3662                                           BTRFS_CHUNK_ITEM_KEY);
3663                 if (ret) {
3664                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3665                         ret = 0;
3666                         break;
3667                 }
3668
3669                 leaf = path->nodes[0];
3670                 slot = path->slots[0];
3671                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3672
3673                 if (found_key.objectid != key.objectid) {
3674                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3675                         break;
3676                 }
3677
3678                 chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
3679                 chunk_type = btrfs_chunk_type(leaf, chunk);
3680
3681                 if (!counting) {
3682                         spin_lock(&fs_info->balance_lock);
3683                         bctl->stat.considered++;
3684                         spin_unlock(&fs_info->balance_lock);
3685                 }
3686
3687                 ret = should_balance_chunk(leaf, chunk, found_key.offset);
3688
3689                 btrfs_release_path(path);
3690                 if (!ret) {
3691                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3692                         goto loop;
3693                 }
3694
3695                 if (counting) {
3696                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3697                         spin_lock(&fs_info->balance_lock);
3698                         bctl->stat.expected++;
3699                         spin_unlock(&fs_info->balance_lock);
3700
3701                         if (chunk_type & BTRFS_BLOCK_GROUP_DATA)
3702                                 count_data++;
3703                         else if (chunk_type & BTRFS_BLOCK_GROUP_SYSTEM)
3704                                 count_sys++;
3705                         else if (chunk_type & BTRFS_BLOCK_GROUP_METADATA)
3706                                 count_meta++;
3707
3708                         goto loop;
3709                 }
3710
3711                 /*
3712                  * Apply limit_min filter, no need to check if the LIMITS
3713                  * filter is used, limit_min is 0 by default
3714                  */
3715                 if (((chunk_type & BTRFS_BLOCK_GROUP_DATA) &&
3716                                         count_data < bctl->data.limit_min)
3717                                 || ((chunk_type & BTRFS_BLOCK_GROUP_METADATA) &&
3718                                         count_meta < bctl->meta.limit_min)
3719                                 || ((chunk_type & BTRFS_BLOCK_GROUP_SYSTEM) &&
3720                                         count_sys < bctl->sys.limit_min)) {
3721                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3722                         goto loop;
3723                 }
3724
3725                 if (!chunk_reserved) {
3726                         /*
3727                          * We may be relocating the only data chunk we have,
3728                          * which could potentially end up with losing data's
3729                          * raid profile, so lets allocate an empty one in
3730                          * advance.
3731                          */
3732                         ret = btrfs_may_alloc_data_chunk(fs_info,
3733                                                          found_key.offset);
3734                         if (ret < 0) {
3735                                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3736                                 goto error;
3737                         } else if (ret == 1) {
3738                                 chunk_reserved = 1;
3739                         }
3740                 }
3741
3742                 ret = btrfs_relocate_chunk(fs_info, found_key.offset);
3743                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
3744                 if (ret == -ENOSPC) {
3745                         enospc_errors++;
3746                 } else if (ret == -ETXTBSY) {
3747                         btrfs_info(fs_info,
3748            "skipping relocation of block group %llu due to active swapfile",
3749                                    found_key.offset);
3750                         ret = 0;
3751                 } else if (ret) {
3752                         goto error;
3753                 } else {
3754                         spin_lock(&fs_info->balance_lock);
3755                         bctl->stat.completed++;
3756                         spin_unlock(&fs_info->balance_lock);
3757                 }
3758 loop:
3759                 if (found_key.offset == 0)
3760                         break;
3761                 key.offset = found_key.offset - 1;
3762         }
3763
3764         if (counting) {
3765                 btrfs_release_path(path);
3766                 counting = false;
3767                 goto again;
3768         }
3769 error:
3770         btrfs_free_path(path);
3771         if (enospc_errors) {
3772                 btrfs_info(fs_info, "%d enospc errors during balance",
3773                            enospc_errors);
3774                 if (!ret)
3775                         ret = -ENOSPC;
3776         }
3777
3778         return ret;
3779 }
3780
3781 /**
3782  * alloc_profile_is_valid - see if a given profile is valid and reduced
3783  * @flags: profile to validate
3784  * @extended: if true @flags is treated as an extended profile
3785  */
3786 static int alloc_profile_is_valid(u64 flags, int extended)
3787 {
3788         u64 mask = (extended ? BTRFS_EXTENDED_PROFILE_MASK :
3789                                BTRFS_BLOCK_GROUP_PROFILE_MASK);
3790
3791         flags &= ~BTRFS_BLOCK_GROUP_TYPE_MASK;
3792
3793         /* 1) check that all other bits are zeroed */
3794         if (flags & ~mask)
3795                 return 0;
3796
3797         /* 2) see if profile is reduced */
3798         if (flags == 0)
3799                 return !extended; /* "0" is valid for usual profiles */
3800
3801         return has_single_bit_set(flags);
3802 }
3803
3804 static inline int balance_need_close(struct btrfs_fs_info *fs_info)
3805 {
3806         /* cancel requested || normal exit path */
3807         return atomic_read(&fs_info->balance_cancel_req) ||
3808                 (atomic_read(&fs_info->balance_pause_req) == 0 &&
3809                  atomic_read(&fs_info->balance_cancel_req) == 0);
3810 }
3811
3812 /*
3813  * Validate target profile against allowed profiles and return true if it's OK.
3814  * Otherwise print the error message and return false.
3815  */
3816 static inline int validate_convert_profile(struct btrfs_fs_info *fs_info,
3817                 const struct btrfs_balance_args *bargs,
3818                 u64 allowed, const char *type)
3819 {
3820         if (!(bargs->flags & BTRFS_BALANCE_ARGS_CONVERT))
3821                 return true;
3822
3823         /* Profile is valid and does not have bits outside of the allowed set */
3824         if (alloc_profile_is_valid(bargs->target, 1) &&
3825             (bargs->target & ~allowed) == 0)
3826                 return true;
3827
3828         btrfs_err(fs_info, "balance: invalid convert %s profile %s",
3829                         type, btrfs_bg_type_to_raid_name(bargs->target));
3830         return false;
3831 }
3832
3833 /*
3834  * Fill @buf with textual description of balance filter flags @bargs, up to
3835  * @size_buf including the terminating null. The output may be trimmed if it
3836  * does not fit into the provided buffer.
3837  */
3838 static void describe_balance_args(struct btrfs_balance_args *bargs, char *buf,
3839                                  u32 size_buf)
3840 {
3841         int ret;
3842         u32 size_bp = size_buf;
3843         char *bp = buf;
3844         u64 flags = bargs->flags;
3845         char tmp_buf[128] = {'\0'};
3846
3847         if (!flags)
3848                 return;
3849
3850 #define CHECK_APPEND_NOARG(a)                                           \
3851         do {                                                            \
3852                 ret = snprintf(bp, size_bp, (a));                       \
3853                 if (ret < 0 || ret >= size_bp)                          \
3854                         goto out_overflow;                              \
3855                 size_bp -= ret;                                         \
3856                 bp += ret;                                              \
3857         } while (0)
3858
3859 #define CHECK_APPEND_1ARG(a, v1)                                        \
3860         do {                                                            \
3861                 ret = snprintf(bp, size_bp, (a), (v1));                 \
3862                 if (ret < 0 || ret >= size_bp)                          \
3863                         goto out_overflow;                              \
3864                 size_bp -= ret;                                         \
3865                 bp += ret;                                              \
3866         } while (0)
3867
3868 #define CHECK_APPEND_2ARG(a, v1, v2)                                    \
3869         do {                                                            \
3870                 ret = snprintf(bp, size_bp, (a), (v1), (v2));           \
3871                 if (ret < 0 || ret >= size_bp)                          \
3872                         goto out_overflow;                              \
3873                 size_bp -= ret;                                         \
3874                 bp += ret;                                              \
3875         } while (0)
3876
3877         if (flags & BTRFS_BALANCE_ARGS_CONVERT)
3878                 CHECK_APPEND_1ARG("convert=%s,",
3879                                   btrfs_bg_type_to_raid_name(bargs->target));
3880
3881         if (flags & BTRFS_BALANCE_ARGS_SOFT)
3882                 CHECK_APPEND_NOARG("soft,");
3883
3884         if (flags & BTRFS_BALANCE_ARGS_PROFILES) {
3885                 btrfs_describe_block_groups(bargs->profiles, tmp_buf,
3886                                             sizeof(tmp_buf));
3887                 CHECK_APPEND_1ARG("profiles=%s,", tmp_buf);
3888         }
3889
3890         if (flags & BTRFS_BALANCE_ARGS_USAGE)
3891                 CHECK_APPEND_1ARG("usage=%llu,", bargs->usage);
3892
3893         if (flags & BTRFS_BALANCE_ARGS_USAGE_RANGE)
3894                 CHECK_APPEND_2ARG("usage=%u..%u,",
3895                                   bargs->usage_min, bargs->usage_max);
3896
3897         if (flags & BTRFS_BALANCE_ARGS_DEVID)
3898                 CHECK_APPEND_1ARG("devid=%llu,", bargs->devid);
3899
3900         if (flags & BTRFS_BALANCE_ARGS_DRANGE)
3901                 CHECK_APPEND_2ARG("drange=%llu..%llu,",
3902                                   bargs->pstart, bargs->pend);
3903
3904         if (flags & BTRFS_BALANCE_ARGS_VRANGE)
3905                 CHECK_APPEND_2ARG("vrange=%llu..%llu,",
3906                                   bargs->vstart, bargs->vend);
3907
3908         if (flags & BTRFS_BALANCE_ARGS_LIMIT)
3909                 CHECK_APPEND_1ARG("limit=%llu,", bargs->limit);
3910
3911         if (flags & BTRFS_BALANCE_ARGS_LIMIT_RANGE)
3912                 CHECK_APPEND_2ARG("limit=%u..%u,",
3913                                 bargs->limit_min, bargs->limit_max);
3914
3915         if (flags & BTRFS_BALANCE_ARGS_STRIPES_RANGE)
3916                 CHECK_APPEND_2ARG("stripes=%u..%u,",
3917                                   bargs->stripes_min, bargs->stripes_max);
3918
3919 #undef CHECK_APPEND_2ARG
3920 #undef CHECK_APPEND_1ARG
3921 #undef CHECK_APPEND_NOARG
3922
3923 out_overflow:
3924
3925         if (size_bp < size_buf)
3926                 buf[size_buf - size_bp - 1] = '\0'; /* remove last , */
3927         else
3928                 buf[0] = '\0';
3929 }
3930
3931 static void describe_balance_start_or_resume(struct btrfs_fs_info *fs_info)
3932 {
3933         u32 size_buf = 1024;
3934         char tmp_buf[192] = {'\0'};
3935         char *buf;
3936         char *bp;
3937         u32 size_bp = size_buf;
3938         int ret;
3939         struct btrfs_balance_control *bctl = fs_info->balance_ctl;
3940
3941         buf = kzalloc(size_buf, GFP_KERNEL);
3942         if (!buf)
3943                 return;
3944
3945         bp = buf;
3946
3947 #define CHECK_APPEND_1ARG(a, v1)                                        \
3948         do {                                                            \
3949                 ret = snprintf(bp, size_bp, (a), (v1));                 \
3950                 if (ret < 0 || ret >= size_bp)                          \
3951                         goto out_overflow;                              \
3952                 size_bp -= ret;                                         \
3953                 bp += ret;                                              \
3954         } while (0)
3955
3956         if (bctl->flags & BTRFS_BALANCE_FORCE)
3957                 CHECK_APPEND_1ARG("%s", "-f ");
3958
3959         if (bctl->flags & BTRFS_BALANCE_DATA) {
3960                 describe_balance_args(&bctl->data, tmp_buf, sizeof(tmp_buf));
3961                 CHECK_APPEND_1ARG("-d%s ", tmp_buf);
3962         }
3963
3964         if (bctl->flags & BTRFS_BALANCE_METADATA) {
3965                 describe_balance_args(&bctl->meta, tmp_buf, sizeof(tmp_buf));
3966                 CHECK_APPEND_1ARG("-m%s ", tmp_buf);
3967         }
3968
3969         if (bctl->flags & BTRFS_BALANCE_SYSTEM) {
3970                 describe_balance_args(&bctl->sys, tmp_buf, sizeof(tmp_buf));
3971                 CHECK_APPEND_1ARG("-s%s ", tmp_buf);
3972         }
3973
3974 #undef CHECK_APPEND_1ARG
3975
3976 out_overflow:
3977
3978         if (size_bp < size_buf)
3979                 buf[size_buf - size_bp - 1] = '\0'; /* remove last " " */
3980         btrfs_info(fs_info, "balance: %s %s",
3981                    (bctl->flags & BTRFS_BALANCE_RESUME) ?
3982                    "resume" : "start", buf);
3983
3984         kfree(buf);
3985 }
3986
3987 /*
3988  * Should be called with balance mutexe held
3989  */
3990 int btrfs_balance(struct btrfs_fs_info *fs_info,
3991                   struct btrfs_balance_control *bctl,
3992                   struct btrfs_ioctl_balance_args *bargs)
3993 {
3994         u64 meta_target, data_target;
3995         u64 allowed;
3996         int mixed = 0;
3997         int ret;
3998         u64 num_devices;
3999         unsigned seq;
4000         bool reducing_redundancy;
4001         int i;
4002
4003         if (btrfs_fs_closing(fs_info) ||
4004             atomic_read(&fs_info->balance_pause_req) ||
4005             btrfs_should_cancel_balance(fs_info)) {
4006                 ret = -EINVAL;
4007                 goto out;
4008         }
4009
4010         allowed = btrfs_super_incompat_flags(fs_info->super_copy);
4011         if (allowed & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS)
4012                 mixed = 1;
4013
4014         /*
4015          * In case of mixed groups both data and meta should be picked,
4016          * and identical options should be given for both of them.
4017          */
4018         allowed = BTRFS_BALANCE_DATA | BTRFS_BALANCE_METADATA;
4019         if (mixed && (bctl->flags & allowed)) {
4020                 if (!(bctl->flags & BTRFS_BALANCE_DATA) ||
4021                     !(bctl->flags & BTRFS_BALANCE_METADATA) ||
4022                     memcmp(&bctl->data, &bctl->meta, sizeof(bctl->data))) {
4023                         btrfs_err(fs_info,
4024           "balance: mixed groups data and metadata options must be the same");
4025                         ret = -EINVAL;
4026                         goto out;
4027                 }
4028         }
4029
4030         /*
4031          * rw_devices will not change at the moment, device add/delete/replace
4032          * are exclusive
4033          */
4034         num_devices = fs_info->fs_devices->rw_devices;
4035
4036         /*
4037          * SINGLE profile on-disk has no profile bit, but in-memory we have a
4038          * special bit for it, to make it easier to distinguish.  Thus we need
4039          * to set it manually, or balance would refuse the profile.
4040          */
4041         allowed = BTRFS_AVAIL_ALLOC_BIT_SINGLE;
4042         for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++)
4043                 if (num_devices >= btrfs_raid_array[i].devs_min)
4044                         allowed |= btrfs_raid_array[i].bg_flag;
4045
4046         if (!validate_convert_profile(fs_info, &bctl->data, allowed, "data") ||
4047             !validate_convert_profile(fs_info, &bctl->meta, allowed, "metadata") ||
4048             !validate_convert_profile(fs_info, &bctl->sys,  allowed, "system")) {
4049                 ret = -EINVAL;
4050                 goto out;
4051         }
4052
4053         /*
4054          * Allow to reduce metadata or system integrity only if force set for
4055          * profiles with redundancy (copies, parity)
4056          */
4057         allowed = 0;
4058         for (i = 0; i < ARRAY_SIZE(btrfs_raid_array); i++) {
4059                 if (btrfs_raid_array[i].ncopies >= 2 ||
4060                     btrfs_raid_array[i].tolerated_failures >= 1)
4061                         allowed |= btrfs_raid_array[i].bg_flag;
4062         }
4063         do {
4064                 seq = read_seqbegin(&fs_info->profiles_lock);
4065
4066                 if (((bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4067                      (fs_info->avail_system_alloc_bits & allowed) &&
4068                      !(bctl->sys.target & allowed)) ||
4069                     ((bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) &&
4070                      (fs_info->avail_metadata_alloc_bits & allowed) &&
4071                      !(bctl->meta.target & allowed)))
4072                         reducing_redundancy = true;
4073                 else
4074                         reducing_redundancy = false;
4075
4076                 /* if we're not converting, the target field is uninitialized */
4077                 meta_target = (bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4078                         bctl->meta.target : fs_info->avail_metadata_alloc_bits;
4079                 data_target = (bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) ?
4080                         bctl->data.target : fs_info->avail_data_alloc_bits;
4081         } while (read_seqretry(&fs_info->profiles_lock, seq));
4082
4083         if (reducing_redundancy) {
4084                 if (bctl->flags & BTRFS_BALANCE_FORCE) {
4085                         btrfs_info(fs_info,
4086                            "balance: force reducing metadata redundancy");
4087                 } else {
4088                         btrfs_err(fs_info,
4089         "balance: reduces metadata redundancy, use --force if you want this");
4090                         ret = -EINVAL;
4091                         goto out;
4092                 }
4093         }
4094
4095         if (btrfs_get_num_tolerated_disk_barrier_failures(meta_target) <
4096                 btrfs_get_num_tolerated_disk_barrier_failures(data_target)) {
4097                 btrfs_warn(fs_info,
4098         "balance: metadata profile %s has lower redundancy than data profile %s",
4099                                 btrfs_bg_type_to_raid_name(meta_target),
4100                                 btrfs_bg_type_to_raid_name(data_target));
4101         }
4102
4103         if (fs_info->send_in_progress) {
4104                 btrfs_warn_rl(fs_info,
4105 "cannot run balance while send operations are in progress (%d in progress)",
4106                               fs_info->send_in_progress);
4107                 ret = -EAGAIN;
4108                 goto out;
4109         }
4110
4111         ret = insert_balance_item(fs_info, bctl);
4112         if (ret && ret != -EEXIST)
4113                 goto out;
4114
4115         if (!(bctl->flags & BTRFS_BALANCE_RESUME)) {
4116                 BUG_ON(ret == -EEXIST);
4117                 BUG_ON(fs_info->balance_ctl);
4118                 spin_lock(&fs_info->balance_lock);
4119                 fs_info->balance_ctl = bctl;
4120                 spin_unlock(&fs_info->balance_lock);
4121         } else {
4122                 BUG_ON(ret != -EEXIST);
4123                 spin_lock(&fs_info->balance_lock);
4124                 update_balance_args(bctl);
4125                 spin_unlock(&fs_info->balance_lock);
4126         }
4127
4128         ASSERT(!test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4129         set_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4130         describe_balance_start_or_resume(fs_info);
4131         mutex_unlock(&fs_info->balance_mutex);
4132
4133         ret = __btrfs_balance(fs_info);
4134
4135         mutex_lock(&fs_info->balance_mutex);
4136         if (ret == -ECANCELED && atomic_read(&fs_info->balance_pause_req))
4137                 btrfs_info(fs_info, "balance: paused");
4138         /*
4139          * Balance can be canceled by:
4140          *
4141          * - Regular cancel request
4142          *   Then ret == -ECANCELED and balance_cancel_req > 0
4143          *
4144          * - Fatal signal to "btrfs" process
4145          *   Either the signal caught by wait_reserve_ticket() and callers
4146          *   got -EINTR, or caught by btrfs_should_cancel_balance() and
4147          *   got -ECANCELED.
4148          *   Either way, in this case balance_cancel_req = 0, and
4149          *   ret == -EINTR or ret == -ECANCELED.
4150          *
4151          * So here we only check the return value to catch canceled balance.
4152          */
4153         else if (ret == -ECANCELED || ret == -EINTR)
4154                 btrfs_info(fs_info, "balance: canceled");
4155         else
4156                 btrfs_info(fs_info, "balance: ended with status: %d", ret);
4157
4158         clear_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags);
4159
4160         if (bargs) {
4161                 memset(bargs, 0, sizeof(*bargs));
4162                 btrfs_update_ioctl_balance_args(fs_info, bargs);
4163         }
4164
4165         if ((ret && ret != -ECANCELED && ret != -ENOSPC) ||
4166             balance_need_close(fs_info)) {
4167                 reset_balance_state(fs_info);
4168                 btrfs_exclop_finish(fs_info);
4169         }
4170
4171         wake_up(&fs_info->balance_wait_q);
4172
4173         return ret;
4174 out:
4175         if (bctl->flags & BTRFS_BALANCE_RESUME)
4176                 reset_balance_state(fs_info);
4177         else
4178                 kfree(bctl);
4179         btrfs_exclop_finish(fs_info);
4180
4181         return ret;
4182 }
4183
4184 static int balance_kthread(void *data)
4185 {
4186         struct btrfs_fs_info *fs_info = data;
4187         int ret = 0;
4188
4189         mutex_lock(&fs_info->balance_mutex);
4190         if (fs_info->balance_ctl)
4191                 ret = btrfs_balance(fs_info, fs_info->balance_ctl, NULL);
4192         mutex_unlock(&fs_info->balance_mutex);
4193
4194         return ret;
4195 }
4196
4197 int btrfs_resume_balance_async(struct btrfs_fs_info *fs_info)
4198 {
4199         struct task_struct *tsk;
4200
4201         mutex_lock(&fs_info->balance_mutex);
4202         if (!fs_info->balance_ctl) {
4203                 mutex_unlock(&fs_info->balance_mutex);
4204                 return 0;
4205         }
4206         mutex_unlock(&fs_info->balance_mutex);
4207
4208         if (btrfs_test_opt(fs_info, SKIP_BALANCE)) {
4209                 btrfs_info(fs_info, "balance: resume skipped");
4210                 return 0;
4211         }
4212
4213         /*
4214          * A ro->rw remount sequence should continue with the paused balance
4215          * regardless of who pauses it, system or the user as of now, so set
4216          * the resume flag.
4217          */
4218         spin_lock(&fs_info->balance_lock);
4219         fs_info->balance_ctl->flags |= BTRFS_BALANCE_RESUME;
4220         spin_unlock(&fs_info->balance_lock);
4221
4222         tsk = kthread_run(balance_kthread, fs_info, "btrfs-balance");
4223         return PTR_ERR_OR_ZERO(tsk);
4224 }
4225
4226 int btrfs_recover_balance(struct btrfs_fs_info *fs_info)
4227 {
4228         struct btrfs_balance_control *bctl;
4229         struct btrfs_balance_item *item;
4230         struct btrfs_disk_balance_args disk_bargs;
4231         struct btrfs_path *path;
4232         struct extent_buffer *leaf;
4233         struct btrfs_key key;
4234         int ret;
4235
4236         path = btrfs_alloc_path();
4237         if (!path)
4238                 return -ENOMEM;
4239
4240         key.objectid = BTRFS_BALANCE_OBJECTID;
4241         key.type = BTRFS_TEMPORARY_ITEM_KEY;
4242         key.offset = 0;
4243
4244         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
4245         if (ret < 0)
4246                 goto out;
4247         if (ret > 0) { /* ret = -ENOENT; */
4248                 ret = 0;
4249                 goto out;
4250         }
4251
4252         bctl = kzalloc(sizeof(*bctl), GFP_NOFS);
4253         if (!bctl) {
4254                 ret = -ENOMEM;
4255                 goto out;
4256         }
4257
4258         leaf = path->nodes[0];
4259         item = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_balance_item);
4260
4261         bctl->flags = btrfs_balance_flags(leaf, item);
4262         bctl->flags |= BTRFS_BALANCE_RESUME;
4263
4264         btrfs_balance_data(leaf, item, &disk_bargs);
4265         btrfs_disk_balance_args_to_cpu(&bctl->data, &disk_bargs);
4266         btrfs_balance_meta(leaf, item, &disk_bargs);
4267         btrfs_disk_balance_args_to_cpu(&bctl->meta, &disk_bargs);
4268         btrfs_balance_sys(leaf, item, &disk_bargs);
4269         btrfs_disk_balance_args_to_cpu(&bctl->sys, &disk_bargs);
4270
4271         /*
4272          * This should never happen, as the paused balance state is recovered
4273          * during mount without any chance of other exclusive ops to collide.
4274          *
4275          * This gives the exclusive op status to balance and keeps in paused
4276          * state until user intervention (cancel or umount). If the ownership
4277          * cannot be assigned, show a message but do not fail. The balance
4278          * is in a paused state and must have fs_info::balance_ctl properly
4279          * set up.
4280          */
4281         if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_BALANCE))
4282                 btrfs_warn(fs_info,
4283         "balance: cannot set exclusive op status, resume manually");
4284
4285         mutex_lock(&fs_info->balance_mutex);
4286         BUG_ON(fs_info->balance_ctl);
4287         spin_lock(&fs_info->balance_lock);
4288         fs_info->balance_ctl = bctl;
4289         spin_unlock(&fs_info->balance_lock);
4290         mutex_unlock(&fs_info->balance_mutex);
4291 out:
4292         btrfs_free_path(path);
4293         return ret;
4294 }
4295
4296 int btrfs_pause_balance(struct btrfs_fs_info *fs_info)
4297 {
4298         int ret = 0;
4299
4300         mutex_lock(&fs_info->balance_mutex);
4301         if (!fs_info->balance_ctl) {
4302                 mutex_unlock(&fs_info->balance_mutex);
4303                 return -ENOTCONN;
4304         }
4305
4306         if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4307                 atomic_inc(&fs_info->balance_pause_req);
4308                 mutex_unlock(&fs_info->balance_mutex);
4309
4310                 wait_event(fs_info->balance_wait_q,
4311                            !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4312
4313                 mutex_lock(&fs_info->balance_mutex);
4314                 /* we are good with balance_ctl ripped off from under us */
4315                 BUG_ON(test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4316                 atomic_dec(&fs_info->balance_pause_req);
4317         } else {
4318                 ret = -ENOTCONN;
4319         }
4320
4321         mutex_unlock(&fs_info->balance_mutex);
4322         return ret;
4323 }
4324
4325 int btrfs_cancel_balance(struct btrfs_fs_info *fs_info)
4326 {
4327         mutex_lock(&fs_info->balance_mutex);
4328         if (!fs_info->balance_ctl) {
4329                 mutex_unlock(&fs_info->balance_mutex);
4330                 return -ENOTCONN;
4331         }
4332
4333         /*
4334          * A paused balance with the item stored on disk can be resumed at
4335          * mount time if the mount is read-write. Otherwise it's still paused
4336          * and we must not allow cancelling as it deletes the item.
4337          */
4338         if (sb_rdonly(fs_info->sb)) {
4339                 mutex_unlock(&fs_info->balance_mutex);
4340                 return -EROFS;
4341         }
4342
4343         atomic_inc(&fs_info->balance_cancel_req);
4344         /*
4345          * if we are running just wait and return, balance item is
4346          * deleted in btrfs_balance in this case
4347          */
4348         if (test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags)) {
4349                 mutex_unlock(&fs_info->balance_mutex);
4350                 wait_event(fs_info->balance_wait_q,
4351                            !test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4352                 mutex_lock(&fs_info->balance_mutex);
4353         } else {
4354                 mutex_unlock(&fs_info->balance_mutex);
4355                 /*
4356                  * Lock released to allow other waiters to continue, we'll
4357                  * reexamine the status again.
4358                  */
4359                 mutex_lock(&fs_info->balance_mutex);
4360
4361                 if (fs_info->balance_ctl) {
4362                         reset_balance_state(fs_info);
4363                         btrfs_exclop_finish(fs_info);
4364                         btrfs_info(fs_info, "balance: canceled");
4365                 }
4366         }
4367
4368         BUG_ON(fs_info->balance_ctl ||
4369                 test_bit(BTRFS_FS_BALANCE_RUNNING, &fs_info->flags));
4370         atomic_dec(&fs_info->balance_cancel_req);
4371         mutex_unlock(&fs_info->balance_mutex);
4372         return 0;
4373 }
4374
4375 int btrfs_uuid_scan_kthread(void *data)
4376 {
4377         struct btrfs_fs_info *fs_info = data;
4378         struct btrfs_root *root = fs_info->tree_root;
4379         struct btrfs_key key;
4380         struct btrfs_path *path = NULL;
4381         int ret = 0;
4382         struct extent_buffer *eb;
4383         int slot;
4384         struct btrfs_root_item root_item;
4385         u32 item_size;
4386         struct btrfs_trans_handle *trans = NULL;
4387         bool closing = false;
4388
4389         path = btrfs_alloc_path();
4390         if (!path) {
4391                 ret = -ENOMEM;
4392                 goto out;
4393         }
4394
4395         key.objectid = 0;
4396         key.type = BTRFS_ROOT_ITEM_KEY;
4397         key.offset = 0;
4398
4399         while (1) {
4400                 if (btrfs_fs_closing(fs_info)) {
4401                         closing = true;
4402                         break;
4403                 }
4404                 ret = btrfs_search_forward(root, &key, path,
4405                                 BTRFS_OLDEST_GENERATION);
4406                 if (ret) {
4407                         if (ret > 0)
4408                                 ret = 0;
4409                         break;
4410                 }
4411
4412                 if (key.type != BTRFS_ROOT_ITEM_KEY ||
4413                     (key.objectid < BTRFS_FIRST_FREE_OBJECTID &&
4414                      key.objectid != BTRFS_FS_TREE_OBJECTID) ||
4415                     key.objectid > BTRFS_LAST_FREE_OBJECTID)
4416                         goto skip;
4417
4418                 eb = path->nodes[0];
4419                 slot = path->slots[0];
4420                 item_size = btrfs_item_size_nr(eb, slot);
4421                 if (item_size < sizeof(root_item))
4422                         goto skip;
4423
4424                 read_extent_buffer(eb, &root_item,
4425                                    btrfs_item_ptr_offset(eb, slot),
4426                                    (int)sizeof(root_item));
4427                 if (btrfs_root_refs(&root_item) == 0)
4428                         goto skip;
4429
4430                 if (!btrfs_is_empty_uuid(root_item.uuid) ||
4431                     !btrfs_is_empty_uuid(root_item.received_uuid)) {
4432                         if (trans)
4433                                 goto update_tree;
4434
4435                         btrfs_release_path(path);
4436                         /*
4437                          * 1 - subvol uuid item
4438                          * 1 - received_subvol uuid item
4439                          */
4440                         trans = btrfs_start_transaction(fs_info->uuid_root, 2);
4441                         if (IS_ERR(trans)) {
4442                                 ret = PTR_ERR(trans);
4443                                 break;
4444                         }
4445                         continue;
4446                 } else {
4447                         goto skip;
4448                 }
4449 update_tree:
4450                 btrfs_release_path(path);
4451                 if (!btrfs_is_empty_uuid(root_item.uuid)) {
4452                         ret = btrfs_uuid_tree_add(trans, root_item.uuid,
4453                                                   BTRFS_UUID_KEY_SUBVOL,
4454                                                   key.objectid);
4455                         if (ret < 0) {
4456                                 btrfs_warn(fs_info, "uuid_tree_add failed %d",
4457                                         ret);
4458                                 break;
4459                         }
4460                 }
4461
4462                 if (!btrfs_is_empty_uuid(root_item.received_uuid)) {
4463                         ret = btrfs_uuid_tree_add(trans,
4464                                                   root_item.received_uuid,
4465                                                  BTRFS_UUID_KEY_RECEIVED_SUBVOL,
4466                                                   key.objectid);
4467                         if (ret < 0) {
4468                                 btrfs_warn(fs_info, "uuid_tree_add failed %d",
4469                                         ret);
4470                                 break;
4471                         }
4472                 }
4473
4474 skip:
4475                 btrfs_release_path(path);
4476                 if (trans) {
4477                         ret = btrfs_end_transaction(trans);
4478                         trans = NULL;
4479                         if (ret)
4480                                 break;
4481                 }
4482
4483                 if (key.offset < (u64)-1) {
4484                         key.offset++;
4485                 } else if (key.type < BTRFS_ROOT_ITEM_KEY) {
4486                         key.offset = 0;
4487                         key.type = BTRFS_ROOT_ITEM_KEY;
4488                 } else if (key.objectid < (u64)-1) {
4489                         key.offset = 0;
4490                         key.type = BTRFS_ROOT_ITEM_KEY;
4491                         key.objectid++;
4492                 } else {
4493                         break;
4494                 }
4495                 cond_resched();
4496         }
4497
4498 out:
4499         btrfs_free_path(path);
4500         if (trans && !IS_ERR(trans))
4501                 btrfs_end_transaction(trans);
4502         if (ret)
4503                 btrfs_warn(fs_info, "btrfs_uuid_scan_kthread failed %d", ret);
4504         else if (!closing)
4505                 set_bit(BTRFS_FS_UPDATE_UUID_TREE_GEN, &fs_info->flags);
4506         up(&fs_info->uuid_tree_rescan_sem);
4507         return 0;
4508 }
4509
4510 int btrfs_create_uuid_tree(struct btrfs_fs_info *fs_info)
4511 {
4512         struct btrfs_trans_handle *trans;
4513         struct btrfs_root *tree_root = fs_info->tree_root;
4514         struct btrfs_root *uuid_root;
4515         struct task_struct *task;
4516         int ret;
4517
4518         /*
4519          * 1 - root node
4520          * 1 - root item
4521          */
4522         trans = btrfs_start_transaction(tree_root, 2);
4523         if (IS_ERR(trans))
4524                 return PTR_ERR(trans);
4525
4526         uuid_root = btrfs_create_tree(trans, BTRFS_UUID_TREE_OBJECTID);
4527         if (IS_ERR(uuid_root)) {
4528                 ret = PTR_ERR(uuid_root);
4529                 btrfs_abort_transaction(trans, ret);
4530                 btrfs_end_transaction(trans);
4531                 return ret;
4532         }
4533
4534         fs_info->uuid_root = uuid_root;
4535
4536         ret = btrfs_commit_transaction(trans);
4537         if (ret)
4538                 return ret;
4539
4540         down(&fs_info->uuid_tree_rescan_sem);
4541         task = kthread_run(btrfs_uuid_scan_kthread, fs_info, "btrfs-uuid");
4542         if (IS_ERR(task)) {
4543                 /* fs_info->update_uuid_tree_gen remains 0 in all error case */
4544                 btrfs_warn(fs_info, "failed to start uuid_scan task");
4545                 up(&fs_info->uuid_tree_rescan_sem);
4546                 return PTR_ERR(task);
4547         }
4548
4549         return 0;
4550 }
4551
4552 /*
4553  * shrinking a device means finding all of the device extents past
4554  * the new size, and then following the back refs to the chunks.
4555  * The chunk relocation code actually frees the device extent
4556  */
4557 int btrfs_shrink_device(struct btrfs_device *device, u64 new_size)
4558 {
4559         struct btrfs_fs_info *fs_info = device->fs_info;
4560         struct btrfs_root *root = fs_info->dev_root;
4561         struct btrfs_trans_handle *trans;
4562         struct btrfs_dev_extent *dev_extent = NULL;
4563         struct btrfs_path *path;
4564         u64 length;
4565         u64 chunk_offset;
4566         int ret;
4567         int slot;
4568         int failed = 0;
4569         bool retried = false;
4570         struct extent_buffer *l;
4571         struct btrfs_key key;
4572         struct btrfs_super_block *super_copy = fs_info->super_copy;
4573         u64 old_total = btrfs_super_total_bytes(super_copy);
4574         u64 old_size = btrfs_device_get_total_bytes(device);
4575         u64 diff;
4576         u64 start;
4577
4578         new_size = round_down(new_size, fs_info->sectorsize);
4579         start = new_size;
4580         diff = round_down(old_size - new_size, fs_info->sectorsize);
4581
4582         if (test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4583                 return -EINVAL;
4584
4585         path = btrfs_alloc_path();
4586         if (!path)
4587                 return -ENOMEM;
4588
4589         path->reada = READA_BACK;
4590
4591         trans = btrfs_start_transaction(root, 0);
4592         if (IS_ERR(trans)) {
4593                 btrfs_free_path(path);
4594                 return PTR_ERR(trans);
4595         }
4596
4597         mutex_lock(&fs_info->chunk_mutex);
4598
4599         btrfs_device_set_total_bytes(device, new_size);
4600         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4601                 device->fs_devices->total_rw_bytes -= diff;
4602                 atomic64_sub(diff, &fs_info->free_chunk_space);
4603         }
4604
4605         /*
4606          * Once the device's size has been set to the new size, ensure all
4607          * in-memory chunks are synced to disk so that the loop below sees them
4608          * and relocates them accordingly.
4609          */
4610         if (contains_pending_extent(device, &start, diff)) {
4611                 mutex_unlock(&fs_info->chunk_mutex);
4612                 ret = btrfs_commit_transaction(trans);
4613                 if (ret)
4614                         goto done;
4615         } else {
4616                 mutex_unlock(&fs_info->chunk_mutex);
4617                 btrfs_end_transaction(trans);
4618         }
4619
4620 again:
4621         key.objectid = device->devid;
4622         key.offset = (u64)-1;
4623         key.type = BTRFS_DEV_EXTENT_KEY;
4624
4625         do {
4626                 mutex_lock(&fs_info->delete_unused_bgs_mutex);
4627                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
4628                 if (ret < 0) {
4629                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4630                         goto done;
4631                 }
4632
4633                 ret = btrfs_previous_item(root, path, 0, key.type);
4634                 if (ret)
4635                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4636                 if (ret < 0)
4637                         goto done;
4638                 if (ret) {
4639                         ret = 0;
4640                         btrfs_release_path(path);
4641                         break;
4642                 }
4643
4644                 l = path->nodes[0];
4645                 slot = path->slots[0];
4646                 btrfs_item_key_to_cpu(l, &key, path->slots[0]);
4647
4648                 if (key.objectid != device->devid) {
4649                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4650                         btrfs_release_path(path);
4651                         break;
4652                 }
4653
4654                 dev_extent = btrfs_item_ptr(l, slot, struct btrfs_dev_extent);
4655                 length = btrfs_dev_extent_length(l, dev_extent);
4656
4657                 if (key.offset + length <= new_size) {
4658                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4659                         btrfs_release_path(path);
4660                         break;
4661                 }
4662
4663                 chunk_offset = btrfs_dev_extent_chunk_offset(l, dev_extent);
4664                 btrfs_release_path(path);
4665
4666                 /*
4667                  * We may be relocating the only data chunk we have,
4668                  * which could potentially end up with losing data's
4669                  * raid profile, so lets allocate an empty one in
4670                  * advance.
4671                  */
4672                 ret = btrfs_may_alloc_data_chunk(fs_info, chunk_offset);
4673                 if (ret < 0) {
4674                         mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4675                         goto done;
4676                 }
4677
4678                 ret = btrfs_relocate_chunk(fs_info, chunk_offset);
4679                 mutex_unlock(&fs_info->delete_unused_bgs_mutex);
4680                 if (ret == -ENOSPC) {
4681                         failed++;
4682                 } else if (ret) {
4683                         if (ret == -ETXTBSY) {
4684                                 btrfs_warn(fs_info,
4685                    "could not shrink block group %llu due to active swapfile",
4686                                            chunk_offset);
4687                         }
4688                         goto done;
4689                 }
4690         } while (key.offset-- > 0);
4691
4692         if (failed && !retried) {
4693                 failed = 0;
4694                 retried = true;
4695                 goto again;
4696         } else if (failed && retried) {
4697                 ret = -ENOSPC;
4698                 goto done;
4699         }
4700
4701         /* Shrinking succeeded, else we would be at "done". */
4702         trans = btrfs_start_transaction(root, 0);
4703         if (IS_ERR(trans)) {
4704                 ret = PTR_ERR(trans);
4705                 goto done;
4706         }
4707
4708         mutex_lock(&fs_info->chunk_mutex);
4709         /* Clear all state bits beyond the shrunk device size */
4710         clear_extent_bits(&device->alloc_state, new_size, (u64)-1,
4711                           CHUNK_STATE_MASK);
4712
4713         btrfs_device_set_disk_total_bytes(device, new_size);
4714         if (list_empty(&device->post_commit_list))
4715                 list_add_tail(&device->post_commit_list,
4716                               &trans->transaction->dev_update_list);
4717
4718         WARN_ON(diff > old_total);
4719         btrfs_set_super_total_bytes(super_copy,
4720                         round_down(old_total - diff, fs_info->sectorsize));
4721         mutex_unlock(&fs_info->chunk_mutex);
4722
4723         /* Now btrfs_update_device() will change the on-disk size. */
4724         ret = btrfs_update_device(trans, device);
4725         if (ret < 0) {
4726                 btrfs_abort_transaction(trans, ret);
4727                 btrfs_end_transaction(trans);
4728         } else {
4729                 ret = btrfs_commit_transaction(trans);
4730         }
4731 done:
4732         btrfs_free_path(path);
4733         if (ret) {
4734                 mutex_lock(&fs_info->chunk_mutex);
4735                 btrfs_device_set_total_bytes(device, old_size);
4736                 if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state))
4737                         device->fs_devices->total_rw_bytes += diff;
4738                 atomic64_add(diff, &fs_info->free_chunk_space);
4739                 mutex_unlock(&fs_info->chunk_mutex);
4740         }
4741         return ret;
4742 }
4743
4744 static int btrfs_add_system_chunk(struct btrfs_fs_info *fs_info,
4745                            struct btrfs_key *key,
4746                            struct btrfs_chunk *chunk, int item_size)
4747 {
4748         struct btrfs_super_block *super_copy = fs_info->super_copy;
4749         struct btrfs_disk_key disk_key;
4750         u32 array_size;
4751         u8 *ptr;
4752
4753         mutex_lock(&fs_info->chunk_mutex);
4754         array_size = btrfs_super_sys_array_size(super_copy);
4755         if (array_size + item_size + sizeof(disk_key)
4756                         > BTRFS_SYSTEM_CHUNK_ARRAY_SIZE) {
4757                 mutex_unlock(&fs_info->chunk_mutex);
4758                 return -EFBIG;
4759         }
4760
4761         ptr = super_copy->sys_chunk_array + array_size;
4762         btrfs_cpu_key_to_disk(&disk_key, key);
4763         memcpy(ptr, &disk_key, sizeof(disk_key));
4764         ptr += sizeof(disk_key);
4765         memcpy(ptr, chunk, item_size);
4766         item_size += sizeof(disk_key);
4767         btrfs_set_super_sys_array_size(super_copy, array_size + item_size);
4768         mutex_unlock(&fs_info->chunk_mutex);
4769
4770         return 0;
4771 }
4772
4773 /*
4774  * sort the devices in descending order by max_avail, total_avail
4775  */
4776 static int btrfs_cmp_device_info(const void *a, const void *b)
4777 {
4778         const struct btrfs_device_info *di_a = a;
4779         const struct btrfs_device_info *di_b = b;
4780
4781         if (di_a->max_avail > di_b->max_avail)
4782                 return -1;
4783         if (di_a->max_avail < di_b->max_avail)
4784                 return 1;
4785         if (di_a->total_avail > di_b->total_avail)
4786                 return -1;
4787         if (di_a->total_avail < di_b->total_avail)
4788                 return 1;
4789         return 0;
4790 }
4791
4792 static void check_raid56_incompat_flag(struct btrfs_fs_info *info, u64 type)
4793 {
4794         if (!(type & BTRFS_BLOCK_GROUP_RAID56_MASK))
4795                 return;
4796
4797         btrfs_set_fs_incompat(info, RAID56);
4798 }
4799
4800 static void check_raid1c34_incompat_flag(struct btrfs_fs_info *info, u64 type)
4801 {
4802         if (!(type & (BTRFS_BLOCK_GROUP_RAID1C3 | BTRFS_BLOCK_GROUP_RAID1C4)))
4803                 return;
4804
4805         btrfs_set_fs_incompat(info, RAID1C34);
4806 }
4807
4808 /*
4809  * Structure used internally for __btrfs_alloc_chunk() function.
4810  * Wraps needed parameters.
4811  */
4812 struct alloc_chunk_ctl {
4813         u64 start;
4814         u64 type;
4815         /* Total number of stripes to allocate */
4816         int num_stripes;
4817         /* sub_stripes info for map */
4818         int sub_stripes;
4819         /* Stripes per device */
4820         int dev_stripes;
4821         /* Maximum number of devices to use */
4822         int devs_max;
4823         /* Minimum number of devices to use */
4824         int devs_min;
4825         /* ndevs has to be a multiple of this */
4826         int devs_increment;
4827         /* Number of copies */
4828         int ncopies;
4829         /* Number of stripes worth of bytes to store parity information */
4830         int nparity;
4831         u64 max_stripe_size;
4832         u64 max_chunk_size;
4833         u64 dev_extent_min;
4834         u64 stripe_size;
4835         u64 chunk_size;
4836         int ndevs;
4837 };
4838
4839 static void init_alloc_chunk_ctl_policy_regular(
4840                                 struct btrfs_fs_devices *fs_devices,
4841                                 struct alloc_chunk_ctl *ctl)
4842 {
4843         u64 type = ctl->type;
4844
4845         if (type & BTRFS_BLOCK_GROUP_DATA) {
4846                 ctl->max_stripe_size = SZ_1G;
4847                 ctl->max_chunk_size = BTRFS_MAX_DATA_CHUNK_SIZE;
4848         } else if (type & BTRFS_BLOCK_GROUP_METADATA) {
4849                 /* For larger filesystems, use larger metadata chunks */
4850                 if (fs_devices->total_rw_bytes > 50ULL * SZ_1G)
4851                         ctl->max_stripe_size = SZ_1G;
4852                 else
4853                         ctl->max_stripe_size = SZ_256M;
4854                 ctl->max_chunk_size = ctl->max_stripe_size;
4855         } else if (type & BTRFS_BLOCK_GROUP_SYSTEM) {
4856                 ctl->max_stripe_size = SZ_32M;
4857                 ctl->max_chunk_size = 2 * ctl->max_stripe_size;
4858                 ctl->devs_max = min_t(int, ctl->devs_max,
4859                                       BTRFS_MAX_DEVS_SYS_CHUNK);
4860         } else {
4861                 BUG();
4862         }
4863
4864         /* We don't want a chunk larger than 10% of writable space */
4865         ctl->max_chunk_size = min(div_factor(fs_devices->total_rw_bytes, 1),
4866                                   ctl->max_chunk_size);
4867         ctl->dev_extent_min = BTRFS_STRIPE_LEN * ctl->dev_stripes;
4868 }
4869
4870 static void init_alloc_chunk_ctl(struct btrfs_fs_devices *fs_devices,
4871                                  struct alloc_chunk_ctl *ctl)
4872 {
4873         int index = btrfs_bg_flags_to_raid_index(ctl->type);
4874
4875         ctl->sub_stripes = btrfs_raid_array[index].sub_stripes;
4876         ctl->dev_stripes = btrfs_raid_array[index].dev_stripes;
4877         ctl->devs_max = btrfs_raid_array[index].devs_max;
4878         if (!ctl->devs_max)
4879                 ctl->devs_max = BTRFS_MAX_DEVS(fs_devices->fs_info);
4880         ctl->devs_min = btrfs_raid_array[index].devs_min;
4881         ctl->devs_increment = btrfs_raid_array[index].devs_increment;
4882         ctl->ncopies = btrfs_raid_array[index].ncopies;
4883         ctl->nparity = btrfs_raid_array[index].nparity;
4884         ctl->ndevs = 0;
4885
4886         switch (fs_devices->chunk_alloc_policy) {
4887         case BTRFS_CHUNK_ALLOC_REGULAR:
4888                 init_alloc_chunk_ctl_policy_regular(fs_devices, ctl);
4889                 break;
4890         default:
4891                 BUG();
4892         }
4893 }
4894
4895 static int gather_device_info(struct btrfs_fs_devices *fs_devices,
4896                               struct alloc_chunk_ctl *ctl,
4897                               struct btrfs_device_info *devices_info)
4898 {
4899         struct btrfs_fs_info *info = fs_devices->fs_info;
4900         struct btrfs_device *device;
4901         u64 total_avail;
4902         u64 dev_extent_want = ctl->max_stripe_size * ctl->dev_stripes;
4903         int ret;
4904         int ndevs = 0;
4905         u64 max_avail;
4906         u64 dev_offset;
4907
4908         /*
4909          * in the first pass through the devices list, we gather information
4910          * about the available holes on each device.
4911          */
4912         list_for_each_entry(device, &fs_devices->alloc_list, dev_alloc_list) {
4913                 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state)) {
4914                         WARN(1, KERN_ERR
4915                                "BTRFS: read-only device in alloc_list\n");
4916                         continue;
4917                 }
4918
4919                 if (!test_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
4920                                         &device->dev_state) ||
4921                     test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state))
4922                         continue;
4923
4924                 if (device->total_bytes > device->bytes_used)
4925                         total_avail = device->total_bytes - device->bytes_used;
4926                 else
4927                         total_avail = 0;
4928
4929                 /* If there is no space on this device, skip it. */
4930                 if (total_avail < ctl->dev_extent_min)
4931                         continue;
4932
4933                 ret = find_free_dev_extent(device, dev_extent_want, &dev_offset,
4934                                            &max_avail);
4935                 if (ret && ret != -ENOSPC)
4936                         return ret;
4937
4938                 if (ret == 0)
4939                         max_avail = dev_extent_want;
4940
4941                 if (max_avail < ctl->dev_extent_min) {
4942                         if (btrfs_test_opt(info, ENOSPC_DEBUG))
4943                                 btrfs_debug(info,
4944                         "%s: devid %llu has no free space, have=%llu want=%llu",
4945                                             __func__, device->devid, max_avail,
4946                                             ctl->dev_extent_min);
4947                         continue;
4948                 }
4949
4950                 if (ndevs == fs_devices->rw_devices) {
4951                         WARN(1, "%s: found more than %llu devices\n",
4952                              __func__, fs_devices->rw_devices);
4953                         break;
4954                 }
4955                 devices_info[ndevs].dev_offset = dev_offset;
4956                 devices_info[ndevs].max_avail = max_avail;
4957                 devices_info[ndevs].total_avail = total_avail;
4958                 devices_info[ndevs].dev = device;
4959                 ++ndevs;
4960         }
4961         ctl->ndevs = ndevs;
4962
4963         /*
4964          * now sort the devices by hole size / available space
4965          */
4966         sort(devices_info, ndevs, sizeof(struct btrfs_device_info),
4967              btrfs_cmp_device_info, NULL);
4968
4969         return 0;
4970 }
4971
4972 static int decide_stripe_size_regular(struct alloc_chunk_ctl *ctl,
4973                                       struct btrfs_device_info *devices_info)
4974 {
4975         /* Number of stripes that count for block group size */
4976         int data_stripes;
4977
4978         /*
4979          * The primary goal is to maximize the number of stripes, so use as
4980          * many devices as possible, even if the stripes are not maximum sized.
4981          *
4982          * The DUP profile stores more than one stripe per device, the
4983          * max_avail is the total size so we have to adjust.
4984          */
4985         ctl->stripe_size = div_u64(devices_info[ctl->ndevs - 1].max_avail,
4986                                    ctl->dev_stripes);
4987         ctl->num_stripes = ctl->ndevs * ctl->dev_stripes;
4988
4989         /* This will have to be fixed for RAID1 and RAID10 over more drives */
4990         data_stripes = (ctl->num_stripes - ctl->nparity) / ctl->ncopies;
4991
4992         /*
4993          * Use the number of data stripes to figure out how big this chunk is
4994          * really going to be in terms of logical address space, and compare
4995          * that answer with the max chunk size. If it's higher, we try to
4996          * reduce stripe_size.
4997          */
4998         if (ctl->stripe_size * data_stripes > ctl->max_chunk_size) {
4999                 /*
5000                  * Reduce stripe_size, round it up to a 16MB boundary again and
5001                  * then use it, unless it ends up being even bigger than the
5002                  * previous value we had already.
5003                  */
5004                 ctl->stripe_size = min(round_up(div_u64(ctl->max_chunk_size,
5005                                                         data_stripes), SZ_16M),
5006                                        ctl->stripe_size);
5007         }
5008
5009         /* Align to BTRFS_STRIPE_LEN */
5010         ctl->stripe_size = round_down(ctl->stripe_size, BTRFS_STRIPE_LEN);
5011         ctl->chunk_size = ctl->stripe_size * data_stripes;
5012
5013         return 0;
5014 }
5015
5016 static int decide_stripe_size(struct btrfs_fs_devices *fs_devices,
5017                               struct alloc_chunk_ctl *ctl,
5018                               struct btrfs_device_info *devices_info)
5019 {
5020         struct btrfs_fs_info *info = fs_devices->fs_info;
5021
5022         /*
5023          * Round down to number of usable stripes, devs_increment can be any
5024          * number so we can't use round_down() that requires power of 2, while
5025          * rounddown is safe.
5026          */
5027         ctl->ndevs = rounddown(ctl->ndevs, ctl->devs_increment);
5028
5029         if (ctl->ndevs < ctl->devs_min) {
5030                 if (btrfs_test_opt(info, ENOSPC_DEBUG)) {
5031                         btrfs_debug(info,
5032         "%s: not enough devices with free space: have=%d minimum required=%d",
5033                                     __func__, ctl->ndevs, ctl->devs_min);
5034                 }
5035                 return -ENOSPC;
5036         }
5037
5038         ctl->ndevs = min(ctl->ndevs, ctl->devs_max);
5039
5040         switch (fs_devices->chunk_alloc_policy) {
5041         case BTRFS_CHUNK_ALLOC_REGULAR:
5042                 return decide_stripe_size_regular(ctl, devices_info);
5043         default:
5044                 BUG();
5045         }
5046 }
5047
5048 static int create_chunk(struct btrfs_trans_handle *trans,
5049                         struct alloc_chunk_ctl *ctl,
5050                         struct btrfs_device_info *devices_info)
5051 {
5052         struct btrfs_fs_info *info = trans->fs_info;
5053         struct map_lookup *map = NULL;
5054         struct extent_map_tree *em_tree;
5055         struct extent_map *em;
5056         u64 start = ctl->start;
5057         u64 type = ctl->type;
5058         int ret;
5059         int i;
5060         int j;
5061
5062         map = kmalloc(map_lookup_size(ctl->num_stripes), GFP_NOFS);
5063         if (!map)
5064                 return -ENOMEM;
5065         map->num_stripes = ctl->num_stripes;
5066
5067         for (i = 0; i < ctl->ndevs; ++i) {
5068                 for (j = 0; j < ctl->dev_stripes; ++j) {
5069                         int s = i * ctl->dev_stripes + j;
5070                         map->stripes[s].dev = devices_info[i].dev;
5071                         map->stripes[s].physical = devices_info[i].dev_offset +
5072                                                    j * ctl->stripe_size;
5073                 }
5074         }
5075         map->stripe_len = BTRFS_STRIPE_LEN;
5076         map->io_align = BTRFS_STRIPE_LEN;
5077         map->io_width = BTRFS_STRIPE_LEN;
5078         map->type = type;
5079         map->sub_stripes = ctl->sub_stripes;
5080
5081         trace_btrfs_chunk_alloc(info, map, start, ctl->chunk_size);
5082
5083         em = alloc_extent_map();
5084         if (!em) {
5085                 kfree(map);
5086                 return -ENOMEM;
5087         }
5088         set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
5089         em->map_lookup = map;
5090         em->start = start;
5091         em->len = ctl->chunk_size;
5092         em->block_start = 0;
5093         em->block_len = em->len;
5094         em->orig_block_len = ctl->stripe_size;
5095
5096         em_tree = &info->mapping_tree;
5097         write_lock(&em_tree->lock);
5098         ret = add_extent_mapping(em_tree, em, 0);
5099         if (ret) {
5100                 write_unlock(&em_tree->lock);
5101                 free_extent_map(em);
5102                 return ret;
5103         }
5104         write_unlock(&em_tree->lock);
5105
5106         ret = btrfs_make_block_group(trans, 0, type, start, ctl->chunk_size);
5107         if (ret)
5108                 goto error_del_extent;
5109
5110         for (i = 0; i < map->num_stripes; i++) {
5111                 struct btrfs_device *dev = map->stripes[i].dev;
5112
5113                 btrfs_device_set_bytes_used(dev,
5114                                             dev->bytes_used + ctl->stripe_size);
5115                 if (list_empty(&dev->post_commit_list))
5116                         list_add_tail(&dev->post_commit_list,
5117                                       &trans->transaction->dev_update_list);
5118         }
5119
5120         atomic64_sub(ctl->stripe_size * map->num_stripes,
5121                      &info->free_chunk_space);
5122
5123         free_extent_map(em);
5124         check_raid56_incompat_flag(info, type);
5125         check_raid1c34_incompat_flag(info, type);
5126
5127         return 0;
5128
5129 error_del_extent:
5130         write_lock(&em_tree->lock);
5131         remove_extent_mapping(em_tree, em);
5132         write_unlock(&em_tree->lock);
5133
5134         /* One for our allocation */
5135         free_extent_map(em);
5136         /* One for the tree reference */
5137         free_extent_map(em);
5138
5139         return ret;
5140 }
5141
5142 int btrfs_alloc_chunk(struct btrfs_trans_handle *trans, u64 type)
5143 {
5144         struct btrfs_fs_info *info = trans->fs_info;
5145         struct btrfs_fs_devices *fs_devices = info->fs_devices;
5146         struct btrfs_device_info *devices_info = NULL;
5147         struct alloc_chunk_ctl ctl;
5148         int ret;
5149
5150         lockdep_assert_held(&info->chunk_mutex);
5151
5152         if (!alloc_profile_is_valid(type, 0)) {
5153                 ASSERT(0);
5154                 return -EINVAL;
5155         }
5156
5157         if (list_empty(&fs_devices->alloc_list)) {
5158                 if (btrfs_test_opt(info, ENOSPC_DEBUG))
5159                         btrfs_debug(info, "%s: no writable device", __func__);
5160                 return -ENOSPC;
5161         }
5162
5163         if (!(type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
5164                 btrfs_err(info, "invalid chunk type 0x%llx requested", type);
5165                 ASSERT(0);
5166                 return -EINVAL;
5167         }
5168
5169         ctl.start = find_next_chunk(info);
5170         ctl.type = type;
5171         init_alloc_chunk_ctl(fs_devices, &ctl);
5172
5173         devices_info = kcalloc(fs_devices->rw_devices, sizeof(*devices_info),
5174                                GFP_NOFS);
5175         if (!devices_info)
5176                 return -ENOMEM;
5177
5178         ret = gather_device_info(fs_devices, &ctl, devices_info);
5179         if (ret < 0)
5180                 goto out;
5181
5182         ret = decide_stripe_size(fs_devices, &ctl, devices_info);
5183         if (ret < 0)
5184                 goto out;
5185
5186         ret = create_chunk(trans, &ctl, devices_info);
5187
5188 out:
5189         kfree(devices_info);
5190         return ret;
5191 }
5192
5193 /*
5194  * Chunk allocation falls into two parts. The first part does work
5195  * that makes the new allocated chunk usable, but does not do any operation
5196  * that modifies the chunk tree. The second part does the work that
5197  * requires modifying the chunk tree. This division is important for the
5198  * bootstrap process of adding storage to a seed btrfs.
5199  */
5200 int btrfs_finish_chunk_alloc(struct btrfs_trans_handle *trans,
5201                              u64 chunk_offset, u64 chunk_size)
5202 {
5203         struct btrfs_fs_info *fs_info = trans->fs_info;
5204         struct btrfs_root *extent_root = fs_info->extent_root;
5205         struct btrfs_root *chunk_root = fs_info->chunk_root;
5206         struct btrfs_key key;
5207         struct btrfs_device *device;
5208         struct btrfs_chunk *chunk;
5209         struct btrfs_stripe *stripe;
5210         struct extent_map *em;
5211         struct map_lookup *map;
5212         size_t item_size;
5213         u64 dev_offset;
5214         u64 stripe_size;
5215         int i = 0;
5216         int ret = 0;
5217
5218         em = btrfs_get_chunk_map(fs_info, chunk_offset, chunk_size);
5219         if (IS_ERR(em))
5220                 return PTR_ERR(em);
5221
5222         map = em->map_lookup;
5223         item_size = btrfs_chunk_item_size(map->num_stripes);
5224         stripe_size = em->orig_block_len;
5225
5226         chunk = kzalloc(item_size, GFP_NOFS);
5227         if (!chunk) {
5228                 ret = -ENOMEM;
5229                 goto out;
5230         }
5231
5232         /*
5233          * Take the device list mutex to prevent races with the final phase of
5234          * a device replace operation that replaces the device object associated
5235          * with the map's stripes, because the device object's id can change
5236          * at any time during that final phase of the device replace operation
5237          * (dev-replace.c:btrfs_dev_replace_finishing()).
5238          */
5239         mutex_lock(&fs_info->fs_devices->device_list_mutex);
5240         for (i = 0; i < map->num_stripes; i++) {
5241                 device = map->stripes[i].dev;
5242                 dev_offset = map->stripes[i].physical;
5243
5244                 ret = btrfs_update_device(trans, device);
5245                 if (ret)
5246                         break;
5247                 ret = btrfs_alloc_dev_extent(trans, device, chunk_offset,
5248                                              dev_offset, stripe_size);
5249                 if (ret)
5250                         break;
5251         }
5252         if (ret) {
5253                 mutex_unlock(&fs_info->fs_devices->device_list_mutex);
5254                 goto out;
5255         }
5256
5257         stripe = &chunk->stripe;
5258         for (i = 0; i < map->num_stripes; i++) {
5259                 device = map->stripes[i].dev;
5260                 dev_offset = map->stripes[i].physical;
5261
5262                 btrfs_set_stack_stripe_devid(stripe, device->devid);
5263                 btrfs_set_stack_stripe_offset(stripe, dev_offset);
5264                 memcpy(stripe->dev_uuid, device->uuid, BTRFS_UUID_SIZE);
5265                 stripe++;
5266         }
5267         mutex_unlock(&fs_info->fs_devices->device_list_mutex);
5268
5269         btrfs_set_stack_chunk_length(chunk, chunk_size);
5270         btrfs_set_stack_chunk_owner(chunk, extent_root->root_key.objectid);
5271         btrfs_set_stack_chunk_stripe_len(chunk, map->stripe_len);
5272         btrfs_set_stack_chunk_type(chunk, map->type);
5273         btrfs_set_stack_chunk_num_stripes(chunk, map->num_stripes);
5274         btrfs_set_stack_chunk_io_align(chunk, map->stripe_len);
5275         btrfs_set_stack_chunk_io_width(chunk, map->stripe_len);
5276         btrfs_set_stack_chunk_sector_size(chunk, fs_info->sectorsize);
5277         btrfs_set_stack_chunk_sub_stripes(chunk, map->sub_stripes);
5278
5279         key.objectid = BTRFS_FIRST_CHUNK_TREE_OBJECTID;
5280         key.type = BTRFS_CHUNK_ITEM_KEY;
5281         key.offset = chunk_offset;
5282
5283         ret = btrfs_insert_item(trans, chunk_root, &key, chunk, item_size);
5284         if (ret == 0 && map->type & BTRFS_BLOCK_GROUP_SYSTEM) {
5285                 /*
5286                  * TODO: Cleanup of inserted chunk root in case of
5287                  * failure.
5288                  */
5289                 ret = btrfs_add_system_chunk(fs_info, &key, chunk, item_size);
5290         }
5291
5292 out:
5293         kfree(chunk);
5294         free_extent_map(em);
5295         return ret;
5296 }
5297
5298 static noinline int init_first_rw_device(struct btrfs_trans_handle *trans)
5299 {
5300         struct btrfs_fs_info *fs_info = trans->fs_info;
5301         u64 alloc_profile;
5302         int ret;
5303
5304         alloc_profile = btrfs_metadata_alloc_profile(fs_info);
5305         ret = btrfs_alloc_chunk(trans, alloc_profile);
5306         if (ret)
5307                 return ret;
5308
5309         alloc_profile = btrfs_system_alloc_profile(fs_info);
5310         ret = btrfs_alloc_chunk(trans, alloc_profile);
5311         return ret;
5312 }
5313
5314 static inline int btrfs_chunk_max_errors(struct map_lookup *map)
5315 {
5316         const int index = btrfs_bg_flags_to_raid_index(map->type);
5317
5318         return btrfs_raid_array[index].tolerated_failures;
5319 }
5320
5321 int btrfs_chunk_readonly(struct btrfs_fs_info *fs_info, u64 chunk_offset)
5322 {
5323         struct extent_map *em;
5324         struct map_lookup *map;
5325         int readonly = 0;
5326         int miss_ndevs = 0;
5327         int i;
5328
5329         em = btrfs_get_chunk_map(fs_info, chunk_offset, 1);
5330         if (IS_ERR(em))
5331                 return 1;
5332
5333         map = em->map_lookup;
5334         for (i = 0; i < map->num_stripes; i++) {
5335                 if (test_bit(BTRFS_DEV_STATE_MISSING,
5336                                         &map->stripes[i].dev->dev_state)) {
5337                         miss_ndevs++;
5338                         continue;
5339                 }
5340                 if (!test_bit(BTRFS_DEV_STATE_WRITEABLE,
5341                                         &map->stripes[i].dev->dev_state)) {
5342                         readonly = 1;
5343                         goto end;
5344                 }
5345         }
5346
5347         /*
5348          * If the number of missing devices is larger than max errors,
5349          * we can not write the data into that chunk successfully, so
5350          * set it readonly.
5351          */
5352         if (miss_ndevs > btrfs_chunk_max_errors(map))
5353                 readonly = 1;
5354 end:
5355         free_extent_map(em);
5356         return readonly;
5357 }
5358
5359 void btrfs_mapping_tree_free(struct extent_map_tree *tree)
5360 {
5361         struct extent_map *em;
5362
5363         while (1) {
5364                 write_lock(&tree->lock);
5365                 em = lookup_extent_mapping(tree, 0, (u64)-1);
5366                 if (em)
5367                         remove_extent_mapping(tree, em);
5368                 write_unlock(&tree->lock);
5369                 if (!em)
5370                         break;
5371                 /* once for us */
5372                 free_extent_map(em);
5373                 /* once for the tree */
5374                 free_extent_map(em);
5375         }
5376 }
5377
5378 int btrfs_num_copies(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5379 {
5380         struct extent_map *em;
5381         struct map_lookup *map;
5382         int ret;
5383
5384         em = btrfs_get_chunk_map(fs_info, logical, len);
5385         if (IS_ERR(em))
5386                 /*
5387                  * We could return errors for these cases, but that could get
5388                  * ugly and we'd probably do the same thing which is just not do
5389                  * anything else and exit, so return 1 so the callers don't try
5390                  * to use other copies.
5391                  */
5392                 return 1;
5393
5394         map = em->map_lookup;
5395         if (map->type & (BTRFS_BLOCK_GROUP_DUP | BTRFS_BLOCK_GROUP_RAID1_MASK))
5396                 ret = map->num_stripes;
5397         else if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5398                 ret = map->sub_stripes;
5399         else if (map->type & BTRFS_BLOCK_GROUP_RAID5)
5400                 ret = 2;
5401         else if (map->type & BTRFS_BLOCK_GROUP_RAID6)
5402                 /*
5403                  * There could be two corrupted data stripes, we need
5404                  * to loop retry in order to rebuild the correct data.
5405                  *
5406                  * Fail a stripe at a time on every retry except the
5407                  * stripe under reconstruction.
5408                  */
5409                 ret = map->num_stripes;
5410         else
5411                 ret = 1;
5412         free_extent_map(em);
5413
5414         down_read(&fs_info->dev_replace.rwsem);
5415         if (btrfs_dev_replace_is_ongoing(&fs_info->dev_replace) &&
5416             fs_info->dev_replace.tgtdev)
5417                 ret++;
5418         up_read(&fs_info->dev_replace.rwsem);
5419
5420         return ret;
5421 }
5422
5423 unsigned long btrfs_full_stripe_len(struct btrfs_fs_info *fs_info,
5424                                     u64 logical)
5425 {
5426         struct extent_map *em;
5427         struct map_lookup *map;
5428         unsigned long len = fs_info->sectorsize;
5429
5430         em = btrfs_get_chunk_map(fs_info, logical, len);
5431
5432         if (!WARN_ON(IS_ERR(em))) {
5433                 map = em->map_lookup;
5434                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5435                         len = map->stripe_len * nr_data_stripes(map);
5436                 free_extent_map(em);
5437         }
5438         return len;
5439 }
5440
5441 int btrfs_is_parity_mirror(struct btrfs_fs_info *fs_info, u64 logical, u64 len)
5442 {
5443         struct extent_map *em;
5444         struct map_lookup *map;
5445         int ret = 0;
5446
5447         em = btrfs_get_chunk_map(fs_info, logical, len);
5448
5449         if(!WARN_ON(IS_ERR(em))) {
5450                 map = em->map_lookup;
5451                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK)
5452                         ret = 1;
5453                 free_extent_map(em);
5454         }
5455         return ret;
5456 }
5457
5458 static int find_live_mirror(struct btrfs_fs_info *fs_info,
5459                             struct map_lookup *map, int first,
5460                             int dev_replace_is_ongoing)
5461 {
5462         int i;
5463         int num_stripes;
5464         int preferred_mirror;
5465         int tolerance;
5466         struct btrfs_device *srcdev;
5467
5468         ASSERT((map->type &
5469                  (BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10)));
5470
5471         if (map->type & BTRFS_BLOCK_GROUP_RAID10)
5472                 num_stripes = map->sub_stripes;
5473         else
5474                 num_stripes = map->num_stripes;
5475
5476         preferred_mirror = first + current->pid % num_stripes;
5477
5478         if (dev_replace_is_ongoing &&
5479             fs_info->dev_replace.cont_reading_from_srcdev_mode ==
5480              BTRFS_DEV_REPLACE_ITEM_CONT_READING_FROM_SRCDEV_MODE_AVOID)
5481                 srcdev = fs_info->dev_replace.srcdev;
5482         else
5483                 srcdev = NULL;
5484
5485         /*
5486          * try to avoid the drive that is the source drive for a
5487          * dev-replace procedure, only choose it if no other non-missing
5488          * mirror is available
5489          */
5490         for (tolerance = 0; tolerance < 2; tolerance++) {
5491                 if (map->stripes[preferred_mirror].dev->bdev &&
5492                     (tolerance || map->stripes[preferred_mirror].dev != srcdev))
5493                         return preferred_mirror;
5494                 for (i = first; i < first + num_stripes; i++) {
5495                         if (map->stripes[i].dev->bdev &&
5496                             (tolerance || map->stripes[i].dev != srcdev))
5497                                 return i;
5498                 }
5499         }
5500
5501         /* we couldn't find one that doesn't fail.  Just return something
5502          * and the io error handling code will clean up eventually
5503          */
5504         return preferred_mirror;
5505 }
5506
5507 /* Bubble-sort the stripe set to put the parity/syndrome stripes last */
5508 static void sort_parity_stripes(struct btrfs_bio *bbio, int num_stripes)
5509 {
5510         int i;
5511         int again = 1;
5512
5513         while (again) {
5514                 again = 0;
5515                 for (i = 0; i < num_stripes - 1; i++) {
5516                         /* Swap if parity is on a smaller index */
5517                         if (bbio->raid_map[i] > bbio->raid_map[i + 1]) {
5518                                 swap(bbio->stripes[i], bbio->stripes[i + 1]);
5519                                 swap(bbio->raid_map[i], bbio->raid_map[i + 1]);
5520                                 again = 1;
5521                         }
5522                 }
5523         }
5524 }
5525
5526 static struct btrfs_bio *alloc_btrfs_bio(int total_stripes, int real_stripes)
5527 {
5528         struct btrfs_bio *bbio = kzalloc(
5529                  /* the size of the btrfs_bio */
5530                 sizeof(struct btrfs_bio) +
5531                 /* plus the variable array for the stripes */
5532                 sizeof(struct btrfs_bio_stripe) * (total_stripes) +
5533                 /* plus the variable array for the tgt dev */
5534                 sizeof(int) * (real_stripes) +
5535                 /*
5536                  * plus the raid_map, which includes both the tgt dev
5537                  * and the stripes
5538                  */
5539                 sizeof(u64) * (total_stripes),
5540                 GFP_NOFS|__GFP_NOFAIL);
5541
5542         atomic_set(&bbio->error, 0);
5543         refcount_set(&bbio->refs, 1);
5544
5545         bbio->tgtdev_map = (int *)(bbio->stripes + total_stripes);
5546         bbio->raid_map = (u64 *)(bbio->tgtdev_map + real_stripes);
5547
5548         return bbio;
5549 }
5550
5551 void btrfs_get_bbio(struct btrfs_bio *bbio)
5552 {
5553         WARN_ON(!refcount_read(&bbio->refs));
5554         refcount_inc(&bbio->refs);
5555 }
5556
5557 void btrfs_put_bbio(struct btrfs_bio *bbio)
5558 {
5559         if (!bbio)
5560                 return;
5561         if (refcount_dec_and_test(&bbio->refs))
5562                 kfree(bbio);
5563 }
5564
5565 /* can REQ_OP_DISCARD be sent with other REQ like REQ_OP_WRITE? */
5566 /*
5567  * Please note that, discard won't be sent to target device of device
5568  * replace.
5569  */
5570 static int __btrfs_map_block_for_discard(struct btrfs_fs_info *fs_info,
5571                                          u64 logical, u64 *length_ret,
5572                                          struct btrfs_bio **bbio_ret)
5573 {
5574         struct extent_map *em;
5575         struct map_lookup *map;
5576         struct btrfs_bio *bbio;
5577         u64 length = *length_ret;
5578         u64 offset;
5579         u64 stripe_nr;
5580         u64 stripe_nr_end;
5581         u64 stripe_end_offset;
5582         u64 stripe_cnt;
5583         u64 stripe_len;
5584         u64 stripe_offset;
5585         u64 num_stripes;
5586         u32 stripe_index;
5587         u32 factor = 0;
5588         u32 sub_stripes = 0;
5589         u64 stripes_per_dev = 0;
5590         u32 remaining_stripes = 0;
5591         u32 last_stripe = 0;
5592         int ret = 0;
5593         int i;
5594
5595         /* discard always return a bbio */
5596         ASSERT(bbio_ret);
5597
5598         em = btrfs_get_chunk_map(fs_info, logical, length);
5599         if (IS_ERR(em))
5600                 return PTR_ERR(em);
5601
5602         map = em->map_lookup;
5603         /* we don't discard raid56 yet */
5604         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5605                 ret = -EOPNOTSUPP;
5606                 goto out;
5607         }
5608
5609         offset = logical - em->start;
5610         length = min_t(u64, em->start + em->len - logical, length);
5611         *length_ret = length;
5612
5613         stripe_len = map->stripe_len;
5614         /*
5615          * stripe_nr counts the total number of stripes we have to stride
5616          * to get to this block
5617          */
5618         stripe_nr = div64_u64(offset, stripe_len);
5619
5620         /* stripe_offset is the offset of this block in its stripe */
5621         stripe_offset = offset - stripe_nr * stripe_len;
5622
5623         stripe_nr_end = round_up(offset + length, map->stripe_len);
5624         stripe_nr_end = div64_u64(stripe_nr_end, map->stripe_len);
5625         stripe_cnt = stripe_nr_end - stripe_nr;
5626         stripe_end_offset = stripe_nr_end * map->stripe_len -
5627                             (offset + length);
5628         /*
5629          * after this, stripe_nr is the number of stripes on this
5630          * device we have to walk to find the data, and stripe_index is
5631          * the number of our device in the stripe array
5632          */
5633         num_stripes = 1;
5634         stripe_index = 0;
5635         if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5636                          BTRFS_BLOCK_GROUP_RAID10)) {
5637                 if (map->type & BTRFS_BLOCK_GROUP_RAID0)
5638                         sub_stripes = 1;
5639                 else
5640                         sub_stripes = map->sub_stripes;
5641
5642                 factor = map->num_stripes / sub_stripes;
5643                 num_stripes = min_t(u64, map->num_stripes,
5644                                     sub_stripes * stripe_cnt);
5645                 stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
5646                 stripe_index *= sub_stripes;
5647                 stripes_per_dev = div_u64_rem(stripe_cnt, factor,
5648                                               &remaining_stripes);
5649                 div_u64_rem(stripe_nr_end - 1, factor, &last_stripe);
5650                 last_stripe *= sub_stripes;
5651         } else if (map->type & (BTRFS_BLOCK_GROUP_RAID1_MASK |
5652                                 BTRFS_BLOCK_GROUP_DUP)) {
5653                 num_stripes = map->num_stripes;
5654         } else {
5655                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
5656                                         &stripe_index);
5657         }
5658
5659         bbio = alloc_btrfs_bio(num_stripes, 0);
5660         if (!bbio) {
5661                 ret = -ENOMEM;
5662                 goto out;
5663         }
5664
5665         for (i = 0; i < num_stripes; i++) {
5666                 bbio->stripes[i].physical =
5667                         map->stripes[stripe_index].physical +
5668                         stripe_offset + stripe_nr * map->stripe_len;
5669                 bbio->stripes[i].dev = map->stripes[stripe_index].dev;
5670
5671                 if (map->type & (BTRFS_BLOCK_GROUP_RAID0 |
5672                                  BTRFS_BLOCK_GROUP_RAID10)) {
5673                         bbio->stripes[i].length = stripes_per_dev *
5674                                 map->stripe_len;
5675
5676                         if (i / sub_stripes < remaining_stripes)
5677                                 bbio->stripes[i].length +=
5678                                         map->stripe_len;
5679
5680                         /*
5681                          * Special for the first stripe and
5682                          * the last stripe:
5683                          *
5684                          * |-------|...|-------|
5685                          *     |----------|
5686                          *    off     end_off
5687                          */
5688                         if (i < sub_stripes)
5689                                 bbio->stripes[i].length -=
5690                                         stripe_offset;
5691
5692                         if (stripe_index >= last_stripe &&
5693                             stripe_index <= (last_stripe +
5694                                              sub_stripes - 1))
5695                                 bbio->stripes[i].length -=
5696                                         stripe_end_offset;
5697
5698                         if (i == sub_stripes - 1)
5699                                 stripe_offset = 0;
5700                 } else {
5701                         bbio->stripes[i].length = length;
5702                 }
5703
5704                 stripe_index++;
5705                 if (stripe_index == map->num_stripes) {
5706                         stripe_index = 0;
5707                         stripe_nr++;
5708                 }
5709         }
5710
5711         *bbio_ret = bbio;
5712         bbio->map_type = map->type;
5713         bbio->num_stripes = num_stripes;
5714 out:
5715         free_extent_map(em);
5716         return ret;
5717 }
5718
5719 /*
5720  * In dev-replace case, for repair case (that's the only case where the mirror
5721  * is selected explicitly when calling btrfs_map_block), blocks left of the
5722  * left cursor can also be read from the target drive.
5723  *
5724  * For REQ_GET_READ_MIRRORS, the target drive is added as the last one to the
5725  * array of stripes.
5726  * For READ, it also needs to be supported using the same mirror number.
5727  *
5728  * If the requested block is not left of the left cursor, EIO is returned. This
5729  * can happen because btrfs_num_copies() returns one more in the dev-replace
5730  * case.
5731  */
5732 static int get_extra_mirror_from_replace(struct btrfs_fs_info *fs_info,
5733                                          u64 logical, u64 length,
5734                                          u64 srcdev_devid, int *mirror_num,
5735                                          u64 *physical)
5736 {
5737         struct btrfs_bio *bbio = NULL;
5738         int num_stripes;
5739         int index_srcdev = 0;
5740         int found = 0;
5741         u64 physical_of_found = 0;
5742         int i;
5743         int ret = 0;
5744
5745         ret = __btrfs_map_block(fs_info, BTRFS_MAP_GET_READ_MIRRORS,
5746                                 logical, &length, &bbio, 0, 0);
5747         if (ret) {
5748                 ASSERT(bbio == NULL);
5749                 return ret;
5750         }
5751
5752         num_stripes = bbio->num_stripes;
5753         if (*mirror_num > num_stripes) {
5754                 /*
5755                  * BTRFS_MAP_GET_READ_MIRRORS does not contain this mirror,
5756                  * that means that the requested area is not left of the left
5757                  * cursor
5758                  */
5759                 btrfs_put_bbio(bbio);
5760                 return -EIO;
5761         }
5762
5763         /*
5764          * process the rest of the function using the mirror_num of the source
5765          * drive. Therefore look it up first.  At the end, patch the device
5766          * pointer to the one of the target drive.
5767          */
5768         for (i = 0; i < num_stripes; i++) {
5769                 if (bbio->stripes[i].dev->devid != srcdev_devid)
5770                         continue;
5771
5772                 /*
5773                  * In case of DUP, in order to keep it simple, only add the
5774                  * mirror with the lowest physical address
5775                  */
5776                 if (found &&
5777                     physical_of_found <= bbio->stripes[i].physical)
5778                         continue;
5779
5780                 index_srcdev = i;
5781                 found = 1;
5782                 physical_of_found = bbio->stripes[i].physical;
5783         }
5784
5785         btrfs_put_bbio(bbio);
5786
5787         ASSERT(found);
5788         if (!found)
5789                 return -EIO;
5790
5791         *mirror_num = index_srcdev + 1;
5792         *physical = physical_of_found;
5793         return ret;
5794 }
5795
5796 static void handle_ops_on_dev_replace(enum btrfs_map_op op,
5797                                       struct btrfs_bio **bbio_ret,
5798                                       struct btrfs_dev_replace *dev_replace,
5799                                       int *num_stripes_ret, int *max_errors_ret)
5800 {
5801         struct btrfs_bio *bbio = *bbio_ret;
5802         u64 srcdev_devid = dev_replace->srcdev->devid;
5803         int tgtdev_indexes = 0;
5804         int num_stripes = *num_stripes_ret;
5805         int max_errors = *max_errors_ret;
5806         int i;
5807
5808         if (op == BTRFS_MAP_WRITE) {
5809                 int index_where_to_add;
5810
5811                 /*
5812                  * duplicate the write operations while the dev replace
5813                  * procedure is running. Since the copying of the old disk to
5814                  * the new disk takes place at run time while the filesystem is
5815                  * mounted writable, the regular write operations to the old
5816                  * disk have to be duplicated to go to the new disk as well.
5817                  *
5818                  * Note that device->missing is handled by the caller, and that
5819                  * the write to the old disk is already set up in the stripes
5820                  * array.
5821                  */
5822                 index_where_to_add = num_stripes;
5823                 for (i = 0; i < num_stripes; i++) {
5824                         if (bbio->stripes[i].dev->devid == srcdev_devid) {
5825                                 /* write to new disk, too */
5826                                 struct btrfs_bio_stripe *new =
5827                                         bbio->stripes + index_where_to_add;
5828                                 struct btrfs_bio_stripe *old =
5829                                         bbio->stripes + i;
5830
5831                                 new->physical = old->physical;
5832                                 new->length = old->length;
5833                                 new->dev = dev_replace->tgtdev;
5834                                 bbio->tgtdev_map[i] = index_where_to_add;
5835                                 index_where_to_add++;
5836                                 max_errors++;
5837                                 tgtdev_indexes++;
5838                         }
5839                 }
5840                 num_stripes = index_where_to_add;
5841         } else if (op == BTRFS_MAP_GET_READ_MIRRORS) {
5842                 int index_srcdev = 0;
5843                 int found = 0;
5844                 u64 physical_of_found = 0;
5845
5846                 /*
5847                  * During the dev-replace procedure, the target drive can also
5848                  * be used to read data in case it is needed to repair a corrupt
5849                  * block elsewhere. This is possible if the requested area is
5850                  * left of the left cursor. In this area, the target drive is a
5851                  * full copy of the source drive.
5852                  */
5853                 for (i = 0; i < num_stripes; i++) {
5854                         if (bbio->stripes[i].dev->devid == srcdev_devid) {
5855                                 /*
5856                                  * In case of DUP, in order to keep it simple,
5857                                  * only add the mirror with the lowest physical
5858                                  * address
5859                                  */
5860                                 if (found &&
5861                                     physical_of_found <=
5862                                      bbio->stripes[i].physical)
5863                                         continue;
5864                                 index_srcdev = i;
5865                                 found = 1;
5866                                 physical_of_found = bbio->stripes[i].physical;
5867                         }
5868                 }
5869                 if (found) {
5870                         struct btrfs_bio_stripe *tgtdev_stripe =
5871                                 bbio->stripes + num_stripes;
5872
5873                         tgtdev_stripe->physical = physical_of_found;
5874                         tgtdev_stripe->length =
5875                                 bbio->stripes[index_srcdev].length;
5876                         tgtdev_stripe->dev = dev_replace->tgtdev;
5877                         bbio->tgtdev_map[index_srcdev] = num_stripes;
5878
5879                         tgtdev_indexes++;
5880                         num_stripes++;
5881                 }
5882         }
5883
5884         *num_stripes_ret = num_stripes;
5885         *max_errors_ret = max_errors;
5886         bbio->num_tgtdevs = tgtdev_indexes;
5887         *bbio_ret = bbio;
5888 }
5889
5890 static bool need_full_stripe(enum btrfs_map_op op)
5891 {
5892         return (op == BTRFS_MAP_WRITE || op == BTRFS_MAP_GET_READ_MIRRORS);
5893 }
5894
5895 /*
5896  * btrfs_get_io_geometry - calculates the geomery of a particular (address, len)
5897  *                     tuple. This information is used to calculate how big a
5898  *                     particular bio can get before it straddles a stripe.
5899  *
5900  * @fs_info - the filesystem
5901  * @logical - address that we want to figure out the geometry of
5902  * @len     - the length of IO we are going to perform, starting at @logical
5903  * @op      - type of operation - write or read
5904  * @io_geom - pointer used to return values
5905  *
5906  * Returns < 0 in case a chunk for the given logical address cannot be found,
5907  * usually shouldn't happen unless @logical is corrupted, 0 otherwise.
5908  */
5909 int btrfs_get_io_geometry(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
5910                         u64 logical, u64 len, struct btrfs_io_geometry *io_geom)
5911 {
5912         struct extent_map *em;
5913         struct map_lookup *map;
5914         u64 offset;
5915         u64 stripe_offset;
5916         u64 stripe_nr;
5917         u64 stripe_len;
5918         u64 raid56_full_stripe_start = (u64)-1;
5919         int data_stripes;
5920         int ret = 0;
5921
5922         ASSERT(op != BTRFS_MAP_DISCARD);
5923
5924         em = btrfs_get_chunk_map(fs_info, logical, len);
5925         if (IS_ERR(em))
5926                 return PTR_ERR(em);
5927
5928         map = em->map_lookup;
5929         /* Offset of this logical address in the chunk */
5930         offset = logical - em->start;
5931         /* Len of a stripe in a chunk */
5932         stripe_len = map->stripe_len;
5933         /* Stripe wher this block falls in */
5934         stripe_nr = div64_u64(offset, stripe_len);
5935         /* Offset of stripe in the chunk */
5936         stripe_offset = stripe_nr * stripe_len;
5937         if (offset < stripe_offset) {
5938                 btrfs_crit(fs_info,
5939 "stripe math has gone wrong, stripe_offset=%llu offset=%llu start=%llu logical=%llu stripe_len=%llu",
5940                         stripe_offset, offset, em->start, logical, stripe_len);
5941                 ret = -EINVAL;
5942                 goto out;
5943         }
5944
5945         /* stripe_offset is the offset of this block in its stripe */
5946         stripe_offset = offset - stripe_offset;
5947         data_stripes = nr_data_stripes(map);
5948
5949         if (map->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
5950                 u64 max_len = stripe_len - stripe_offset;
5951
5952                 /*
5953                  * In case of raid56, we need to know the stripe aligned start
5954                  */
5955                 if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
5956                         unsigned long full_stripe_len = stripe_len * data_stripes;
5957                         raid56_full_stripe_start = offset;
5958
5959                         /*
5960                          * Allow a write of a full stripe, but make sure we
5961                          * don't allow straddling of stripes
5962                          */
5963                         raid56_full_stripe_start = div64_u64(raid56_full_stripe_start,
5964                                         full_stripe_len);
5965                         raid56_full_stripe_start *= full_stripe_len;
5966
5967                         /*
5968                          * For writes to RAID[56], allow a full stripeset across
5969                          * all disks. For other RAID types and for RAID[56]
5970                          * reads, just allow a single stripe (on a single disk).
5971                          */
5972                         if (op == BTRFS_MAP_WRITE) {
5973                                 max_len = stripe_len * data_stripes -
5974                                           (offset - raid56_full_stripe_start);
5975                         }
5976                 }
5977                 len = min_t(u64, em->len - offset, max_len);
5978         } else {
5979                 len = em->len - offset;
5980         }
5981
5982         io_geom->len = len;
5983         io_geom->offset = offset;
5984         io_geom->stripe_len = stripe_len;
5985         io_geom->stripe_nr = stripe_nr;
5986         io_geom->stripe_offset = stripe_offset;
5987         io_geom->raid56_stripe_offset = raid56_full_stripe_start;
5988
5989 out:
5990         /* once for us */
5991         free_extent_map(em);
5992         return ret;
5993 }
5994
5995 static int __btrfs_map_block(struct btrfs_fs_info *fs_info,
5996                              enum btrfs_map_op op,
5997                              u64 logical, u64 *length,
5998                              struct btrfs_bio **bbio_ret,
5999                              int mirror_num, int need_raid_map)
6000 {
6001         struct extent_map *em;
6002         struct map_lookup *map;
6003         u64 stripe_offset;
6004         u64 stripe_nr;
6005         u64 stripe_len;
6006         u32 stripe_index;
6007         int data_stripes;
6008         int i;
6009         int ret = 0;
6010         int num_stripes;
6011         int max_errors = 0;
6012         int tgtdev_indexes = 0;
6013         struct btrfs_bio *bbio = NULL;
6014         struct btrfs_dev_replace *dev_replace = &fs_info->dev_replace;
6015         int dev_replace_is_ongoing = 0;
6016         int num_alloc_stripes;
6017         int patch_the_first_stripe_for_dev_replace = 0;
6018         u64 physical_to_patch_in_first_stripe = 0;
6019         u64 raid56_full_stripe_start = (u64)-1;
6020         struct btrfs_io_geometry geom;
6021
6022         ASSERT(bbio_ret);
6023         ASSERT(op != BTRFS_MAP_DISCARD);
6024
6025         ret = btrfs_get_io_geometry(fs_info, op, logical, *length, &geom);
6026         if (ret < 0)
6027                 return ret;
6028
6029         em = btrfs_get_chunk_map(fs_info, logical, *length);
6030         ASSERT(!IS_ERR(em));
6031         map = em->map_lookup;
6032
6033         *length = geom.len;
6034         stripe_len = geom.stripe_len;
6035         stripe_nr = geom.stripe_nr;
6036         stripe_offset = geom.stripe_offset;
6037         raid56_full_stripe_start = geom.raid56_stripe_offset;
6038         data_stripes = nr_data_stripes(map);
6039
6040         down_read(&dev_replace->rwsem);
6041         dev_replace_is_ongoing = btrfs_dev_replace_is_ongoing(dev_replace);
6042         /*
6043          * Hold the semaphore for read during the whole operation, write is
6044          * requested at commit time but must wait.
6045          */
6046         if (!dev_replace_is_ongoing)
6047                 up_read(&dev_replace->rwsem);
6048
6049         if (dev_replace_is_ongoing && mirror_num == map->num_stripes + 1 &&
6050             !need_full_stripe(op) && dev_replace->tgtdev != NULL) {
6051                 ret = get_extra_mirror_from_replace(fs_info, logical, *length,
6052                                                     dev_replace->srcdev->devid,
6053                                                     &mirror_num,
6054                                             &physical_to_patch_in_first_stripe);
6055                 if (ret)
6056                         goto out;
6057                 else
6058                         patch_the_first_stripe_for_dev_replace = 1;
6059         } else if (mirror_num > map->num_stripes) {
6060                 mirror_num = 0;
6061         }
6062
6063         num_stripes = 1;
6064         stripe_index = 0;
6065         if (map->type & BTRFS_BLOCK_GROUP_RAID0) {
6066                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6067                                 &stripe_index);
6068                 if (!need_full_stripe(op))
6069                         mirror_num = 1;
6070         } else if (map->type & BTRFS_BLOCK_GROUP_RAID1_MASK) {
6071                 if (need_full_stripe(op))
6072                         num_stripes = map->num_stripes;
6073                 else if (mirror_num)
6074                         stripe_index = mirror_num - 1;
6075                 else {
6076                         stripe_index = find_live_mirror(fs_info, map, 0,
6077                                             dev_replace_is_ongoing);
6078                         mirror_num = stripe_index + 1;
6079                 }
6080
6081         } else if (map->type & BTRFS_BLOCK_GROUP_DUP) {
6082                 if (need_full_stripe(op)) {
6083                         num_stripes = map->num_stripes;
6084                 } else if (mirror_num) {
6085                         stripe_index = mirror_num - 1;
6086                 } else {
6087                         mirror_num = 1;
6088                 }
6089
6090         } else if (map->type & BTRFS_BLOCK_GROUP_RAID10) {
6091                 u32 factor = map->num_stripes / map->sub_stripes;
6092
6093                 stripe_nr = div_u64_rem(stripe_nr, factor, &stripe_index);
6094                 stripe_index *= map->sub_stripes;
6095
6096                 if (need_full_stripe(op))
6097                         num_stripes = map->sub_stripes;
6098                 else if (mirror_num)
6099                         stripe_index += mirror_num - 1;
6100                 else {
6101                         int old_stripe_index = stripe_index;
6102                         stripe_index = find_live_mirror(fs_info, map,
6103                                               stripe_index,
6104                                               dev_replace_is_ongoing);
6105                         mirror_num = stripe_index - old_stripe_index + 1;
6106                 }
6107
6108         } else if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK) {
6109                 if (need_raid_map && (need_full_stripe(op) || mirror_num > 1)) {
6110                         /* push stripe_nr back to the start of the full stripe */
6111                         stripe_nr = div64_u64(raid56_full_stripe_start,
6112                                         stripe_len * data_stripes);
6113
6114                         /* RAID[56] write or recovery. Return all stripes */
6115                         num_stripes = map->num_stripes;
6116                         max_errors = nr_parity_stripes(map);
6117
6118                         *length = map->stripe_len;
6119                         stripe_index = 0;
6120                         stripe_offset = 0;
6121                 } else {
6122                         /*
6123                          * Mirror #0 or #1 means the original data block.
6124                          * Mirror #2 is RAID5 parity block.
6125                          * Mirror #3 is RAID6 Q block.
6126                          */
6127                         stripe_nr = div_u64_rem(stripe_nr,
6128                                         data_stripes, &stripe_index);
6129                         if (mirror_num > 1)
6130                                 stripe_index = data_stripes + mirror_num - 2;
6131
6132                         /* We distribute the parity blocks across stripes */
6133                         div_u64_rem(stripe_nr + stripe_index, map->num_stripes,
6134                                         &stripe_index);
6135                         if (!need_full_stripe(op) && mirror_num <= 1)
6136                                 mirror_num = 1;
6137                 }
6138         } else {
6139                 /*
6140                  * after this, stripe_nr is the number of stripes on this
6141                  * device we have to walk to find the data, and stripe_index is
6142                  * the number of our device in the stripe array
6143                  */
6144                 stripe_nr = div_u64_rem(stripe_nr, map->num_stripes,
6145                                 &stripe_index);
6146                 mirror_num = stripe_index + 1;
6147         }
6148         if (stripe_index >= map->num_stripes) {
6149                 btrfs_crit(fs_info,
6150                            "stripe index math went horribly wrong, got stripe_index=%u, num_stripes=%u",
6151                            stripe_index, map->num_stripes);
6152                 ret = -EINVAL;
6153                 goto out;
6154         }
6155
6156         num_alloc_stripes = num_stripes;
6157         if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL) {
6158                 if (op == BTRFS_MAP_WRITE)
6159                         num_alloc_stripes <<= 1;
6160                 if (op == BTRFS_MAP_GET_READ_MIRRORS)
6161                         num_alloc_stripes++;
6162                 tgtdev_indexes = num_stripes;
6163         }
6164
6165         bbio = alloc_btrfs_bio(num_alloc_stripes, tgtdev_indexes);
6166         if (!bbio) {
6167                 ret = -ENOMEM;
6168                 goto out;
6169         }
6170
6171         for (i = 0; i < num_stripes; i++) {
6172                 bbio->stripes[i].physical = map->stripes[stripe_index].physical +
6173                         stripe_offset + stripe_nr * map->stripe_len;
6174                 bbio->stripes[i].dev = map->stripes[stripe_index].dev;
6175                 stripe_index++;
6176         }
6177
6178         /* build raid_map */
6179         if (map->type & BTRFS_BLOCK_GROUP_RAID56_MASK && need_raid_map &&
6180             (need_full_stripe(op) || mirror_num > 1)) {
6181                 u64 tmp;
6182                 unsigned rot;
6183
6184                 /* Work out the disk rotation on this stripe-set */
6185                 div_u64_rem(stripe_nr, num_stripes, &rot);
6186
6187                 /* Fill in the logical address of each stripe */
6188                 tmp = stripe_nr * data_stripes;
6189                 for (i = 0; i < data_stripes; i++)
6190                         bbio->raid_map[(i+rot) % num_stripes] =
6191                                 em->start + (tmp + i) * map->stripe_len;
6192
6193                 bbio->raid_map[(i+rot) % map->num_stripes] = RAID5_P_STRIPE;
6194                 if (map->type & BTRFS_BLOCK_GROUP_RAID6)
6195                         bbio->raid_map[(i+rot+1) % num_stripes] =
6196                                 RAID6_Q_STRIPE;
6197
6198                 sort_parity_stripes(bbio, num_stripes);
6199         }
6200
6201         if (need_full_stripe(op))
6202                 max_errors = btrfs_chunk_max_errors(map);
6203
6204         if (dev_replace_is_ongoing && dev_replace->tgtdev != NULL &&
6205             need_full_stripe(op)) {
6206                 handle_ops_on_dev_replace(op, &bbio, dev_replace, &num_stripes,
6207                                           &max_errors);
6208         }
6209
6210         *bbio_ret = bbio;
6211         bbio->map_type = map->type;
6212         bbio->num_stripes = num_stripes;
6213         bbio->max_errors = max_errors;
6214         bbio->mirror_num = mirror_num;
6215
6216         /*
6217          * this is the case that REQ_READ && dev_replace_is_ongoing &&
6218          * mirror_num == num_stripes + 1 && dev_replace target drive is
6219          * available as a mirror
6220          */
6221         if (patch_the_first_stripe_for_dev_replace && num_stripes > 0) {
6222                 WARN_ON(num_stripes > 1);
6223                 bbio->stripes[0].dev = dev_replace->tgtdev;
6224                 bbio->stripes[0].physical = physical_to_patch_in_first_stripe;
6225                 bbio->mirror_num = map->num_stripes + 1;
6226         }
6227 out:
6228         if (dev_replace_is_ongoing) {
6229                 lockdep_assert_held(&dev_replace->rwsem);
6230                 /* Unlock and let waiting writers proceed */
6231                 up_read(&dev_replace->rwsem);
6232         }
6233         free_extent_map(em);
6234         return ret;
6235 }
6236
6237 int btrfs_map_block(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6238                       u64 logical, u64 *length,
6239                       struct btrfs_bio **bbio_ret, int mirror_num)
6240 {
6241         if (op == BTRFS_MAP_DISCARD)
6242                 return __btrfs_map_block_for_discard(fs_info, logical,
6243                                                      length, bbio_ret);
6244
6245         return __btrfs_map_block(fs_info, op, logical, length, bbio_ret,
6246                                  mirror_num, 0);
6247 }
6248
6249 /* For Scrub/replace */
6250 int btrfs_map_sblock(struct btrfs_fs_info *fs_info, enum btrfs_map_op op,
6251                      u64 logical, u64 *length,
6252                      struct btrfs_bio **bbio_ret)
6253 {
6254         return __btrfs_map_block(fs_info, op, logical, length, bbio_ret, 0, 1);
6255 }
6256
6257 static inline void btrfs_end_bbio(struct btrfs_bio *bbio, struct bio *bio)
6258 {
6259         bio->bi_private = bbio->private;
6260         bio->bi_end_io = bbio->end_io;
6261         bio_endio(bio);
6262
6263         btrfs_put_bbio(bbio);
6264 }
6265
6266 static void btrfs_end_bio(struct bio *bio)
6267 {
6268         struct btrfs_bio *bbio = bio->bi_private;
6269         int is_orig_bio = 0;
6270
6271         if (bio->bi_status) {
6272                 atomic_inc(&bbio->error);
6273                 if (bio->bi_status == BLK_STS_IOERR ||
6274                     bio->bi_status == BLK_STS_TARGET) {
6275                         struct btrfs_device *dev = btrfs_io_bio(bio)->device;
6276
6277                         ASSERT(dev->bdev);
6278                         if (bio_op(bio) == REQ_OP_WRITE)
6279                                 btrfs_dev_stat_inc_and_print(dev,
6280                                                 BTRFS_DEV_STAT_WRITE_ERRS);
6281                         else if (!(bio->bi_opf & REQ_RAHEAD))
6282                                 btrfs_dev_stat_inc_and_print(dev,
6283                                                 BTRFS_DEV_STAT_READ_ERRS);
6284                         if (bio->bi_opf & REQ_PREFLUSH)
6285                                 btrfs_dev_stat_inc_and_print(dev,
6286                                                 BTRFS_DEV_STAT_FLUSH_ERRS);
6287                 }
6288         }
6289
6290         if (bio == bbio->orig_bio)
6291                 is_orig_bio = 1;
6292
6293         btrfs_bio_counter_dec(bbio->fs_info);
6294
6295         if (atomic_dec_and_test(&bbio->stripes_pending)) {
6296                 if (!is_orig_bio) {
6297                         bio_put(bio);
6298                         bio = bbio->orig_bio;
6299                 }
6300
6301                 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6302                 /* only send an error to the higher layers if it is
6303                  * beyond the tolerance of the btrfs bio
6304                  */
6305                 if (atomic_read(&bbio->error) > bbio->max_errors) {
6306                         bio->bi_status = BLK_STS_IOERR;
6307                 } else {
6308                         /*
6309                          * this bio is actually up to date, we didn't
6310                          * go over the max number of errors
6311                          */
6312                         bio->bi_status = BLK_STS_OK;
6313                 }
6314
6315                 btrfs_end_bbio(bbio, bio);
6316         } else if (!is_orig_bio) {
6317                 bio_put(bio);
6318         }
6319 }
6320
6321 static void submit_stripe_bio(struct btrfs_bio *bbio, struct bio *bio,
6322                               u64 physical, struct btrfs_device *dev)
6323 {
6324         struct btrfs_fs_info *fs_info = bbio->fs_info;
6325
6326         bio->bi_private = bbio;
6327         btrfs_io_bio(bio)->device = dev;
6328         bio->bi_end_io = btrfs_end_bio;
6329         bio->bi_iter.bi_sector = physical >> 9;
6330         btrfs_debug_in_rcu(fs_info,
6331         "btrfs_map_bio: rw %d 0x%x, sector=%llu, dev=%lu (%s id %llu), size=%u",
6332                 bio_op(bio), bio->bi_opf, (u64)bio->bi_iter.bi_sector,
6333                 (unsigned long)dev->bdev->bd_dev, rcu_str_deref(dev->name),
6334                 dev->devid, bio->bi_iter.bi_size);
6335         bio_set_dev(bio, dev->bdev);
6336
6337         btrfs_bio_counter_inc_noblocked(fs_info);
6338
6339         btrfsic_submit_bio(bio);
6340 }
6341
6342 static void bbio_error(struct btrfs_bio *bbio, struct bio *bio, u64 logical)
6343 {
6344         atomic_inc(&bbio->error);
6345         if (atomic_dec_and_test(&bbio->stripes_pending)) {
6346                 /* Should be the original bio. */
6347                 WARN_ON(bio != bbio->orig_bio);
6348
6349                 btrfs_io_bio(bio)->mirror_num = bbio->mirror_num;
6350                 bio->bi_iter.bi_sector = logical >> 9;
6351                 if (atomic_read(&bbio->error) > bbio->max_errors)
6352                         bio->bi_status = BLK_STS_IOERR;
6353                 else
6354                         bio->bi_status = BLK_STS_OK;
6355                 btrfs_end_bbio(bbio, bio);
6356         }
6357 }
6358
6359 blk_status_t btrfs_map_bio(struct btrfs_fs_info *fs_info, struct bio *bio,
6360                            int mirror_num)
6361 {
6362         struct btrfs_device *dev;
6363         struct bio *first_bio = bio;
6364         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
6365         u64 length = 0;
6366         u64 map_length;
6367         int ret;
6368         int dev_nr;
6369         int total_devs;
6370         struct btrfs_bio *bbio = NULL;
6371
6372         length = bio->bi_iter.bi_size;
6373         map_length = length;
6374
6375         btrfs_bio_counter_inc_blocked(fs_info);
6376         ret = __btrfs_map_block(fs_info, btrfs_op(bio), logical,
6377                                 &map_length, &bbio, mirror_num, 1);
6378         if (ret) {
6379                 btrfs_bio_counter_dec(fs_info);
6380                 return errno_to_blk_status(ret);
6381         }
6382
6383         total_devs = bbio->num_stripes;
6384         bbio->orig_bio = first_bio;
6385         bbio->private = first_bio->bi_private;
6386         bbio->end_io = first_bio->bi_end_io;
6387         bbio->fs_info = fs_info;
6388         atomic_set(&bbio->stripes_pending, bbio->num_stripes);
6389
6390         if ((bbio->map_type & BTRFS_BLOCK_GROUP_RAID56_MASK) &&
6391             ((bio_op(bio) == REQ_OP_WRITE) || (mirror_num > 1))) {
6392                 /* In this case, map_length has been set to the length of
6393                    a single stripe; not the whole write */
6394                 if (bio_op(bio) == REQ_OP_WRITE) {
6395                         ret = raid56_parity_write(fs_info, bio, bbio,
6396                                                   map_length);
6397                 } else {
6398                         ret = raid56_parity_recover(fs_info, bio, bbio,
6399                                                     map_length, mirror_num, 1);
6400                 }
6401
6402                 btrfs_bio_counter_dec(fs_info);
6403                 return errno_to_blk_status(ret);
6404         }
6405
6406         if (map_length < length) {
6407                 btrfs_crit(fs_info,
6408                            "mapping failed logical %llu bio len %llu len %llu",
6409                            logical, length, map_length);
6410                 BUG();
6411         }
6412
6413         for (dev_nr = 0; dev_nr < total_devs; dev_nr++) {
6414                 dev = bbio->stripes[dev_nr].dev;
6415                 if (!dev || !dev->bdev || test_bit(BTRFS_DEV_STATE_MISSING,
6416                                                    &dev->dev_state) ||
6417                     (bio_op(first_bio) == REQ_OP_WRITE &&
6418                     !test_bit(BTRFS_DEV_STATE_WRITEABLE, &dev->dev_state))) {
6419                         bbio_error(bbio, first_bio, logical);
6420                         continue;
6421                 }
6422
6423                 if (dev_nr < total_devs - 1)
6424                         bio = btrfs_bio_clone(first_bio);
6425                 else
6426                         bio = first_bio;
6427
6428                 submit_stripe_bio(bbio, bio, bbio->stripes[dev_nr].physical, dev);
6429         }
6430         btrfs_bio_counter_dec(fs_info);
6431         return BLK_STS_OK;
6432 }
6433
6434 /*
6435  * Find a device specified by @devid or @uuid in the list of @fs_devices, or
6436  * return NULL.
6437  *
6438  * If devid and uuid are both specified, the match must be exact, otherwise
6439  * only devid is used.
6440  *
6441  * If @seed is true, traverse through the seed devices.
6442  */
6443 struct btrfs_device *btrfs_find_device(struct btrfs_fs_devices *fs_devices,
6444                                        u64 devid, u8 *uuid, u8 *fsid,
6445                                        bool seed)
6446 {
6447         struct btrfs_device *device;
6448         struct btrfs_fs_devices *seed_devs;
6449
6450         if (!fsid || !memcmp(fs_devices->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
6451                 list_for_each_entry(device, &fs_devices->devices, dev_list) {
6452                         if (device->devid == devid &&
6453                             (!uuid || memcmp(device->uuid, uuid,
6454                                              BTRFS_UUID_SIZE) == 0))
6455                                 return device;
6456                 }
6457         }
6458
6459         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
6460                 if (!fsid ||
6461                     !memcmp(seed_devs->metadata_uuid, fsid, BTRFS_FSID_SIZE)) {
6462                         list_for_each_entry(device, &seed_devs->devices,
6463                                             dev_list) {
6464                                 if (device->devid == devid &&
6465                                     (!uuid || memcmp(device->uuid, uuid,
6466                                                      BTRFS_UUID_SIZE) == 0))
6467                                         return device;
6468                         }
6469                 }
6470         }
6471
6472         return NULL;
6473 }
6474
6475 static struct btrfs_device *add_missing_dev(struct btrfs_fs_devices *fs_devices,
6476                                             u64 devid, u8 *dev_uuid)
6477 {
6478         struct btrfs_device *device;
6479         unsigned int nofs_flag;
6480
6481         /*
6482          * We call this under the chunk_mutex, so we want to use NOFS for this
6483          * allocation, however we don't want to change btrfs_alloc_device() to
6484          * always do NOFS because we use it in a lot of other GFP_KERNEL safe
6485          * places.
6486          */
6487         nofs_flag = memalloc_nofs_save();
6488         device = btrfs_alloc_device(NULL, &devid, dev_uuid);
6489         memalloc_nofs_restore(nofs_flag);
6490         if (IS_ERR(device))
6491                 return device;
6492
6493         list_add(&device->dev_list, &fs_devices->devices);
6494         device->fs_devices = fs_devices;
6495         fs_devices->num_devices++;
6496
6497         set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6498         fs_devices->missing_devices++;
6499
6500         return device;
6501 }
6502
6503 /**
6504  * btrfs_alloc_device - allocate struct btrfs_device
6505  * @fs_info:    used only for generating a new devid, can be NULL if
6506  *              devid is provided (i.e. @devid != NULL).
6507  * @devid:      a pointer to devid for this device.  If NULL a new devid
6508  *              is generated.
6509  * @uuid:       a pointer to UUID for this device.  If NULL a new UUID
6510  *              is generated.
6511  *
6512  * Return: a pointer to a new &struct btrfs_device on success; ERR_PTR()
6513  * on error.  Returned struct is not linked onto any lists and must be
6514  * destroyed with btrfs_free_device.
6515  */
6516 struct btrfs_device *btrfs_alloc_device(struct btrfs_fs_info *fs_info,
6517                                         const u64 *devid,
6518                                         const u8 *uuid)
6519 {
6520         struct btrfs_device *dev;
6521         u64 tmp;
6522
6523         if (WARN_ON(!devid && !fs_info))
6524                 return ERR_PTR(-EINVAL);
6525
6526         dev = __alloc_device(fs_info);
6527         if (IS_ERR(dev))
6528                 return dev;
6529
6530         if (devid)
6531                 tmp = *devid;
6532         else {
6533                 int ret;
6534
6535                 ret = find_next_devid(fs_info, &tmp);
6536                 if (ret) {
6537                         btrfs_free_device(dev);
6538                         return ERR_PTR(ret);
6539                 }
6540         }
6541         dev->devid = tmp;
6542
6543         if (uuid)
6544                 memcpy(dev->uuid, uuid, BTRFS_UUID_SIZE);
6545         else
6546                 generate_random_uuid(dev->uuid);
6547
6548         return dev;
6549 }
6550
6551 static void btrfs_report_missing_device(struct btrfs_fs_info *fs_info,
6552                                         u64 devid, u8 *uuid, bool error)
6553 {
6554         if (error)
6555                 btrfs_err_rl(fs_info, "devid %llu uuid %pU is missing",
6556                               devid, uuid);
6557         else
6558                 btrfs_warn_rl(fs_info, "devid %llu uuid %pU is missing",
6559                               devid, uuid);
6560 }
6561
6562 static u64 calc_stripe_length(u64 type, u64 chunk_len, int num_stripes)
6563 {
6564         int index = btrfs_bg_flags_to_raid_index(type);
6565         int ncopies = btrfs_raid_array[index].ncopies;
6566         const int nparity = btrfs_raid_array[index].nparity;
6567         int data_stripes;
6568
6569         if (nparity)
6570                 data_stripes = num_stripes - nparity;
6571         else
6572                 data_stripes = num_stripes / ncopies;
6573
6574         return div_u64(chunk_len, data_stripes);
6575 }
6576
6577 static int read_one_chunk(struct btrfs_key *key, struct extent_buffer *leaf,
6578                           struct btrfs_chunk *chunk)
6579 {
6580         struct btrfs_fs_info *fs_info = leaf->fs_info;
6581         struct extent_map_tree *map_tree = &fs_info->mapping_tree;
6582         struct map_lookup *map;
6583         struct extent_map *em;
6584         u64 logical;
6585         u64 length;
6586         u64 devid;
6587         u8 uuid[BTRFS_UUID_SIZE];
6588         int num_stripes;
6589         int ret;
6590         int i;
6591
6592         logical = key->offset;
6593         length = btrfs_chunk_length(leaf, chunk);
6594         num_stripes = btrfs_chunk_num_stripes(leaf, chunk);
6595
6596         /*
6597          * Only need to verify chunk item if we're reading from sys chunk array,
6598          * as chunk item in tree block is already verified by tree-checker.
6599          */
6600         if (leaf->start == BTRFS_SUPER_INFO_OFFSET) {
6601                 ret = btrfs_check_chunk_valid(leaf, chunk, logical);
6602                 if (ret)
6603                         return ret;
6604         }
6605
6606         read_lock(&map_tree->lock);
6607         em = lookup_extent_mapping(map_tree, logical, 1);
6608         read_unlock(&map_tree->lock);
6609
6610         /* already mapped? */
6611         if (em && em->start <= logical && em->start + em->len > logical) {
6612                 free_extent_map(em);
6613                 return 0;
6614         } else if (em) {
6615                 free_extent_map(em);
6616         }
6617
6618         em = alloc_extent_map();
6619         if (!em)
6620                 return -ENOMEM;
6621         map = kmalloc(map_lookup_size(num_stripes), GFP_NOFS);
6622         if (!map) {
6623                 free_extent_map(em);
6624                 return -ENOMEM;
6625         }
6626
6627         set_bit(EXTENT_FLAG_FS_MAPPING, &em->flags);
6628         em->map_lookup = map;
6629         em->start = logical;
6630         em->len = length;
6631         em->orig_start = 0;
6632         em->block_start = 0;
6633         em->block_len = em->len;
6634
6635         map->num_stripes = num_stripes;
6636         map->io_width = btrfs_chunk_io_width(leaf, chunk);
6637         map->io_align = btrfs_chunk_io_align(leaf, chunk);
6638         map->stripe_len = btrfs_chunk_stripe_len(leaf, chunk);
6639         map->type = btrfs_chunk_type(leaf, chunk);
6640         map->sub_stripes = btrfs_chunk_sub_stripes(leaf, chunk);
6641         map->verified_stripes = 0;
6642         em->orig_block_len = calc_stripe_length(map->type, em->len,
6643                                                 map->num_stripes);
6644         for (i = 0; i < num_stripes; i++) {
6645                 map->stripes[i].physical =
6646                         btrfs_stripe_offset_nr(leaf, chunk, i);
6647                 devid = btrfs_stripe_devid_nr(leaf, chunk, i);
6648                 read_extent_buffer(leaf, uuid, (unsigned long)
6649                                    btrfs_stripe_dev_uuid_nr(chunk, i),
6650                                    BTRFS_UUID_SIZE);
6651                 map->stripes[i].dev = btrfs_find_device(fs_info->fs_devices,
6652                                                         devid, uuid, NULL, true);
6653                 if (!map->stripes[i].dev &&
6654                     !btrfs_test_opt(fs_info, DEGRADED)) {
6655                         free_extent_map(em);
6656                         btrfs_report_missing_device(fs_info, devid, uuid, true);
6657                         return -ENOENT;
6658                 }
6659                 if (!map->stripes[i].dev) {
6660                         map->stripes[i].dev =
6661                                 add_missing_dev(fs_info->fs_devices, devid,
6662                                                 uuid);
6663                         if (IS_ERR(map->stripes[i].dev)) {
6664                                 free_extent_map(em);
6665                                 btrfs_err(fs_info,
6666                                         "failed to init missing dev %llu: %ld",
6667                                         devid, PTR_ERR(map->stripes[i].dev));
6668                                 return PTR_ERR(map->stripes[i].dev);
6669                         }
6670                         btrfs_report_missing_device(fs_info, devid, uuid, false);
6671                 }
6672                 set_bit(BTRFS_DEV_STATE_IN_FS_METADATA,
6673                                 &(map->stripes[i].dev->dev_state));
6674
6675         }
6676
6677         write_lock(&map_tree->lock);
6678         ret = add_extent_mapping(map_tree, em, 0);
6679         write_unlock(&map_tree->lock);
6680         if (ret < 0) {
6681                 btrfs_err(fs_info,
6682                           "failed to add chunk map, start=%llu len=%llu: %d",
6683                           em->start, em->len, ret);
6684         }
6685         free_extent_map(em);
6686
6687         return ret;
6688 }
6689
6690 static void fill_device_from_item(struct extent_buffer *leaf,
6691                                  struct btrfs_dev_item *dev_item,
6692                                  struct btrfs_device *device)
6693 {
6694         unsigned long ptr;
6695
6696         device->devid = btrfs_device_id(leaf, dev_item);
6697         device->disk_total_bytes = btrfs_device_total_bytes(leaf, dev_item);
6698         device->total_bytes = device->disk_total_bytes;
6699         device->commit_total_bytes = device->disk_total_bytes;
6700         device->bytes_used = btrfs_device_bytes_used(leaf, dev_item);
6701         device->commit_bytes_used = device->bytes_used;
6702         device->type = btrfs_device_type(leaf, dev_item);
6703         device->io_align = btrfs_device_io_align(leaf, dev_item);
6704         device->io_width = btrfs_device_io_width(leaf, dev_item);
6705         device->sector_size = btrfs_device_sector_size(leaf, dev_item);
6706         WARN_ON(device->devid == BTRFS_DEV_REPLACE_DEVID);
6707         clear_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state);
6708
6709         ptr = btrfs_device_uuid(dev_item);
6710         read_extent_buffer(leaf, device->uuid, ptr, BTRFS_UUID_SIZE);
6711 }
6712
6713 static struct btrfs_fs_devices *open_seed_devices(struct btrfs_fs_info *fs_info,
6714                                                   u8 *fsid)
6715 {
6716         struct btrfs_fs_devices *fs_devices;
6717         int ret;
6718
6719         lockdep_assert_held(&uuid_mutex);
6720         ASSERT(fsid);
6721
6722         /* This will match only for multi-device seed fs */
6723         list_for_each_entry(fs_devices, &fs_info->fs_devices->seed_list, seed_list)
6724                 if (!memcmp(fs_devices->fsid, fsid, BTRFS_FSID_SIZE))
6725                         return fs_devices;
6726
6727
6728         fs_devices = find_fsid(fsid, NULL);
6729         if (!fs_devices) {
6730                 if (!btrfs_test_opt(fs_info, DEGRADED))
6731                         return ERR_PTR(-ENOENT);
6732
6733                 fs_devices = alloc_fs_devices(fsid, NULL);
6734                 if (IS_ERR(fs_devices))
6735                         return fs_devices;
6736
6737                 fs_devices->seeding = true;
6738                 fs_devices->opened = 1;
6739                 return fs_devices;
6740         }
6741
6742         /*
6743          * Upon first call for a seed fs fsid, just create a private copy of the
6744          * respective fs_devices and anchor it at fs_info->fs_devices->seed_list
6745          */
6746         fs_devices = clone_fs_devices(fs_devices);
6747         if (IS_ERR(fs_devices))
6748                 return fs_devices;
6749
6750         ret = open_fs_devices(fs_devices, FMODE_READ, fs_info->bdev_holder);
6751         if (ret) {
6752                 free_fs_devices(fs_devices);
6753                 return ERR_PTR(ret);
6754         }
6755
6756         if (!fs_devices->seeding) {
6757                 close_fs_devices(fs_devices);
6758                 free_fs_devices(fs_devices);
6759                 return ERR_PTR(-EINVAL);
6760         }
6761
6762         list_add(&fs_devices->seed_list, &fs_info->fs_devices->seed_list);
6763
6764         return fs_devices;
6765 }
6766
6767 static int read_one_dev(struct extent_buffer *leaf,
6768                         struct btrfs_dev_item *dev_item)
6769 {
6770         struct btrfs_fs_info *fs_info = leaf->fs_info;
6771         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
6772         struct btrfs_device *device;
6773         u64 devid;
6774         int ret;
6775         u8 fs_uuid[BTRFS_FSID_SIZE];
6776         u8 dev_uuid[BTRFS_UUID_SIZE];
6777
6778         devid = btrfs_device_id(leaf, dev_item);
6779         read_extent_buffer(leaf, dev_uuid, btrfs_device_uuid(dev_item),
6780                            BTRFS_UUID_SIZE);
6781         read_extent_buffer(leaf, fs_uuid, btrfs_device_fsid(dev_item),
6782                            BTRFS_FSID_SIZE);
6783
6784         if (memcmp(fs_uuid, fs_devices->metadata_uuid, BTRFS_FSID_SIZE)) {
6785                 fs_devices = open_seed_devices(fs_info, fs_uuid);
6786                 if (IS_ERR(fs_devices))
6787                         return PTR_ERR(fs_devices);
6788         }
6789
6790         device = btrfs_find_device(fs_info->fs_devices, devid, dev_uuid,
6791                                    fs_uuid, true);
6792         if (!device) {
6793                 if (!btrfs_test_opt(fs_info, DEGRADED)) {
6794                         btrfs_report_missing_device(fs_info, devid,
6795                                                         dev_uuid, true);
6796                         return -ENOENT;
6797                 }
6798
6799                 device = add_missing_dev(fs_devices, devid, dev_uuid);
6800                 if (IS_ERR(device)) {
6801                         btrfs_err(fs_info,
6802                                 "failed to add missing dev %llu: %ld",
6803                                 devid, PTR_ERR(device));
6804                         return PTR_ERR(device);
6805                 }
6806                 btrfs_report_missing_device(fs_info, devid, dev_uuid, false);
6807         } else {
6808                 if (!device->bdev) {
6809                         if (!btrfs_test_opt(fs_info, DEGRADED)) {
6810                                 btrfs_report_missing_device(fs_info,
6811                                                 devid, dev_uuid, true);
6812                                 return -ENOENT;
6813                         }
6814                         btrfs_report_missing_device(fs_info, devid,
6815                                                         dev_uuid, false);
6816                 }
6817
6818                 if (!device->bdev &&
6819                     !test_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state)) {
6820                         /*
6821                          * this happens when a device that was properly setup
6822                          * in the device info lists suddenly goes bad.
6823                          * device->bdev is NULL, and so we have to set
6824                          * device->missing to one here
6825                          */
6826                         device->fs_devices->missing_devices++;
6827                         set_bit(BTRFS_DEV_STATE_MISSING, &device->dev_state);
6828                 }
6829
6830                 /* Move the device to its own fs_devices */
6831                 if (device->fs_devices != fs_devices) {
6832                         ASSERT(test_bit(BTRFS_DEV_STATE_MISSING,
6833                                                         &device->dev_state));
6834
6835                         list_move(&device->dev_list, &fs_devices->devices);
6836                         device->fs_devices->num_devices--;
6837                         fs_devices->num_devices++;
6838
6839                         device->fs_devices->missing_devices--;
6840                         fs_devices->missing_devices++;
6841
6842                         device->fs_devices = fs_devices;
6843                 }
6844         }
6845
6846         if (device->fs_devices != fs_info->fs_devices) {
6847                 BUG_ON(test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state));
6848                 if (device->generation !=
6849                     btrfs_device_generation(leaf, dev_item))
6850                         return -EINVAL;
6851         }
6852
6853         fill_device_from_item(leaf, dev_item, device);
6854         set_bit(BTRFS_DEV_STATE_IN_FS_METADATA, &device->dev_state);
6855         if (test_bit(BTRFS_DEV_STATE_WRITEABLE, &device->dev_state) &&
6856            !test_bit(BTRFS_DEV_STATE_REPLACE_TGT, &device->dev_state)) {
6857                 device->fs_devices->total_rw_bytes += device->total_bytes;
6858                 atomic64_add(device->total_bytes - device->bytes_used,
6859                                 &fs_info->free_chunk_space);
6860         }
6861         ret = 0;
6862         return ret;
6863 }
6864
6865 int btrfs_read_sys_array(struct btrfs_fs_info *fs_info)
6866 {
6867         struct btrfs_root *root = fs_info->tree_root;
6868         struct btrfs_super_block *super_copy = fs_info->super_copy;
6869         struct extent_buffer *sb;
6870         struct btrfs_disk_key *disk_key;
6871         struct btrfs_chunk *chunk;
6872         u8 *array_ptr;
6873         unsigned long sb_array_offset;
6874         int ret = 0;
6875         u32 num_stripes;
6876         u32 array_size;
6877         u32 len = 0;
6878         u32 cur_offset;
6879         u64 type;
6880         struct btrfs_key key;
6881
6882         ASSERT(BTRFS_SUPER_INFO_SIZE <= fs_info->nodesize);
6883         /*
6884          * This will create extent buffer of nodesize, superblock size is
6885          * fixed to BTRFS_SUPER_INFO_SIZE. If nodesize > sb size, this will
6886          * overallocate but we can keep it as-is, only the first page is used.
6887          */
6888         sb = btrfs_find_create_tree_block(fs_info, BTRFS_SUPER_INFO_OFFSET);
6889         if (IS_ERR(sb))
6890                 return PTR_ERR(sb);
6891         set_extent_buffer_uptodate(sb);
6892         btrfs_set_buffer_lockdep_class(root->root_key.objectid, sb, 0);
6893         /*
6894          * The sb extent buffer is artificial and just used to read the system array.
6895          * set_extent_buffer_uptodate() call does not properly mark all it's
6896          * pages up-to-date when the page is larger: extent does not cover the
6897          * whole page and consequently check_page_uptodate does not find all
6898          * the page's extents up-to-date (the hole beyond sb),
6899          * write_extent_buffer then triggers a WARN_ON.
6900          *
6901          * Regular short extents go through mark_extent_buffer_dirty/writeback cycle,
6902          * but sb spans only this function. Add an explicit SetPageUptodate call
6903          * to silence the warning eg. on PowerPC 64.
6904          */
6905         if (PAGE_SIZE > BTRFS_SUPER_INFO_SIZE)
6906                 SetPageUptodate(sb->pages[0]);
6907
6908         write_extent_buffer(sb, super_copy, 0, BTRFS_SUPER_INFO_SIZE);
6909         array_size = btrfs_super_sys_array_size(super_copy);
6910
6911         array_ptr = super_copy->sys_chunk_array;
6912         sb_array_offset = offsetof(struct btrfs_super_block, sys_chunk_array);
6913         cur_offset = 0;
6914
6915         while (cur_offset < array_size) {
6916                 disk_key = (struct btrfs_disk_key *)array_ptr;
6917                 len = sizeof(*disk_key);
6918                 if (cur_offset + len > array_size)
6919                         goto out_short_read;
6920
6921                 btrfs_disk_key_to_cpu(&key, disk_key);
6922
6923                 array_ptr += len;
6924                 sb_array_offset += len;
6925                 cur_offset += len;
6926
6927                 if (key.type != BTRFS_CHUNK_ITEM_KEY) {
6928                         btrfs_err(fs_info,
6929                             "unexpected item type %u in sys_array at offset %u",
6930                                   (u32)key.type, cur_offset);
6931                         ret = -EIO;
6932                         break;
6933                 }
6934
6935                 chunk = (struct btrfs_chunk *)sb_array_offset;
6936                 /*
6937                  * At least one btrfs_chunk with one stripe must be present,
6938                  * exact stripe count check comes afterwards
6939                  */
6940                 len = btrfs_chunk_item_size(1);
6941                 if (cur_offset + len > array_size)
6942                         goto out_short_read;
6943
6944                 num_stripes = btrfs_chunk_num_stripes(sb, chunk);
6945                 if (!num_stripes) {
6946                         btrfs_err(fs_info,
6947                         "invalid number of stripes %u in sys_array at offset %u",
6948                                   num_stripes, cur_offset);
6949                         ret = -EIO;
6950                         break;
6951                 }
6952
6953                 type = btrfs_chunk_type(sb, chunk);
6954                 if ((type & BTRFS_BLOCK_GROUP_SYSTEM) == 0) {
6955                         btrfs_err(fs_info,
6956                         "invalid chunk type %llu in sys_array at offset %u",
6957                                   type, cur_offset);
6958                         ret = -EIO;
6959                         break;
6960                 }
6961
6962                 len = btrfs_chunk_item_size(num_stripes);
6963                 if (cur_offset + len > array_size)
6964                         goto out_short_read;
6965
6966                 ret = read_one_chunk(&key, sb, chunk);
6967                 if (ret)
6968                         break;
6969
6970                 array_ptr += len;
6971                 sb_array_offset += len;
6972                 cur_offset += len;
6973         }
6974         clear_extent_buffer_uptodate(sb);
6975         free_extent_buffer_stale(sb);
6976         return ret;
6977
6978 out_short_read:
6979         btrfs_err(fs_info, "sys_array too short to read %u bytes at offset %u",
6980                         len, cur_offset);
6981         clear_extent_buffer_uptodate(sb);
6982         free_extent_buffer_stale(sb);
6983         return -EIO;
6984 }
6985
6986 /*
6987  * Check if all chunks in the fs are OK for read-write degraded mount
6988  *
6989  * If the @failing_dev is specified, it's accounted as missing.
6990  *
6991  * Return true if all chunks meet the minimal RW mount requirements.
6992  * Return false if any chunk doesn't meet the minimal RW mount requirements.
6993  */
6994 bool btrfs_check_rw_degradable(struct btrfs_fs_info *fs_info,
6995                                         struct btrfs_device *failing_dev)
6996 {
6997         struct extent_map_tree *map_tree = &fs_info->mapping_tree;
6998         struct extent_map *em;
6999         u64 next_start = 0;
7000         bool ret = true;
7001
7002         read_lock(&map_tree->lock);
7003         em = lookup_extent_mapping(map_tree, 0, (u64)-1);
7004         read_unlock(&map_tree->lock);
7005         /* No chunk at all? Return false anyway */
7006         if (!em) {
7007                 ret = false;
7008                 goto out;
7009         }
7010         while (em) {
7011                 struct map_lookup *map;
7012                 int missing = 0;
7013                 int max_tolerated;
7014                 int i;
7015
7016                 map = em->map_lookup;
7017                 max_tolerated =
7018                         btrfs_get_num_tolerated_disk_barrier_failures(
7019                                         map->type);
7020                 for (i = 0; i < map->num_stripes; i++) {
7021                         struct btrfs_device *dev = map->stripes[i].dev;
7022
7023                         if (!dev || !dev->bdev ||
7024                             test_bit(BTRFS_DEV_STATE_MISSING, &dev->dev_state) ||
7025                             dev->last_flush_error)
7026                                 missing++;
7027                         else if (failing_dev && failing_dev == dev)
7028                                 missing++;
7029                 }
7030                 if (missing > max_tolerated) {
7031                         if (!failing_dev)
7032                                 btrfs_warn(fs_info,
7033         "chunk %llu missing %d devices, max tolerance is %d for writable mount",
7034                                    em->start, missing, max_tolerated);
7035                         free_extent_map(em);
7036                         ret = false;
7037                         goto out;
7038                 }
7039                 next_start = extent_map_end(em);
7040                 free_extent_map(em);
7041
7042                 read_lock(&map_tree->lock);
7043                 em = lookup_extent_mapping(map_tree, next_start,
7044                                            (u64)(-1) - next_start);
7045                 read_unlock(&map_tree->lock);
7046         }
7047 out:
7048         return ret;
7049 }
7050
7051 static void readahead_tree_node_children(struct extent_buffer *node)
7052 {
7053         int i;
7054         const int nr_items = btrfs_header_nritems(node);
7055
7056         for (i = 0; i < nr_items; i++) {
7057                 u64 start;
7058
7059                 start = btrfs_node_blockptr(node, i);
7060                 readahead_tree_block(node->fs_info, start);
7061         }
7062 }
7063
7064 int btrfs_read_chunk_tree(struct btrfs_fs_info *fs_info)
7065 {
7066         struct btrfs_root *root = fs_info->chunk_root;
7067         struct btrfs_path *path;
7068         struct extent_buffer *leaf;
7069         struct btrfs_key key;
7070         struct btrfs_key found_key;
7071         int ret;
7072         int slot;
7073         u64 total_dev = 0;
7074         u64 last_ra_node = 0;
7075
7076         path = btrfs_alloc_path();
7077         if (!path)
7078                 return -ENOMEM;
7079
7080         /*
7081          * uuid_mutex is needed only if we are mounting a sprout FS
7082          * otherwise we don't need it.
7083          */
7084         mutex_lock(&uuid_mutex);
7085
7086         /*
7087          * It is possible for mount and umount to race in such a way that
7088          * we execute this code path, but open_fs_devices failed to clear
7089          * total_rw_bytes. We certainly want it cleared before reading the
7090          * device items, so clear it here.
7091          */
7092         fs_info->fs_devices->total_rw_bytes = 0;
7093
7094         /*
7095          * Read all device items, and then all the chunk items. All
7096          * device items are found before any chunk item (their object id
7097          * is smaller than the lowest possible object id for a chunk
7098          * item - BTRFS_FIRST_CHUNK_TREE_OBJECTID).
7099          */
7100         key.objectid = BTRFS_DEV_ITEMS_OBJECTID;
7101         key.offset = 0;
7102         key.type = 0;
7103         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7104         if (ret < 0)
7105                 goto error;
7106         while (1) {
7107                 struct extent_buffer *node;
7108
7109                 leaf = path->nodes[0];
7110                 slot = path->slots[0];
7111                 if (slot >= btrfs_header_nritems(leaf)) {
7112                         ret = btrfs_next_leaf(root, path);
7113                         if (ret == 0)
7114                                 continue;
7115                         if (ret < 0)
7116                                 goto error;
7117                         break;
7118                 }
7119                 /*
7120                  * The nodes on level 1 are not locked but we don't need to do
7121                  * that during mount time as nothing else can access the tree
7122                  */
7123                 node = path->nodes[1];
7124                 if (node) {
7125                         if (last_ra_node != node->start) {
7126                                 readahead_tree_node_children(node);
7127                                 last_ra_node = node->start;
7128                         }
7129                 }
7130                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
7131                 if (found_key.type == BTRFS_DEV_ITEM_KEY) {
7132                         struct btrfs_dev_item *dev_item;
7133                         dev_item = btrfs_item_ptr(leaf, slot,
7134                                                   struct btrfs_dev_item);
7135                         ret = read_one_dev(leaf, dev_item);
7136                         if (ret)
7137                                 goto error;
7138                         total_dev++;
7139                 } else if (found_key.type == BTRFS_CHUNK_ITEM_KEY) {
7140                         struct btrfs_chunk *chunk;
7141                         chunk = btrfs_item_ptr(leaf, slot, struct btrfs_chunk);
7142                         mutex_lock(&fs_info->chunk_mutex);
7143                         ret = read_one_chunk(&found_key, leaf, chunk);
7144                         mutex_unlock(&fs_info->chunk_mutex);
7145                         if (ret)
7146                                 goto error;
7147                 }
7148                 path->slots[0]++;
7149         }
7150
7151         /*
7152          * After loading chunk tree, we've got all device information,
7153          * do another round of validation checks.
7154          */
7155         if (total_dev != fs_info->fs_devices->total_devices) {
7156                 btrfs_err(fs_info,
7157            "super_num_devices %llu mismatch with num_devices %llu found here",
7158                           btrfs_super_num_devices(fs_info->super_copy),
7159                           total_dev);
7160                 ret = -EINVAL;
7161                 goto error;
7162         }
7163         if (btrfs_super_total_bytes(fs_info->super_copy) <
7164             fs_info->fs_devices->total_rw_bytes) {
7165                 btrfs_err(fs_info,
7166         "super_total_bytes %llu mismatch with fs_devices total_rw_bytes %llu",
7167                           btrfs_super_total_bytes(fs_info->super_copy),
7168                           fs_info->fs_devices->total_rw_bytes);
7169                 ret = -EINVAL;
7170                 goto error;
7171         }
7172         ret = 0;
7173 error:
7174         mutex_unlock(&uuid_mutex);
7175
7176         btrfs_free_path(path);
7177         return ret;
7178 }
7179
7180 void btrfs_init_devices_late(struct btrfs_fs_info *fs_info)
7181 {
7182         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7183         struct btrfs_device *device;
7184
7185         fs_devices->fs_info = fs_info;
7186
7187         mutex_lock(&fs_devices->device_list_mutex);
7188         list_for_each_entry(device, &fs_devices->devices, dev_list)
7189                 device->fs_info = fs_info;
7190
7191         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7192                 list_for_each_entry(device, &seed_devs->devices, dev_list)
7193                         device->fs_info = fs_info;
7194
7195                 seed_devs->fs_info = fs_info;
7196         }
7197         mutex_unlock(&fs_devices->device_list_mutex);
7198 }
7199
7200 static u64 btrfs_dev_stats_value(const struct extent_buffer *eb,
7201                                  const struct btrfs_dev_stats_item *ptr,
7202                                  int index)
7203 {
7204         u64 val;
7205
7206         read_extent_buffer(eb, &val,
7207                            offsetof(struct btrfs_dev_stats_item, values) +
7208                             ((unsigned long)ptr) + (index * sizeof(u64)),
7209                            sizeof(val));
7210         return val;
7211 }
7212
7213 static void btrfs_set_dev_stats_value(struct extent_buffer *eb,
7214                                       struct btrfs_dev_stats_item *ptr,
7215                                       int index, u64 val)
7216 {
7217         write_extent_buffer(eb, &val,
7218                             offsetof(struct btrfs_dev_stats_item, values) +
7219                              ((unsigned long)ptr) + (index * sizeof(u64)),
7220                             sizeof(val));
7221 }
7222
7223 static int btrfs_device_init_dev_stats(struct btrfs_device *device,
7224                                        struct btrfs_path *path)
7225 {
7226         struct btrfs_dev_stats_item *ptr;
7227         struct extent_buffer *eb;
7228         struct btrfs_key key;
7229         int item_size;
7230         int i, ret, slot;
7231
7232         key.objectid = BTRFS_DEV_STATS_OBJECTID;
7233         key.type = BTRFS_PERSISTENT_ITEM_KEY;
7234         key.offset = device->devid;
7235         ret = btrfs_search_slot(NULL, device->fs_info->dev_root, &key, path, 0, 0);
7236         if (ret) {
7237                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7238                         btrfs_dev_stat_set(device, i, 0);
7239                 device->dev_stats_valid = 1;
7240                 btrfs_release_path(path);
7241                 return ret < 0 ? ret : 0;
7242         }
7243         slot = path->slots[0];
7244         eb = path->nodes[0];
7245         item_size = btrfs_item_size_nr(eb, slot);
7246
7247         ptr = btrfs_item_ptr(eb, slot, struct btrfs_dev_stats_item);
7248
7249         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7250                 if (item_size >= (1 + i) * sizeof(__le64))
7251                         btrfs_dev_stat_set(device, i,
7252                                            btrfs_dev_stats_value(eb, ptr, i));
7253                 else
7254                         btrfs_dev_stat_set(device, i, 0);
7255         }
7256
7257         device->dev_stats_valid = 1;
7258         btrfs_dev_stat_print_on_load(device);
7259         btrfs_release_path(path);
7260
7261         return 0;
7262 }
7263
7264 int btrfs_init_dev_stats(struct btrfs_fs_info *fs_info)
7265 {
7266         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices, *seed_devs;
7267         struct btrfs_device *device;
7268         struct btrfs_path *path = NULL;
7269         int ret = 0;
7270
7271         path = btrfs_alloc_path();
7272         if (!path)
7273                 return -ENOMEM;
7274
7275         mutex_lock(&fs_devices->device_list_mutex);
7276         list_for_each_entry(device, &fs_devices->devices, dev_list) {
7277                 ret = btrfs_device_init_dev_stats(device, path);
7278                 if (ret)
7279                         goto out;
7280         }
7281         list_for_each_entry(seed_devs, &fs_devices->seed_list, seed_list) {
7282                 list_for_each_entry(device, &seed_devs->devices, dev_list) {
7283                         ret = btrfs_device_init_dev_stats(device, path);
7284                         if (ret)
7285                                 goto out;
7286                 }
7287         }
7288 out:
7289         mutex_unlock(&fs_devices->device_list_mutex);
7290
7291         btrfs_free_path(path);
7292         return ret;
7293 }
7294
7295 static int update_dev_stat_item(struct btrfs_trans_handle *trans,
7296                                 struct btrfs_device *device)
7297 {
7298         struct btrfs_fs_info *fs_info = trans->fs_info;
7299         struct btrfs_root *dev_root = fs_info->dev_root;
7300         struct btrfs_path *path;
7301         struct btrfs_key key;
7302         struct extent_buffer *eb;
7303         struct btrfs_dev_stats_item *ptr;
7304         int ret;
7305         int i;
7306
7307         key.objectid = BTRFS_DEV_STATS_OBJECTID;
7308         key.type = BTRFS_PERSISTENT_ITEM_KEY;
7309         key.offset = device->devid;
7310
7311         path = btrfs_alloc_path();
7312         if (!path)
7313                 return -ENOMEM;
7314         ret = btrfs_search_slot(trans, dev_root, &key, path, -1, 1);
7315         if (ret < 0) {
7316                 btrfs_warn_in_rcu(fs_info,
7317                         "error %d while searching for dev_stats item for device %s",
7318                               ret, rcu_str_deref(device->name));
7319                 goto out;
7320         }
7321
7322         if (ret == 0 &&
7323             btrfs_item_size_nr(path->nodes[0], path->slots[0]) < sizeof(*ptr)) {
7324                 /* need to delete old one and insert a new one */
7325                 ret = btrfs_del_item(trans, dev_root, path);
7326                 if (ret != 0) {
7327                         btrfs_warn_in_rcu(fs_info,
7328                                 "delete too small dev_stats item for device %s failed %d",
7329                                       rcu_str_deref(device->name), ret);
7330                         goto out;
7331                 }
7332                 ret = 1;
7333         }
7334
7335         if (ret == 1) {
7336                 /* need to insert a new item */
7337                 btrfs_release_path(path);
7338                 ret = btrfs_insert_empty_item(trans, dev_root, path,
7339                                               &key, sizeof(*ptr));
7340                 if (ret < 0) {
7341                         btrfs_warn_in_rcu(fs_info,
7342                                 "insert dev_stats item for device %s failed %d",
7343                                 rcu_str_deref(device->name), ret);
7344                         goto out;
7345                 }
7346         }
7347
7348         eb = path->nodes[0];
7349         ptr = btrfs_item_ptr(eb, path->slots[0], struct btrfs_dev_stats_item);
7350         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7351                 btrfs_set_dev_stats_value(eb, ptr, i,
7352                                           btrfs_dev_stat_read(device, i));
7353         btrfs_mark_buffer_dirty(eb);
7354
7355 out:
7356         btrfs_free_path(path);
7357         return ret;
7358 }
7359
7360 /*
7361  * called from commit_transaction. Writes all changed device stats to disk.
7362  */
7363 int btrfs_run_dev_stats(struct btrfs_trans_handle *trans)
7364 {
7365         struct btrfs_fs_info *fs_info = trans->fs_info;
7366         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7367         struct btrfs_device *device;
7368         int stats_cnt;
7369         int ret = 0;
7370
7371         mutex_lock(&fs_devices->device_list_mutex);
7372         list_for_each_entry(device, &fs_devices->devices, dev_list) {
7373                 stats_cnt = atomic_read(&device->dev_stats_ccnt);
7374                 if (!device->dev_stats_valid || stats_cnt == 0)
7375                         continue;
7376
7377
7378                 /*
7379                  * There is a LOAD-LOAD control dependency between the value of
7380                  * dev_stats_ccnt and updating the on-disk values which requires
7381                  * reading the in-memory counters. Such control dependencies
7382                  * require explicit read memory barriers.
7383                  *
7384                  * This memory barriers pairs with smp_mb__before_atomic in
7385                  * btrfs_dev_stat_inc/btrfs_dev_stat_set and with the full
7386                  * barrier implied by atomic_xchg in
7387                  * btrfs_dev_stats_read_and_reset
7388                  */
7389                 smp_rmb();
7390
7391                 ret = update_dev_stat_item(trans, device);
7392                 if (!ret)
7393                         atomic_sub(stats_cnt, &device->dev_stats_ccnt);
7394         }
7395         mutex_unlock(&fs_devices->device_list_mutex);
7396
7397         return ret;
7398 }
7399
7400 void btrfs_dev_stat_inc_and_print(struct btrfs_device *dev, int index)
7401 {
7402         btrfs_dev_stat_inc(dev, index);
7403         btrfs_dev_stat_print_on_error(dev);
7404 }
7405
7406 static void btrfs_dev_stat_print_on_error(struct btrfs_device *dev)
7407 {
7408         if (!dev->dev_stats_valid)
7409                 return;
7410         btrfs_err_rl_in_rcu(dev->fs_info,
7411                 "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7412                            rcu_str_deref(dev->name),
7413                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7414                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7415                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7416                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7417                            btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7418 }
7419
7420 static void btrfs_dev_stat_print_on_load(struct btrfs_device *dev)
7421 {
7422         int i;
7423
7424         for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7425                 if (btrfs_dev_stat_read(dev, i) != 0)
7426                         break;
7427         if (i == BTRFS_DEV_STAT_VALUES_MAX)
7428                 return; /* all values == 0, suppress message */
7429
7430         btrfs_info_in_rcu(dev->fs_info,
7431                 "bdev %s errs: wr %u, rd %u, flush %u, corrupt %u, gen %u",
7432                rcu_str_deref(dev->name),
7433                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_WRITE_ERRS),
7434                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_READ_ERRS),
7435                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_FLUSH_ERRS),
7436                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_CORRUPTION_ERRS),
7437                btrfs_dev_stat_read(dev, BTRFS_DEV_STAT_GENERATION_ERRS));
7438 }
7439
7440 int btrfs_get_dev_stats(struct btrfs_fs_info *fs_info,
7441                         struct btrfs_ioctl_get_dev_stats *stats)
7442 {
7443         struct btrfs_device *dev;
7444         struct btrfs_fs_devices *fs_devices = fs_info->fs_devices;
7445         int i;
7446
7447         mutex_lock(&fs_devices->device_list_mutex);
7448         dev = btrfs_find_device(fs_info->fs_devices, stats->devid, NULL, NULL,
7449                                 true);
7450         mutex_unlock(&fs_devices->device_list_mutex);
7451
7452         if (!dev) {
7453                 btrfs_warn(fs_info, "get dev_stats failed, device not found");
7454                 return -ENODEV;
7455         } else if (!dev->dev_stats_valid) {
7456                 btrfs_warn(fs_info, "get dev_stats failed, not yet valid");
7457                 return -ENODEV;
7458         } else if (stats->flags & BTRFS_DEV_STATS_RESET) {
7459                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++) {
7460                         if (stats->nr_items > i)
7461                                 stats->values[i] =
7462                                         btrfs_dev_stat_read_and_reset(dev, i);
7463                         else
7464                                 btrfs_dev_stat_set(dev, i, 0);
7465                 }
7466                 btrfs_info(fs_info, "device stats zeroed by %s (%d)",
7467                            current->comm, task_pid_nr(current));
7468         } else {
7469                 for (i = 0; i < BTRFS_DEV_STAT_VALUES_MAX; i++)
7470                         if (stats->nr_items > i)
7471                                 stats->values[i] = btrfs_dev_stat_read(dev, i);
7472         }
7473         if (stats->nr_items > BTRFS_DEV_STAT_VALUES_MAX)
7474                 stats->nr_items = BTRFS_DEV_STAT_VALUES_MAX;
7475         return 0;
7476 }
7477
7478 /*
7479  * Update the size and bytes used for each device where it changed.  This is
7480  * delayed since we would otherwise get errors while writing out the
7481  * superblocks.
7482  *
7483  * Must be invoked during transaction commit.
7484  */
7485 void btrfs_commit_device_sizes(struct btrfs_transaction *trans)
7486 {
7487         struct btrfs_device *curr, *next;
7488
7489         ASSERT(trans->state == TRANS_STATE_COMMIT_DOING);
7490
7491         if (list_empty(&trans->dev_update_list))
7492                 return;
7493
7494         /*
7495          * We don't need the device_list_mutex here.  This list is owned by the
7496          * transaction and the transaction must complete before the device is
7497          * released.
7498          */
7499         mutex_lock(&trans->fs_info->chunk_mutex);
7500         list_for_each_entry_safe(curr, next, &trans->dev_update_list,
7501                                  post_commit_list) {
7502                 list_del_init(&curr->post_commit_list);
7503                 curr->commit_total_bytes = curr->disk_total_bytes;
7504                 curr->commit_bytes_used = curr->bytes_used;
7505         }
7506         mutex_unlock(&trans->fs_info->chunk_mutex);
7507 }
7508
7509 /*
7510  * Multiplicity factor for simple profiles: DUP, RAID1-like and RAID10.
7511  */
7512 int btrfs_bg_type_to_factor(u64 flags)
7513 {
7514         const int index = btrfs_bg_flags_to_raid_index(flags);
7515
7516         return btrfs_raid_array[index].ncopies;
7517 }
7518
7519
7520
7521 static int verify_one_dev_extent(struct btrfs_fs_info *fs_info,
7522                                  u64 chunk_offset, u64 devid,
7523                                  u64 physical_offset, u64 physical_len)
7524 {
7525         struct extent_map_tree *em_tree = &fs_info->mapping_tree;
7526         struct extent_map *em;
7527         struct map_lookup *map;
7528         struct btrfs_device *dev;
7529         u64 stripe_len;
7530         bool found = false;
7531         int ret = 0;
7532         int i;
7533
7534         read_lock(&em_tree->lock);
7535         em = lookup_extent_mapping(em_tree, chunk_offset, 1);
7536         read_unlock(&em_tree->lock);
7537
7538         if (!em) {
7539                 btrfs_err(fs_info,
7540 "dev extent physical offset %llu on devid %llu doesn't have corresponding chunk",
7541                           physical_offset, devid);
7542                 ret = -EUCLEAN;
7543                 goto out;
7544         }
7545
7546         map = em->map_lookup;
7547         stripe_len = calc_stripe_length(map->type, em->len, map->num_stripes);
7548         if (physical_len != stripe_len) {
7549                 btrfs_err(fs_info,
7550 "dev extent physical offset %llu on devid %llu length doesn't match chunk %llu, have %llu expect %llu",
7551                           physical_offset, devid, em->start, physical_len,
7552                           stripe_len);
7553                 ret = -EUCLEAN;
7554                 goto out;
7555         }
7556
7557         for (i = 0; i < map->num_stripes; i++) {
7558                 if (map->stripes[i].dev->devid == devid &&
7559                     map->stripes[i].physical == physical_offset) {
7560                         found = true;
7561                         if (map->verified_stripes >= map->num_stripes) {
7562                                 btrfs_err(fs_info,
7563                                 "too many dev extents for chunk %llu found",
7564                                           em->start);
7565                                 ret = -EUCLEAN;
7566                                 goto out;
7567                         }
7568                         map->verified_stripes++;
7569                         break;
7570                 }
7571         }
7572         if (!found) {
7573                 btrfs_err(fs_info,
7574         "dev extent physical offset %llu devid %llu has no corresponding chunk",
7575                         physical_offset, devid);
7576                 ret = -EUCLEAN;
7577         }
7578
7579         /* Make sure no dev extent is beyond device bondary */
7580         dev = btrfs_find_device(fs_info->fs_devices, devid, NULL, NULL, true);
7581         if (!dev) {
7582                 btrfs_err(fs_info, "failed to find devid %llu", devid);
7583                 ret = -EUCLEAN;
7584                 goto out;
7585         }
7586
7587         /* It's possible this device is a dummy for seed device */
7588         if (dev->disk_total_bytes == 0) {
7589                 struct btrfs_fs_devices *devs;
7590
7591                 devs = list_first_entry(&fs_info->fs_devices->seed_list,
7592                                         struct btrfs_fs_devices, seed_list);
7593                 dev = btrfs_find_device(devs, devid, NULL, NULL, false);
7594                 if (!dev) {
7595                         btrfs_err(fs_info, "failed to find seed devid %llu",
7596                                   devid);
7597                         ret = -EUCLEAN;
7598                         goto out;
7599                 }
7600         }
7601
7602         if (physical_offset + physical_len > dev->disk_total_bytes) {
7603                 btrfs_err(fs_info,
7604 "dev extent devid %llu physical offset %llu len %llu is beyond device boundary %llu",
7605                           devid, physical_offset, physical_len,
7606                           dev->disk_total_bytes);
7607                 ret = -EUCLEAN;
7608                 goto out;
7609         }
7610 out:
7611         free_extent_map(em);
7612         return ret;
7613 }
7614
7615 static int verify_chunk_dev_extent_mapping(struct btrfs_fs_info *fs_info)
7616 {
7617         struct extent_map_tree *em_tree = &fs_info->mapping_tree;
7618         struct extent_map *em;
7619         struct rb_node *node;
7620         int ret = 0;
7621
7622         read_lock(&em_tree->lock);
7623         for (node = rb_first_cached(&em_tree->map); node; node = rb_next(node)) {
7624                 em = rb_entry(node, struct extent_map, rb_node);
7625                 if (em->map_lookup->num_stripes !=
7626                     em->map_lookup->verified_stripes) {
7627                         btrfs_err(fs_info,
7628                         "chunk %llu has missing dev extent, have %d expect %d",
7629                                   em->start, em->map_lookup->verified_stripes,
7630                                   em->map_lookup->num_stripes);
7631                         ret = -EUCLEAN;
7632                         goto out;
7633                 }
7634         }
7635 out:
7636         read_unlock(&em_tree->lock);
7637         return ret;
7638 }
7639
7640 /*
7641  * Ensure that all dev extents are mapped to correct chunk, otherwise
7642  * later chunk allocation/free would cause unexpected behavior.
7643  *
7644  * NOTE: This will iterate through the whole device tree, which should be of
7645  * the same size level as the chunk tree.  This slightly increases mount time.
7646  */
7647 int btrfs_verify_dev_extents(struct btrfs_fs_info *fs_info)
7648 {
7649         struct btrfs_path *path;
7650         struct btrfs_root *root = fs_info->dev_root;
7651         struct btrfs_key key;
7652         u64 prev_devid = 0;
7653         u64 prev_dev_ext_end = 0;
7654         int ret = 0;
7655
7656         key.objectid = 1;
7657         key.type = BTRFS_DEV_EXTENT_KEY;
7658         key.offset = 0;
7659
7660         path = btrfs_alloc_path();
7661         if (!path)
7662                 return -ENOMEM;
7663
7664         path->reada = READA_FORWARD;
7665         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
7666         if (ret < 0)
7667                 goto out;
7668
7669         if (path->slots[0] >= btrfs_header_nritems(path->nodes[0])) {
7670                 ret = btrfs_next_item(root, path);
7671                 if (ret < 0)
7672                         goto out;
7673                 /* No dev extents at all? Not good */
7674                 if (ret > 0) {
7675                         ret = -EUCLEAN;
7676                         goto out;
7677                 }
7678         }
7679         while (1) {
7680                 struct extent_buffer *leaf = path->nodes[0];
7681                 struct btrfs_dev_extent *dext;
7682                 int slot = path->slots[0];
7683                 u64 chunk_offset;
7684                 u64 physical_offset;
7685                 u64 physical_len;
7686                 u64 devid;
7687
7688                 btrfs_item_key_to_cpu(leaf, &key, slot);
7689                 if (key.type != BTRFS_DEV_EXTENT_KEY)
7690                         break;
7691                 devid = key.objectid;
7692                 physical_offset = key.offset;
7693
7694                 dext = btrfs_item_ptr(leaf, slot, struct btrfs_dev_extent);
7695                 chunk_offset = btrfs_dev_extent_chunk_offset(leaf, dext);
7696                 physical_len = btrfs_dev_extent_length(leaf, dext);
7697
7698                 /* Check if this dev extent overlaps with the previous one */
7699                 if (devid == prev_devid && physical_offset < prev_dev_ext_end) {
7700                         btrfs_err(fs_info,
7701 "dev extent devid %llu physical offset %llu overlap with previous dev extent end %llu",
7702                                   devid, physical_offset, prev_dev_ext_end);
7703                         ret = -EUCLEAN;
7704                         goto out;
7705                 }
7706
7707                 ret = verify_one_dev_extent(fs_info, chunk_offset, devid,
7708                                             physical_offset, physical_len);
7709                 if (ret < 0)
7710                         goto out;
7711                 prev_devid = devid;
7712                 prev_dev_ext_end = physical_offset + physical_len;
7713
7714                 ret = btrfs_next_item(root, path);
7715                 if (ret < 0)
7716                         goto out;
7717                 if (ret > 0) {
7718                         ret = 0;
7719                         break;
7720                 }
7721         }
7722
7723         /* Ensure all chunks have corresponding dev extents */
7724         ret = verify_chunk_dev_extent_mapping(fs_info);
7725 out:
7726         btrfs_free_path(path);
7727         return ret;
7728 }
7729
7730 /*
7731  * Check whether the given block group or device is pinned by any inode being
7732  * used as a swapfile.
7733  */
7734 bool btrfs_pinned_by_swapfile(struct btrfs_fs_info *fs_info, void *ptr)
7735 {
7736         struct btrfs_swapfile_pin *sp;
7737         struct rb_node *node;
7738
7739         spin_lock(&fs_info->swapfile_pins_lock);
7740         node = fs_info->swapfile_pins.rb_node;
7741         while (node) {
7742                 sp = rb_entry(node, struct btrfs_swapfile_pin, node);
7743                 if (ptr < sp->ptr)
7744                         node = node->rb_left;
7745                 else if (ptr > sp->ptr)
7746                         node = node->rb_right;
7747                 else
7748                         break;
7749         }
7750         spin_unlock(&fs_info->swapfile_pins_lock);
7751         return node != NULL;
7752 }