fs/btrfs/inode.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/kernel.h>
   7 #include <linux/bio.h>
   8 #include <linux/buffer_head.h>
   9 #include <linux/file.h>
  10 #include <linux/fs.h>
  11 #include <linux/pagemap.h>
  12 #include <linux/highmem.h>
  13 #include <linux/time.h>
  14 #include <linux/init.h>
  15 #include <linux/string.h>
  16 #include <linux/backing-dev.h>
  17 #include <linux/mpage.h>
  18 #include <linux/swap.h>
  19 #include <linux/writeback.h>
  20 #include <linux/compat.h>
  21 #include <linux/bit_spinlock.h>
  22 #include <linux/xattr.h>
  23 #include <linux/posix_acl.h>
  24 #include <linux/falloc.h>
  25 #include <linux/slab.h>
  26 #include <linux/ratelimit.h>
  27 #include <linux/mount.h>
  28 #include <linux/btrfs.h>
  29 #include <linux/blkdev.h>
  30 #include <linux/posix_acl_xattr.h>
  31 #include <linux/uio.h>
  32 #include <linux/magic.h>
  33 #include <linux/iversion.h>
  34 #include "ctree.h"
  35 #include "disk-io.h"
  36 #include "transaction.h"
  37 #include "btrfs_inode.h"
  38 #include "print-tree.h"
  39 #include "ordered-data.h"
  40 #include "xattr.h"
  41 #include "tree-log.h"
  42 #include "volumes.h"
  43 #include "compression.h"
  44 #include "locking.h"
  45 #include "free-space-cache.h"
  46 #include "inode-map.h"
  47 #include "backref.h"
  48 #include "props.h"
  49 #include "qgroup.h"
  50 #include "dedupe.h"
  51
  52 struct btrfs_iget_args {
  53         struct btrfs_key *location;
  54         struct btrfs_root *root;
  55 };
  56
  57 struct btrfs_dio_data {
  58         u64 reserve;
  59         u64 unsubmitted_oe_range_start;
  60         u64 unsubmitted_oe_range_end;
  61         int overwrite;
  62 };
  63
  64 static const struct inode_operations btrfs_dir_inode_operations;
  65 static const struct inode_operations btrfs_symlink_inode_operations;
  66 static const struct inode_operations btrfs_dir_ro_inode_operations;
  67 static const struct inode_operations btrfs_special_inode_operations;
  68 static const struct inode_operations btrfs_file_inode_operations;
  69 static const struct address_space_operations btrfs_aops;
  70 static const struct address_space_operations btrfs_symlink_aops;
  71 static const struct file_operations btrfs_dir_file_operations;
  72 static const struct extent_io_ops btrfs_extent_io_ops;
  73
  74 static struct kmem_cache *btrfs_inode_cachep;
  75 struct kmem_cache *btrfs_trans_handle_cachep;
  76 struct kmem_cache *btrfs_path_cachep;
  77 struct kmem_cache *btrfs_free_space_cachep;
  78
  79 #define S_SHIFT 12
  80 static const unsigned char btrfs_type_by_mode[S_IFMT >> S_SHIFT] = {
  81         [S_IFREG >> S_SHIFT]    = BTRFS_FT_REG_FILE,
  82         [S_IFDIR >> S_SHIFT]    = BTRFS_FT_DIR,
  83         [S_IFCHR >> S_SHIFT]    = BTRFS_FT_CHRDEV,
  84         [S_IFBLK >> S_SHIFT]    = BTRFS_FT_BLKDEV,
  85         [S_IFIFO >> S_SHIFT]    = BTRFS_FT_FIFO,
  86         [S_IFSOCK >> S_SHIFT]   = BTRFS_FT_SOCK,
  87         [S_IFLNK >> S_SHIFT]    = BTRFS_FT_SYMLINK,
  88 };
  89
  90 static int btrfs_setsize(struct inode *inode, struct iattr *attr);
  91 static int btrfs_truncate(struct inode *inode, bool skip_writeback);
  92 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
  93 static noinline int cow_file_range(struct inode *inode,
  94                                    struct page *locked_page,
  95                                    u64 start, u64 end, u64 delalloc_end,
  96                                    int *page_started, unsigned long *nr_written,
  97                                    int unlock, struct btrfs_dedupe_hash *hash);
  98 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
  99                                        u64 orig_start, u64 block_start,
 100                                        u64 block_len, u64 orig_block_len,
 101                                        u64 ram_bytes, int compress_type,
 102                                        int type);
 103
 104 static void __endio_write_update_ordered(struct inode *inode,
 105                                          const u64 offset, const u64 bytes,
 106                                          const bool uptodate);
 107
 108 /*
 109  * Cleanup all submitted ordered extents in specified range to handle errors
 110  * from the fill_dellaloc() callback.
 111  *
 112  * NOTE: caller must ensure that when an error happens, it can not call
 113  * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
 114  * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
 115  * to be released, which we want to happen only when finishing the ordered
 116  * extent (btrfs_finish_ordered_io()). Also note that the caller of the
 117  * fill_delalloc() callback already does proper cleanup for the first page of
 118  * the range, that is, it invokes the callback writepage_end_io_hook() for the
 119  * range of the first page.
 120  */
 121 static inline void btrfs_cleanup_ordered_extents(struct inode *inode,
 122                                                  const u64 offset,
 123                                                  const u64 bytes)
 124 {
 125         unsigned long index = offset >> PAGE_SHIFT;
 126         unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
 127         struct page *page;
 128
 129         while (index <= end_index) {
 130                 page = find_get_page(inode->i_mapping, index);
 131                 index++;
 132                 if (!page)
 133                         continue;
 134                 ClearPagePrivate2(page);
 135                 put_page(page);
 136         }
 137         return __endio_write_update_ordered(inode, offset + PAGE_SIZE,
 138                                             bytes - PAGE_SIZE, false);
 139 }
 140
 141 static int btrfs_dirty_inode(struct inode *inode);
 142
 143 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
 144 void btrfs_test_inode_set_ops(struct inode *inode)
 145 {
 146         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
 147 }
 148 #endif
 149
 150 static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
 151                                      struct inode *inode,  struct inode *dir,
 152                                      const struct qstr *qstr)
 153 {
 154         int err;
 155
 156         err = btrfs_init_acl(trans, inode, dir);
 157         if (!err)
 158                 err = btrfs_xattr_security_init(trans, inode, dir, qstr);
 159         return err;
 160 }
 161
 162 /*
 163  * this does all the hard work for inserting an inline extent into
 164  * the btree.  The caller should have done a btrfs_drop_extents so that
 165  * no overlapping inline items exist in the btree
 166  */
 167 static int insert_inline_extent(struct btrfs_trans_handle *trans,
 168                                 struct btrfs_path *path, int extent_inserted,
 169                                 struct btrfs_root *root, struct inode *inode,
 170                                 u64 start, size_t size, size_t compressed_size,
 171                                 int compress_type,
 172                                 struct page **compressed_pages)
 173 {
 174         struct extent_buffer *leaf;
 175         struct page *page = NULL;
 176         char *kaddr;
 177         unsigned long ptr;
 178         struct btrfs_file_extent_item *ei;
 179         int ret;
 180         size_t cur_size = size;
 181         unsigned long offset;
 182
 183         if (compressed_size && compressed_pages)
 184                 cur_size = compressed_size;
 185
 186         inode_add_bytes(inode, size);
 187
 188         if (!extent_inserted) {
 189                 struct btrfs_key key;
 190                 size_t datasize;
 191
 192                 key.objectid = btrfs_ino(BTRFS_I(inode));
 193                 key.offset = start;
 194                 key.type = BTRFS_EXTENT_DATA_KEY;
 195
 196                 datasize = btrfs_file_extent_calc_inline_size(cur_size);
 197                 path->leave_spinning = 1;
 198                 ret = btrfs_insert_empty_item(trans, root, path, &key,
 199                                               datasize);
 200                 if (ret)
 201                         goto fail;
 202         }
 203         leaf = path->nodes[0];
 204         ei = btrfs_item_ptr(leaf, path->slots[0],
 205                             struct btrfs_file_extent_item);
 206         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
 207         btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
 208         btrfs_set_file_extent_encryption(leaf, ei, 0);
 209         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
 210         btrfs_set_file_extent_ram_bytes(leaf, ei, size);
 211         ptr = btrfs_file_extent_inline_start(ei);
 212
 213         if (compress_type != BTRFS_COMPRESS_NONE) {
 214                 struct page *cpage;
 215                 int i = 0;
 216                 while (compressed_size > 0) {
 217                         cpage = compressed_pages[i];
 218                         cur_size = min_t(unsigned long, compressed_size,
 219                                        PAGE_SIZE);
 220
 221                         kaddr = kmap_atomic(cpage);
 222                         write_extent_buffer(leaf, kaddr, ptr, cur_size);
 223                         kunmap_atomic(kaddr);
 224
 225                         i++;
 226                         ptr += cur_size;
 227                         compressed_size -= cur_size;
 228                 }
 229                 btrfs_set_file_extent_compression(leaf, ei,
 230                                                   compress_type);
 231         } else {
 232                 page = find_get_page(inode->i_mapping,
 233                                      start >> PAGE_SHIFT);
 234                 btrfs_set_file_extent_compression(leaf, ei, 0);
 235                 kaddr = kmap_atomic(page);
 236                 offset = start & (PAGE_SIZE - 1);
 237                 write_extent_buffer(leaf, kaddr + offset, ptr, size);
 238                 kunmap_atomic(kaddr);
 239                 put_page(page);
 240         }
 241         btrfs_mark_buffer_dirty(leaf);
 242         btrfs_release_path(path);
 243
 244         /*
 245          * we're an inline extent, so nobody can
 246          * extend the file past i_size without locking
 247          * a page we already have locked.
 248          *
 249          * We must do any isize and inode updates
 250          * before we unlock the pages.  Otherwise we
 251          * could end up racing with unlink.
 252          */
 253         BTRFS_I(inode)->disk_i_size = inode->i_size;
 254         ret = btrfs_update_inode(trans, root, inode);
 255
 256 fail:
 257         return ret;
 258 }
 259
 260
 261 /*
 262  * conditionally insert an inline extent into the file.  This
 263  * does the checks required to make sure the data is small enough
 264  * to fit as an inline extent.
 265  */
 266 static noinline int cow_file_range_inline(struct inode *inode, u64 start,
 267                                           u64 end, size_t compressed_size,
 268                                           int compress_type,
 269                                           struct page **compressed_pages)
 270 {
 271         struct btrfs_root *root = BTRFS_I(inode)->root;
 272         struct btrfs_fs_info *fs_info = root->fs_info;
 273         struct btrfs_trans_handle *trans;
 274         u64 isize = i_size_read(inode);
 275         u64 actual_end = min(end + 1, isize);
 276         u64 inline_len = actual_end - start;
 277         u64 aligned_end = ALIGN(end, fs_info->sectorsize);
 278         u64 data_len = inline_len;
 279         int ret;
 280         struct btrfs_path *path;
 281         int extent_inserted = 0;
 282         u32 extent_item_size;
 283
 284         if (compressed_size)
 285                 data_len = compressed_size;
 286
 287         if (start > 0 ||
 288             actual_end > fs_info->sectorsize ||
 289             data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
 290             (!compressed_size &&
 291             (actual_end & (fs_info->sectorsize - 1)) == 0) ||
 292             end + 1 < isize ||
 293             data_len > fs_info->max_inline) {
 294                 return 1;
 295         }
 296
 297         path = btrfs_alloc_path();
 298         if (!path)
 299                 return -ENOMEM;
 300
 301         trans = btrfs_join_transaction(root);
 302         if (IS_ERR(trans)) {
 303                 btrfs_free_path(path);
 304                 return PTR_ERR(trans);
 305         }
 306         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
 307
 308         if (compressed_size && compressed_pages)
 309                 extent_item_size = btrfs_file_extent_calc_inline_size(
 310                    compressed_size);
 311         else
 312                 extent_item_size = btrfs_file_extent_calc_inline_size(
 313                     inline_len);
 314
 315         ret = __btrfs_drop_extents(trans, root, inode, path,
 316                                    start, aligned_end, NULL,
 317                                    1, 1, extent_item_size, &extent_inserted);
 318         if (ret) {
 319                 btrfs_abort_transaction(trans, ret);
 320                 goto out;
 321         }
 322
 323         if (isize > actual_end)
 324                 inline_len = min_t(u64, isize, actual_end);
 325         ret = insert_inline_extent(trans, path, extent_inserted,
 326                                    root, inode, start,
 327                                    inline_len, compressed_size,
 328                                    compress_type, compressed_pages);
 329         if (ret && ret != -ENOSPC) {
 330                 btrfs_abort_transaction(trans, ret);
 331                 goto out;
 332         } else if (ret == -ENOSPC) {
 333                 ret = 1;
 334                 goto out;
 335         }
 336
 337         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
 338         btrfs_drop_extent_cache(BTRFS_I(inode), start, aligned_end - 1, 0);
 339 out:
 340         /*
 341          * Don't forget to free the reserved space, as for inlined extent
 342          * it won't count as data extent, free them directly here.
 343          * And at reserve time, it's always aligned to page size, so
 344          * just free one page here.
 345          */
 346         btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
 347         btrfs_free_path(path);
 348         btrfs_end_transaction(trans);
 349         return ret;
 350 }
 351
 352 struct async_extent {
 353         u64 start;
 354         u64 ram_size;
 355         u64 compressed_size;
 356         struct page **pages;
 357         unsigned long nr_pages;
 358         int compress_type;
 359         struct list_head list;
 360 };
 361
 362 struct async_cow {
 363         struct inode *inode;
 364         struct btrfs_root *root;
 365         struct page *locked_page;
 366         u64 start;
 367         u64 end;
 368         unsigned int write_flags;
 369         struct list_head extents;
 370         struct btrfs_work work;
 371 };
 372
 373 static noinline int add_async_extent(struct async_cow *cow,
 374                                      u64 start, u64 ram_size,
 375                                      u64 compressed_size,
 376                                      struct page **pages,
 377                                      unsigned long nr_pages,
 378                                      int compress_type)
 379 {
 380         struct async_extent *async_extent;
 381
 382         async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
 383         BUG_ON(!async_extent); /* -ENOMEM */
 384         async_extent->start = start;
 385         async_extent->ram_size = ram_size;
 386         async_extent->compressed_size = compressed_size;
 387         async_extent->pages = pages;
 388         async_extent->nr_pages = nr_pages;
 389         async_extent->compress_type = compress_type;
 390         list_add_tail(&async_extent->list, &cow->extents);
 391         return 0;
 392 }
 393
 394 static inline int inode_need_compress(struct inode *inode, u64 start, u64 end)
 395 {
 396         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 397
 398         /* force compress */
 399         if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
 400                 return 1;
 401         /* defrag ioctl */
 402         if (BTRFS_I(inode)->defrag_compress)
 403                 return 1;
 404         /* bad compression ratios */
 405         if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS)
 406                 return 0;
 407         if (btrfs_test_opt(fs_info, COMPRESS) ||
 408             BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS ||
 409             BTRFS_I(inode)->prop_compress)
 410                 return btrfs_compress_heuristic(inode, start, end);
 411         return 0;
 412 }
 413
 414 static inline void inode_should_defrag(struct btrfs_inode *inode,
 415                 u64 start, u64 end, u64 num_bytes, u64 small_write)
 416 {
 417         /* If this is a small write inside eof, kick off a defrag */
 418         if (num_bytes < small_write &&
 419             (start > 0 || end + 1 < inode->disk_i_size))
 420                 btrfs_add_inode_defrag(NULL, inode);
 421 }
 422
 423 /*
 424  * we create compressed extents in two phases.  The first
 425  * phase compresses a range of pages that have already been
 426  * locked (both pages and state bits are locked).
 427  *
 428  * This is done inside an ordered work queue, and the compression
 429  * is spread across many cpus.  The actual IO submission is step
 430  * two, and the ordered work queue takes care of making sure that
 431  * happens in the same order things were put onto the queue by
 432  * writepages and friends.
 433  *
 434  * If this code finds it can't get good compression, it puts an
 435  * entry onto the work queue to write the uncompressed bytes.  This
 436  * makes sure that both compressed inodes and uncompressed inodes
 437  * are written in the same order that the flusher thread sent them
 438  * down.
 439  */
 440 static noinline void compress_file_range(struct inode *inode,
 441                                         struct page *locked_page,
 442                                         u64 start, u64 end,
 443                                         struct async_cow *async_cow,
 444                                         int *num_added)
 445 {
 446         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 447         u64 blocksize = fs_info->sectorsize;
 448         u64 actual_end;
 449         u64 isize = i_size_read(inode);
 450         int ret = 0;
 451         struct page **pages = NULL;
 452         unsigned long nr_pages;
 453         unsigned long total_compressed = 0;
 454         unsigned long total_in = 0;
 455         int i;
 456         int will_compress;
 457         int compress_type = fs_info->compress_type;
 458         int redirty = 0;
 459
 460         inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
 461                         SZ_16K);
 462
 463         actual_end = min_t(u64, isize, end + 1);
 464 again:
 465         will_compress = 0;
 466         nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
 467         BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
 468         nr_pages = min_t(unsigned long, nr_pages,
 469                         BTRFS_MAX_COMPRESSED / PAGE_SIZE);
 470
 471         /*
 472          * we don't want to send crud past the end of i_size through
 473          * compression, that's just a waste of CPU time.  So, if the
 474          * end of the file is before the start of our current
 475          * requested range of bytes, we bail out to the uncompressed
 476          * cleanup code that can deal with all of this.
 477          *
 478          * It isn't really the fastest way to fix things, but this is a
 479          * very uncommon corner.
 480          */
 481         if (actual_end <= start)
 482                 goto cleanup_and_bail_uncompressed;
 483
 484         total_compressed = actual_end - start;
 485
 486         /*
 487          * skip compression for a small file range(<=blocksize) that
 488          * isn't an inline extent, since it doesn't save disk space at all.
 489          */
 490         if (total_compressed <= blocksize &&
 491            (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
 492                 goto cleanup_and_bail_uncompressed;
 493
 494         total_compressed = min_t(unsigned long, total_compressed,
 495                         BTRFS_MAX_UNCOMPRESSED);
 496         total_in = 0;
 497         ret = 0;
 498
 499         /*
 500          * we do compression for mount -o compress and when the
 501          * inode has not been flagged as nocompress.  This flag can
 502          * change at any time if we discover bad compression ratios.
 503          */
 504         if (inode_need_compress(inode, start, end)) {
 505                 WARN_ON(pages);
 506                 pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
 507                 if (!pages) {
 508                         /* just bail out to the uncompressed code */
 509                         goto cont;
 510                 }
 511
 512                 if (BTRFS_I(inode)->defrag_compress)
 513                         compress_type = BTRFS_I(inode)->defrag_compress;
 514                 else if (BTRFS_I(inode)->prop_compress)
 515                         compress_type = BTRFS_I(inode)->prop_compress;
 516
 517                 /*
 518                  * we need to call clear_page_dirty_for_io on each
 519                  * page in the range.  Otherwise applications with the file
 520                  * mmap'd can wander in and change the page contents while
 521                  * we are compressing them.
 522                  *
 523                  * If the compression fails for any reason, we set the pages
 524                  * dirty again later on.
 525                  *
 526                  * Note that the remaining part is redirtied, the start pointer
 527                  * has moved, the end is the original one.
 528                  */
 529                 if (!redirty) {
 530                         extent_range_clear_dirty_for_io(inode, start, end);
 531                         redirty = 1;
 532                 }
 533
 534                 /* Compression level is applied here and only here */
 535                 ret = btrfs_compress_pages(
 536                         compress_type | (fs_info->compress_level << 4),
 537                                            inode->i_mapping, start,
 538                                            pages,
 539                                            &nr_pages,
 540                                            &total_in,
 541                                            &total_compressed);
 542
 543                 if (!ret) {
 544                         unsigned long offset = total_compressed &
 545                                 (PAGE_SIZE - 1);
 546                         struct page *page = pages[nr_pages - 1];
 547                         char *kaddr;
 548
 549                         /* zero the tail end of the last page, we might be
 550                          * sending it down to disk
 551                          */
 552                         if (offset) {
 553                                 kaddr = kmap_atomic(page);
 554                                 memset(kaddr + offset, 0,
 555                                        PAGE_SIZE - offset);
 556                                 kunmap_atomic(kaddr);
 557                         }
 558                         will_compress = 1;
 559                 }
 560         }
 561 cont:
 562         if (start == 0) {
 563                 /* lets try to make an inline extent */
 564                 if (ret || total_in < actual_end) {
 565                         /* we didn't compress the entire range, try
 566                          * to make an uncompressed inline extent.
 567                          */
 568                         ret = cow_file_range_inline(inode, start, end, 0,
 569                                                     BTRFS_COMPRESS_NONE, NULL);
 570                 } else {
 571                         /* try making a compressed inline extent */
 572                         ret = cow_file_range_inline(inode, start, end,
 573                                                     total_compressed,
 574                                                     compress_type, pages);
 575                 }
 576                 if (ret <= 0) {
 577                         unsigned long clear_flags = EXTENT_DELALLOC |
 578                                 EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
 579                                 EXTENT_DO_ACCOUNTING;
 580                         unsigned long page_error_op;
 581
 582                         page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
 583
 584                         /*
 585                          * inline extent creation worked or returned error,
 586                          * we don't need to create any more async work items.
 587                          * Unlock and free up our temp pages.
 588                          *
 589                          * We use DO_ACCOUNTING here because we need the
 590                          * delalloc_release_metadata to be done _after_ we drop
 591                          * our outstanding extent for clearing delalloc for this
 592                          * range.
 593                          */
 594                         extent_clear_unlock_delalloc(inode, start, end, end,
 595                                                      NULL, clear_flags,
 596                                                      PAGE_UNLOCK |
 597                                                      PAGE_CLEAR_DIRTY |
 598                                                      PAGE_SET_WRITEBACK |
 599                                                      page_error_op |
 600                                                      PAGE_END_WRITEBACK);
 601                         goto free_pages_out;
 602                 }
 603         }
 604
 605         if (will_compress) {
 606                 /*
 607                  * we aren't doing an inline extent round the compressed size
 608                  * up to a block size boundary so the allocator does sane
 609                  * things
 610                  */
 611                 total_compressed = ALIGN(total_compressed, blocksize);
 612
 613                 /*
 614                  * one last check to make sure the compression is really a
 615                  * win, compare the page count read with the blocks on disk,
 616                  * compression must free at least one sector size
 617                  */
 618                 total_in = ALIGN(total_in, PAGE_SIZE);
 619                 if (total_compressed + blocksize <= total_in) {
 620                         *num_added += 1;
 621
 622                         /*
 623                          * The async work queues will take care of doing actual
 624                          * allocation on disk for these compressed pages, and
 625                          * will submit them to the elevator.
 626                          */
 627                         add_async_extent(async_cow, start, total_in,
 628                                         total_compressed, pages, nr_pages,
 629                                         compress_type);
 630
 631                         if (start + total_in < end) {
 632                                 start += total_in;
 633                                 pages = NULL;
 634                                 cond_resched();
 635                                 goto again;
 636                         }
 637                         return;
 638                 }
 639         }
 640         if (pages) {
 641                 /*
 642                  * the compression code ran but failed to make things smaller,
 643                  * free any pages it allocated and our page pointer array
 644                  */
 645                 for (i = 0; i < nr_pages; i++) {
 646                         WARN_ON(pages[i]->mapping);
 647                         put_page(pages[i]);
 648                 }
 649                 kfree(pages);
 650                 pages = NULL;
 651                 total_compressed = 0;
 652                 nr_pages = 0;
 653
 654                 /* flag the file so we don't compress in the future */
 655                 if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
 656                     !(BTRFS_I(inode)->prop_compress)) {
 657                         BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
 658                 }
 659         }
 660 cleanup_and_bail_uncompressed:
 661         /*
 662          * No compression, but we still need to write the pages in the file
 663          * we've been given so far.  redirty the locked page if it corresponds
 664          * to our extent and set things up for the async work queue to run
 665          * cow_file_range to do the normal delalloc dance.
 666          */
 667         if (page_offset(locked_page) >= start &&
 668             page_offset(locked_page) <= end)
 669                 __set_page_dirty_nobuffers(locked_page);
 670                 /* unlocked later on in the async handlers */
 671
 672         if (redirty)
 673                 extent_range_redirty_for_io(inode, start, end);
 674         add_async_extent(async_cow, start, end - start + 1, 0, NULL, 0,
 675                          BTRFS_COMPRESS_NONE);
 676         *num_added += 1;
 677
 678         return;
 679
 680 free_pages_out:
 681         for (i = 0; i < nr_pages; i++) {
 682                 WARN_ON(pages[i]->mapping);
 683                 put_page(pages[i]);
 684         }
 685         kfree(pages);
 686 }
 687
 688 static void free_async_extent_pages(struct async_extent *async_extent)
 689 {
 690         int i;
 691
 692         if (!async_extent->pages)
 693                 return;
 694
 695         for (i = 0; i < async_extent->nr_pages; i++) {
 696                 WARN_ON(async_extent->pages[i]->mapping);
 697                 put_page(async_extent->pages[i]);
 698         }
 699         kfree(async_extent->pages);
 700         async_extent->nr_pages = 0;
 701         async_extent->pages = NULL;
 702 }
 703
 704 /*
 705  * phase two of compressed writeback.  This is the ordered portion
 706  * of the code, which only gets called in the order the work was
 707  * queued.  We walk all the async extents created by compress_file_range
 708  * and send them down to the disk.
 709  */
 710 static noinline void submit_compressed_extents(struct inode *inode,
 711                                               struct async_cow *async_cow)
 712 {
 713         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 714         struct async_extent *async_extent;
 715         u64 alloc_hint = 0;
 716         struct btrfs_key ins;
 717         struct extent_map *em;
 718         struct btrfs_root *root = BTRFS_I(inode)->root;
 719         struct extent_io_tree *io_tree;
 720         int ret = 0;
 721
 722 again:
 723         while (!list_empty(&async_cow->extents)) {
 724                 async_extent = list_entry(async_cow->extents.next,
 725                                           struct async_extent, list);
 726                 list_del(&async_extent->list);
 727
 728                 io_tree = &BTRFS_I(inode)->io_tree;
 729
 730 retry:
 731                 /* did the compression code fall back to uncompressed IO? */
 732                 if (!async_extent->pages) {
 733                         int page_started = 0;
 734                         unsigned long nr_written = 0;
 735
 736                         lock_extent(io_tree, async_extent->start,
 737                                          async_extent->start +
 738                                          async_extent->ram_size - 1);
 739
 740                         /* allocate blocks */
 741                         ret = cow_file_range(inode, async_cow->locked_page,
 742                                              async_extent->start,
 743                                              async_extent->start +
 744                                              async_extent->ram_size - 1,
 745                                              async_extent->start +
 746                                              async_extent->ram_size - 1,
 747                                              &page_started, &nr_written, 0,
 748                                              NULL);
 749
 750                         /* JDM XXX */
 751
 752                         /*
 753                          * if page_started, cow_file_range inserted an
 754                          * inline extent and took care of all the unlocking
 755                          * and IO for us.  Otherwise, we need to submit
 756                          * all those pages down to the drive.
 757                          */
 758                         if (!page_started && !ret)
 759                                 extent_write_locked_range(inode,
 760                                                   async_extent->start,
 761                                                   async_extent->start +
 762                                                   async_extent->ram_size - 1,
 763                                                   WB_SYNC_ALL);
 764                         else if (ret)
 765                                 unlock_page(async_cow->locked_page);
 766                         kfree(async_extent);
 767                         cond_resched();
 768                         continue;
 769                 }
 770
 771                 lock_extent(io_tree, async_extent->start,
 772                             async_extent->start + async_extent->ram_size - 1);
 773
 774                 ret = btrfs_reserve_extent(root, async_extent->ram_size,
 775                                            async_extent->compressed_size,
 776                                            async_extent->compressed_size,
 777                                            0, alloc_hint, &ins, 1, 1);
 778                 if (ret) {
 779                         free_async_extent_pages(async_extent);
 780
 781                         if (ret == -ENOSPC) {
 782                                 unlock_extent(io_tree, async_extent->start,
 783                                               async_extent->start +
 784                                               async_extent->ram_size - 1);
 785
 786                                 /*
 787                                  * we need to redirty the pages if we decide to
 788                                  * fallback to uncompressed IO, otherwise we
 789                                  * will not submit these pages down to lower
 790                                  * layers.
 791                                  */
 792                                 extent_range_redirty_for_io(inode,
 793                                                 async_extent->start,
 794                                                 async_extent->start +
 795                                                 async_extent->ram_size - 1);
 796
 797                                 goto retry;
 798                         }
 799                         goto out_free;
 800                 }
 801                 /*
 802                  * here we're doing allocation and writeback of the
 803                  * compressed pages
 804                  */
 805                 em = create_io_em(inode, async_extent->start,
 806                                   async_extent->ram_size, /* len */
 807                                   async_extent->start, /* orig_start */
 808                                   ins.objectid, /* block_start */
 809                                   ins.offset, /* block_len */
 810                                   ins.offset, /* orig_block_len */
 811                                   async_extent->ram_size, /* ram_bytes */
 812                                   async_extent->compress_type,
 813                                   BTRFS_ORDERED_COMPRESSED);
 814                 if (IS_ERR(em))
 815                         /* ret value is not necessary due to void function */
 816                         goto out_free_reserve;
 817                 free_extent_map(em);
 818
 819                 ret = btrfs_add_ordered_extent_compress(inode,
 820                                                 async_extent->start,
 821                                                 ins.objectid,
 822                                                 async_extent->ram_size,
 823                                                 ins.offset,
 824                                                 BTRFS_ORDERED_COMPRESSED,
 825                                                 async_extent->compress_type);
 826                 if (ret) {
 827                         btrfs_drop_extent_cache(BTRFS_I(inode),
 828                                                 async_extent->start,
 829                                                 async_extent->start +
 830                                                 async_extent->ram_size - 1, 0);
 831                         goto out_free_reserve;
 832                 }
 833                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 834
 835                 /*
 836                  * clear dirty, set writeback and unlock the pages.
 837                  */
 838                 extent_clear_unlock_delalloc(inode, async_extent->start,
 839                                 async_extent->start +
 840                                 async_extent->ram_size - 1,
 841                                 async_extent->start +
 842                                 async_extent->ram_size - 1,
 843                                 NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
 844                                 PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 845                                 PAGE_SET_WRITEBACK);
 846                 if (btrfs_submit_compressed_write(inode,
 847                                     async_extent->start,
 848                                     async_extent->ram_size,
 849                                     ins.objectid,
 850                                     ins.offset, async_extent->pages,
 851                                     async_extent->nr_pages,
 852                                     async_cow->write_flags)) {
 853                         struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
 854                         struct page *p = async_extent->pages[0];
 855                         const u64 start = async_extent->start;
 856                         const u64 end = start + async_extent->ram_size - 1;
 857
 858                         p->mapping = inode->i_mapping;
 859                         tree->ops->writepage_end_io_hook(p, start, end,
 860                                                          NULL, 0);
 861                         p->mapping = NULL;
 862                         extent_clear_unlock_delalloc(inode, start, end, end,
 863                                                      NULL, 0,
 864                                                      PAGE_END_WRITEBACK |
 865                                                      PAGE_SET_ERROR);
 866                         free_async_extent_pages(async_extent);
 867                 }
 868                 alloc_hint = ins.objectid + ins.offset;
 869                 kfree(async_extent);
 870                 cond_resched();
 871         }
 872         return;
 873 out_free_reserve:
 874         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
 875         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
 876 out_free:
 877         extent_clear_unlock_delalloc(inode, async_extent->start,
 878                                      async_extent->start +
 879                                      async_extent->ram_size - 1,
 880                                      async_extent->start +
 881                                      async_extent->ram_size - 1,
 882                                      NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
 883                                      EXTENT_DELALLOC_NEW |
 884                                      EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
 885                                      PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
 886                                      PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
 887                                      PAGE_SET_ERROR);
 888         free_async_extent_pages(async_extent);
 889         kfree(async_extent);
 890         goto again;
 891 }
 892
 893 static u64 get_extent_allocation_hint(struct inode *inode, u64 start,
 894                                       u64 num_bytes)
 895 {
 896         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
 897         struct extent_map *em;
 898         u64 alloc_hint = 0;
 899
 900         read_lock(&em_tree->lock);
 901         em = search_extent_mapping(em_tree, start, num_bytes);
 902         if (em) {
 903                 /*
 904                  * if block start isn't an actual block number then find the
 905                  * first block in this inode and use that as a hint.  If that
 906                  * block is also bogus then just don't worry about it.
 907                  */
 908                 if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
 909                         free_extent_map(em);
 910                         em = search_extent_mapping(em_tree, 0, 0);
 911                         if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
 912                                 alloc_hint = em->block_start;
 913                         if (em)
 914                                 free_extent_map(em);
 915                 } else {
 916                         alloc_hint = em->block_start;
 917                         free_extent_map(em);
 918                 }
 919         }
 920         read_unlock(&em_tree->lock);
 921
 922         return alloc_hint;
 923 }
 924
 925 /*
 926  * when extent_io.c finds a delayed allocation range in the file,
 927  * the call backs end up in this code.  The basic idea is to
 928  * allocate extents on disk for the range, and create ordered data structs
 929  * in ram to track those extents.
 930  *
 931  * locked_page is the page that writepage had locked already.  We use
 932  * it to make sure we don't do extra locks or unlocks.
 933  *
 934  * *page_started is set to one if we unlock locked_page and do everything
 935  * required to start IO on it.  It may be clean and already done with
 936  * IO when we return.
 937  */
 938 static noinline int cow_file_range(struct inode *inode,
 939                                    struct page *locked_page,
 940                                    u64 start, u64 end, u64 delalloc_end,
 941                                    int *page_started, unsigned long *nr_written,
 942                                    int unlock, struct btrfs_dedupe_hash *hash)
 943 {
 944         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
 945         struct btrfs_root *root = BTRFS_I(inode)->root;
 946         u64 alloc_hint = 0;
 947         u64 num_bytes;
 948         unsigned long ram_size;
 949         u64 cur_alloc_size = 0;
 950         u64 blocksize = fs_info->sectorsize;
 951         struct btrfs_key ins;
 952         struct extent_map *em;
 953         unsigned clear_bits;
 954         unsigned long page_ops;
 955         bool extent_reserved = false;
 956         int ret = 0;
 957
 958         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
 959                 WARN_ON_ONCE(1);
 960                 ret = -EINVAL;
 961                 goto out_unlock;
 962         }
 963
 964         num_bytes = ALIGN(end - start + 1, blocksize);
 965         num_bytes = max(blocksize,  num_bytes);
 966         ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
 967
 968         inode_should_defrag(BTRFS_I(inode), start, end, num_bytes, SZ_64K);
 969
 970         if (start == 0) {
 971                 /* lets try to make an inline extent */
 972                 ret = cow_file_range_inline(inode, start, end, 0,
 973                                             BTRFS_COMPRESS_NONE, NULL);
 974                 if (ret == 0) {
 975                         /*
 976                          * We use DO_ACCOUNTING here because we need the
 977                          * delalloc_release_metadata to be run _after_ we drop
 978                          * our outstanding extent for clearing delalloc for this
 979                          * range.
 980                          */
 981                         extent_clear_unlock_delalloc(inode, start, end,
 982                                      delalloc_end, NULL,
 983                                      EXTENT_LOCKED | EXTENT_DELALLOC |
 984                                      EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
 985                                      EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
 986                                      PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
 987                                      PAGE_END_WRITEBACK);
 988                         *nr_written = *nr_written +
 989                              (end - start + PAGE_SIZE) / PAGE_SIZE;
 990                         *page_started = 1;
 991                         goto out;
 992                 } else if (ret < 0) {
 993                         goto out_unlock;
 994                 }
 995         }
 996
 997         alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
 998         btrfs_drop_extent_cache(BTRFS_I(inode), start,
 999                         start + num_bytes - 1, 0);
1000
1001         while (num_bytes > 0) {
1002                 cur_alloc_size = num_bytes;
1003                 ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
1004                                            fs_info->sectorsize, 0, alloc_hint,
1005                                            &ins, 1, 1);
1006                 if (ret < 0)
1007                         goto out_unlock;
1008                 cur_alloc_size = ins.offset;
1009                 extent_reserved = true;
1010
1011                 ram_size = ins.offset;
1012                 em = create_io_em(inode, start, ins.offset, /* len */
1013                                   start, /* orig_start */
1014                                   ins.objectid, /* block_start */
1015                                   ins.offset, /* block_len */
1016                                   ins.offset, /* orig_block_len */
1017                                   ram_size, /* ram_bytes */
1018                                   BTRFS_COMPRESS_NONE, /* compress_type */
1019                                   BTRFS_ORDERED_REGULAR /* type */);
1020                 if (IS_ERR(em))
1021                         goto out_reserve;
1022                 free_extent_map(em);
1023
1024                 ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
1025                                                ram_size, cur_alloc_size, 0);
1026                 if (ret)
1027                         goto out_drop_extent_cache;
1028
1029                 if (root->root_key.objectid ==
1030                     BTRFS_DATA_RELOC_TREE_OBJECTID) {
1031                         ret = btrfs_reloc_clone_csums(inode, start,
1032                                                       cur_alloc_size);
1033                         /*
1034                          * Only drop cache here, and process as normal.
1035                          *
1036                          * We must not allow extent_clear_unlock_delalloc()
1037                          * at out_unlock label to free meta of this ordered
1038                          * extent, as its meta should be freed by
1039                          * btrfs_finish_ordered_io().
1040                          *
1041                          * So we must continue until @start is increased to
1042                          * skip current ordered extent.
1043                          */
1044                         if (ret)
1045                                 btrfs_drop_extent_cache(BTRFS_I(inode), start,
1046                                                 start + ram_size - 1, 0);
1047                 }
1048
1049                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1050
1051                 /* we're not doing compressed IO, don't unlock the first
1052                  * page (which the caller expects to stay locked), don't
1053                  * clear any dirty bits and don't set any writeback bits
1054                  *
1055                  * Do set the Private2 bit so we know this page was properly
1056                  * setup for writepage
1057                  */
1058                 page_ops = unlock ? PAGE_UNLOCK : 0;
1059                 page_ops |= PAGE_SET_PRIVATE2;
1060
1061                 extent_clear_unlock_delalloc(inode, start,
1062                                              start + ram_size - 1,
1063                                              delalloc_end, locked_page,
1064                                              EXTENT_LOCKED | EXTENT_DELALLOC,
1065                                              page_ops);
1066                 if (num_bytes < cur_alloc_size)
1067                         num_bytes = 0;
1068                 else
1069                         num_bytes -= cur_alloc_size;
1070                 alloc_hint = ins.objectid + ins.offset;
1071                 start += cur_alloc_size;
1072                 extent_reserved = false;
1073
1074                 /*
1075                  * btrfs_reloc_clone_csums() error, since start is increased
1076                  * extent_clear_unlock_delalloc() at out_unlock label won't
1077                  * free metadata of current ordered extent, we're OK to exit.
1078                  */
1079                 if (ret)
1080                         goto out_unlock;
1081         }
1082 out:
1083         return ret;
1084
1085 out_drop_extent_cache:
1086         btrfs_drop_extent_cache(BTRFS_I(inode), start, start + ram_size - 1, 0);
1087 out_reserve:
1088         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
1089         btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
1090 out_unlock:
1091         clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
1092                 EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
1093         page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
1094                 PAGE_END_WRITEBACK;
1095         /*
1096          * If we reserved an extent for our delalloc range (or a subrange) and
1097          * failed to create the respective ordered extent, then it means that
1098          * when we reserved the extent we decremented the extent's size from
1099          * the data space_info's bytes_may_use counter and incremented the
1100          * space_info's bytes_reserved counter by the same amount. We must make
1101          * sure extent_clear_unlock_delalloc() does not try to decrement again
1102          * the data space_info's bytes_may_use counter, therefore we do not pass
1103          * it the flag EXTENT_CLEAR_DATA_RESV.
1104          */
1105         if (extent_reserved) {
1106                 extent_clear_unlock_delalloc(inode, start,
1107                                              start + cur_alloc_size,
1108                                              start + cur_alloc_size,
1109                                              locked_page,
1110                                              clear_bits,
1111                                              page_ops);
1112                 start += cur_alloc_size;
1113                 if (start >= end)
1114                         goto out;
1115         }
1116         extent_clear_unlock_delalloc(inode, start, end, delalloc_end,
1117                                      locked_page,
1118                                      clear_bits | EXTENT_CLEAR_DATA_RESV,
1119                                      page_ops);
1120         goto out;
1121 }
1122
1123 /*
1124  * work queue call back to started compression on a file and pages
1125  */
1126 static noinline void async_cow_start(struct btrfs_work *work)
1127 {
1128         struct async_cow *async_cow;
1129         int num_added = 0;
1130         async_cow = container_of(work, struct async_cow, work);
1131
1132         compress_file_range(async_cow->inode, async_cow->locked_page,
1133                             async_cow->start, async_cow->end, async_cow,
1134                             &num_added);
1135         if (num_added == 0) {
1136                 btrfs_add_delayed_iput(async_cow->inode);
1137                 async_cow->inode = NULL;
1138         }
1139 }
1140
1141 /*
1142  * work queue call back to submit previously compressed pages
1143  */
1144 static noinline void async_cow_submit(struct btrfs_work *work)
1145 {
1146         struct btrfs_fs_info *fs_info;
1147         struct async_cow *async_cow;
1148         struct btrfs_root *root;
1149         unsigned long nr_pages;
1150
1151         async_cow = container_of(work, struct async_cow, work);
1152
1153         root = async_cow->root;
1154         fs_info = root->fs_info;
1155         nr_pages = (async_cow->end - async_cow->start + PAGE_SIZE) >>
1156                 PAGE_SHIFT;
1157
1158         /*
1159          * atomic_sub_return implies a barrier for waitqueue_active
1160          */
1161         if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
1162             5 * SZ_1M &&
1163             waitqueue_active(&fs_info->async_submit_wait))
1164                 wake_up(&fs_info->async_submit_wait);
1165
1166         if (async_cow->inode)
1167                 submit_compressed_extents(async_cow->inode, async_cow);
1168 }
1169
1170 static noinline void async_cow_free(struct btrfs_work *work)
1171 {
1172         struct async_cow *async_cow;
1173         async_cow = container_of(work, struct async_cow, work);
1174         if (async_cow->inode)
1175                 btrfs_add_delayed_iput(async_cow->inode);
1176         kfree(async_cow);
1177 }
1178
1179 static int cow_file_range_async(struct inode *inode, struct page *locked_page,
1180                                 u64 start, u64 end, int *page_started,
1181                                 unsigned long *nr_written,
1182                                 unsigned int write_flags)
1183 {
1184         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1185         struct async_cow *async_cow;
1186         struct btrfs_root *root = BTRFS_I(inode)->root;
1187         unsigned long nr_pages;
1188         u64 cur_end;
1189
1190         clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, EXTENT_LOCKED,
1191                          1, 0, NULL);
1192         while (start < end) {
1193                 async_cow = kmalloc(sizeof(*async_cow), GFP_NOFS);
1194                 BUG_ON(!async_cow); /* -ENOMEM */
1195                 async_cow->inode = igrab(inode);
1196                 async_cow->root = root;
1197                 async_cow->locked_page = locked_page;
1198                 async_cow->start = start;
1199                 async_cow->write_flags = write_flags;
1200
1201                 if (BTRFS_I(inode)->flags & BTRFS_INODE_NOCOMPRESS &&
1202                     !btrfs_test_opt(fs_info, FORCE_COMPRESS))
1203                         cur_end = end;
1204                 else
1205                         cur_end = min(end, start + SZ_512K - 1);
1206
1207                 async_cow->end = cur_end;
1208                 INIT_LIST_HEAD(&async_cow->extents);
1209
1210                 btrfs_init_work(&async_cow->work,
1211                                 btrfs_delalloc_helper,
1212                                 async_cow_start, async_cow_submit,
1213                                 async_cow_free);
1214
1215                 nr_pages = (cur_end - start + PAGE_SIZE) >>
1216                         PAGE_SHIFT;
1217                 atomic_add(nr_pages, &fs_info->async_delalloc_pages);
1218
1219                 btrfs_queue_work(fs_info->delalloc_workers, &async_cow->work);
1220
1221                 *nr_written += nr_pages;
1222                 start = cur_end + 1;
1223         }
1224         *page_started = 1;
1225         return 0;
1226 }
1227
1228 static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
1229                                         u64 bytenr, u64 num_bytes)
1230 {
1231         int ret;
1232         struct btrfs_ordered_sum *sums;
1233         LIST_HEAD(list);
1234
1235         ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
1236                                        bytenr + num_bytes - 1, &list, 0);
1237         if (ret == 0 && list_empty(&list))
1238                 return 0;
1239
1240         while (!list_empty(&list)) {
1241                 sums = list_entry(list.next, struct btrfs_ordered_sum, list);
1242                 list_del(&sums->list);
1243                 kfree(sums);
1244         }
1245         if (ret < 0)
1246                 return ret;
1247         return 1;
1248 }
1249
1250 /*
1251  * when nowcow writeback call back.  This checks for snapshots or COW copies
1252  * of the extents that exist in the file, and COWs the file as required.
1253  *
1254  * If no cow copies or snapshots exist, we write directly to the existing
1255  * blocks on disk
1256  */
1257 static noinline int run_delalloc_nocow(struct inode *inode,
1258                                        struct page *locked_page,
1259                               u64 start, u64 end, int *page_started, int force,
1260                               unsigned long *nr_written)
1261 {
1262         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1263         struct btrfs_root *root = BTRFS_I(inode)->root;
1264         struct extent_buffer *leaf;
1265         struct btrfs_path *path;
1266         struct btrfs_file_extent_item *fi;
1267         struct btrfs_key found_key;
1268         struct extent_map *em;
1269         u64 cow_start;
1270         u64 cur_offset;
1271         u64 extent_end;
1272         u64 extent_offset;
1273         u64 disk_bytenr;
1274         u64 num_bytes;
1275         u64 disk_num_bytes;
1276         u64 ram_bytes;
1277         int extent_type;
1278         int ret, err;
1279         int type;
1280         int nocow;
1281         int check_prev = 1;
1282         bool nolock;
1283         u64 ino = btrfs_ino(BTRFS_I(inode));
1284
1285         path = btrfs_alloc_path();
1286         if (!path) {
1287                 extent_clear_unlock_delalloc(inode, start, end, end,
1288                                              locked_page,
1289                                              EXTENT_LOCKED | EXTENT_DELALLOC |
1290                                              EXTENT_DO_ACCOUNTING |
1291                                              EXTENT_DEFRAG, PAGE_UNLOCK |
1292                                              PAGE_CLEAR_DIRTY |
1293                                              PAGE_SET_WRITEBACK |
1294                                              PAGE_END_WRITEBACK);
1295                 return -ENOMEM;
1296         }
1297
1298         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
1299
1300         cow_start = (u64)-1;
1301         cur_offset = start;
1302         while (1) {
1303                 ret = btrfs_lookup_file_extent(NULL, root, path, ino,
1304                                                cur_offset, 0);
1305                 if (ret < 0)
1306                         goto error;
1307                 if (ret > 0 && path->slots[0] > 0 && check_prev) {
1308                         leaf = path->nodes[0];
1309                         btrfs_item_key_to_cpu(leaf, &found_key,
1310                                               path->slots[0] - 1);
1311                         if (found_key.objectid == ino &&
1312                             found_key.type == BTRFS_EXTENT_DATA_KEY)
1313                                 path->slots[0]--;
1314                 }
1315                 check_prev = 0;
1316 next_slot:
1317                 leaf = path->nodes[0];
1318                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
1319                         ret = btrfs_next_leaf(root, path);
1320                         if (ret < 0) {
1321                                 if (cow_start != (u64)-1)
1322                                         cur_offset = cow_start;
1323                                 goto error;
1324                         }
1325                         if (ret > 0)
1326                                 break;
1327                         leaf = path->nodes[0];
1328                 }
1329
1330                 nocow = 0;
1331                 disk_bytenr = 0;
1332                 num_bytes = 0;
1333                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1334
1335                 if (found_key.objectid > ino)
1336                         break;
1337                 if (WARN_ON_ONCE(found_key.objectid < ino) ||
1338                     found_key.type < BTRFS_EXTENT_DATA_KEY) {
1339                         path->slots[0]++;
1340                         goto next_slot;
1341                 }
1342                 if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
1343                     found_key.offset > end)
1344                         break;
1345
1346                 if (found_key.offset > cur_offset) {
1347                         extent_end = found_key.offset;
1348                         extent_type = 0;
1349                         goto out_check;
1350                 }
1351
1352                 fi = btrfs_item_ptr(leaf, path->slots[0],
1353                                     struct btrfs_file_extent_item);
1354                 extent_type = btrfs_file_extent_type(leaf, fi);
1355
1356                 ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
1357                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
1358                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1359                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1360                         extent_offset = btrfs_file_extent_offset(leaf, fi);
1361                         extent_end = found_key.offset +
1362                                 btrfs_file_extent_num_bytes(leaf, fi);
1363                         disk_num_bytes =
1364                                 btrfs_file_extent_disk_num_bytes(leaf, fi);
1365                         if (extent_end <= start) {
1366                                 path->slots[0]++;
1367                                 goto next_slot;
1368                         }
1369                         if (disk_bytenr == 0)
1370                                 goto out_check;
1371                         if (btrfs_file_extent_compression(leaf, fi) ||
1372                             btrfs_file_extent_encryption(leaf, fi) ||
1373                             btrfs_file_extent_other_encoding(leaf, fi))
1374                                 goto out_check;
1375                         if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
1376                                 goto out_check;
1377                         if (btrfs_extent_readonly(fs_info, disk_bytenr))
1378                                 goto out_check;
1379                         ret = btrfs_cross_ref_exist(root, ino,
1380                                                     found_key.offset -
1381                                                     extent_offset, disk_bytenr);
1382                         if (ret) {
1383                                 /*
1384                                  * ret could be -EIO if the above fails to read
1385                                  * metadata.
1386                                  */
1387                                 if (ret < 0) {
1388                                         if (cow_start != (u64)-1)
1389                                                 cur_offset = cow_start;
1390                                         goto error;
1391                                 }
1392
1393                                 WARN_ON_ONCE(nolock);
1394                                 goto out_check;
1395                         }
1396                         disk_bytenr += extent_offset;
1397                         disk_bytenr += cur_offset - found_key.offset;
1398                         num_bytes = min(end + 1, extent_end) - cur_offset;
1399                         /*
1400                          * if there are pending snapshots for this root,
1401                          * we fall into common COW way.
1402                          */
1403                         if (!nolock) {
1404                                 err = btrfs_start_write_no_snapshotting(root);
1405                                 if (!err)
1406                                         goto out_check;
1407                         }
1408                         /*
1409                          * force cow if csum exists in the range.
1410                          * this ensure that csum for a given extent are
1411                          * either valid or do not exist.
1412                          */
1413                         ret = csum_exist_in_range(fs_info, disk_bytenr,
1414                                                   num_bytes);
1415                         if (ret) {
1416                                 if (!nolock)
1417                                         btrfs_end_write_no_snapshotting(root);
1418
1419                                 /*
1420                                  * ret could be -EIO if the above fails to read
1421                                  * metadata.
1422                                  */
1423                                 if (ret < 0) {
1424                                         if (cow_start != (u64)-1)
1425                                                 cur_offset = cow_start;
1426                                         goto error;
1427                                 }
1428                                 WARN_ON_ONCE(nolock);
1429                                 goto out_check;
1430                         }
1431                         if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr)) {
1432                                 if (!nolock)
1433                                         btrfs_end_write_no_snapshotting(root);
1434                                 goto out_check;
1435                         }
1436                         nocow = 1;
1437                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
1438                         extent_end = found_key.offset +
1439                                 btrfs_file_extent_inline_len(leaf,
1440                                                      path->slots[0], fi);
1441                         extent_end = ALIGN(extent_end,
1442                                            fs_info->sectorsize);
1443                 } else {
1444                         BUG_ON(1);
1445                 }
1446 out_check:
1447                 if (extent_end <= start) {
1448                         path->slots[0]++;
1449                         if (!nolock && nocow)
1450                                 btrfs_end_write_no_snapshotting(root);
1451                         if (nocow)
1452                                 btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1453                         goto next_slot;
1454                 }
1455                 if (!nocow) {
1456                         if (cow_start == (u64)-1)
1457                                 cow_start = cur_offset;
1458                         cur_offset = extent_end;
1459                         if (cur_offset > end)
1460                                 break;
1461                         path->slots[0]++;
1462                         goto next_slot;
1463                 }
1464
1465                 btrfs_release_path(path);
1466                 if (cow_start != (u64)-1) {
1467                         ret = cow_file_range(inode, locked_page,
1468                                              cow_start, found_key.offset - 1,
1469                                              end, page_started, nr_written, 1,
1470                                              NULL);
1471                         if (ret) {
1472                                 if (!nolock && nocow)
1473                                         btrfs_end_write_no_snapshotting(root);
1474                                 if (nocow)
1475                                         btrfs_dec_nocow_writers(fs_info,
1476                                                                 disk_bytenr);
1477                                 goto error;
1478                         }
1479                         cow_start = (u64)-1;
1480                 }
1481
1482                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1483                         u64 orig_start = found_key.offset - extent_offset;
1484
1485                         em = create_io_em(inode, cur_offset, num_bytes,
1486                                           orig_start,
1487                                           disk_bytenr, /* block_start */
1488                                           num_bytes, /* block_len */
1489                                           disk_num_bytes, /* orig_block_len */
1490                                           ram_bytes, BTRFS_COMPRESS_NONE,
1491                                           BTRFS_ORDERED_PREALLOC);
1492                         if (IS_ERR(em)) {
1493                                 if (!nolock && nocow)
1494                                         btrfs_end_write_no_snapshotting(root);
1495                                 if (nocow)
1496                                         btrfs_dec_nocow_writers(fs_info,
1497                                                                 disk_bytenr);
1498                                 ret = PTR_ERR(em);
1499                                 goto error;
1500                         }
1501                         free_extent_map(em);
1502                 }
1503
1504                 if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
1505                         type = BTRFS_ORDERED_PREALLOC;
1506                 } else {
1507                         type = BTRFS_ORDERED_NOCOW;
1508                 }
1509
1510                 ret = btrfs_add_ordered_extent(inode, cur_offset, disk_bytenr,
1511                                                num_bytes, num_bytes, type);
1512                 if (nocow)
1513                         btrfs_dec_nocow_writers(fs_info, disk_bytenr);
1514                 BUG_ON(ret); /* -ENOMEM */
1515
1516                 if (root->root_key.objectid ==
1517                     BTRFS_DATA_RELOC_TREE_OBJECTID)
1518                         /*
1519                          * Error handled later, as we must prevent
1520                          * extent_clear_unlock_delalloc() in error handler
1521                          * from freeing metadata of created ordered extent.
1522                          */
1523                         ret = btrfs_reloc_clone_csums(inode, cur_offset,
1524                                                       num_bytes);
1525
1526                 extent_clear_unlock_delalloc(inode, cur_offset,
1527                                              cur_offset + num_bytes - 1, end,
1528                                              locked_page, EXTENT_LOCKED |
1529                                              EXTENT_DELALLOC |
1530                                              EXTENT_CLEAR_DATA_RESV,
1531                                              PAGE_UNLOCK | PAGE_SET_PRIVATE2);
1532
1533                 if (!nolock && nocow)
1534                         btrfs_end_write_no_snapshotting(root);
1535                 cur_offset = extent_end;
1536
1537                 /*
1538                  * btrfs_reloc_clone_csums() error, now we're OK to call error
1539                  * handler, as metadata for created ordered extent will only
1540                  * be freed by btrfs_finish_ordered_io().
1541                  */
1542                 if (ret)
1543                         goto error;
1544                 if (cur_offset > end)
1545                         break;
1546         }
1547         btrfs_release_path(path);
1548
1549         if (cur_offset <= end && cow_start == (u64)-1) {
1550                 cow_start = cur_offset;
1551                 cur_offset = end;
1552         }
1553
1554         if (cow_start != (u64)-1) {
1555                 ret = cow_file_range(inode, locked_page, cow_start, end, end,
1556                                      page_started, nr_written, 1, NULL);
1557                 if (ret)
1558                         goto error;
1559         }
1560
1561 error:
1562         if (ret && cur_offset < end)
1563                 extent_clear_unlock_delalloc(inode, cur_offset, end, end,
1564                                              locked_page, EXTENT_LOCKED |
1565                                              EXTENT_DELALLOC | EXTENT_DEFRAG |
1566                                              EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
1567                                              PAGE_CLEAR_DIRTY |
1568                                              PAGE_SET_WRITEBACK |
1569                                              PAGE_END_WRITEBACK);
1570         btrfs_free_path(path);
1571         return ret;
1572 }
1573
1574 static inline int need_force_cow(struct inode *inode, u64 start, u64 end)
1575 {
1576
1577         if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
1578             !(BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC))
1579                 return 0;
1580
1581         /*
1582          * @defrag_bytes is a hint value, no spinlock held here,
1583          * if is not zero, it means the file is defragging.
1584          * Force cow if given extent needs to be defragged.
1585          */
1586         if (BTRFS_I(inode)->defrag_bytes &&
1587             test_range_bit(&BTRFS_I(inode)->io_tree, start, end,
1588                            EXTENT_DEFRAG, 0, NULL))
1589                 return 1;
1590
1591         return 0;
1592 }
1593
1594 /*
1595  * extent_io.c call back to do delayed allocation processing
1596  */
1597 static int run_delalloc_range(void *private_data, struct page *locked_page,
1598                               u64 start, u64 end, int *page_started,
1599                               unsigned long *nr_written,
1600                               struct writeback_control *wbc)
1601 {
1602         struct inode *inode = private_data;
1603         int ret;
1604         int force_cow = need_force_cow(inode, start, end);
1605         unsigned int write_flags = wbc_to_write_flags(wbc);
1606
1607         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW && !force_cow) {
1608                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1609                                          page_started, 1, nr_written);
1610         } else if (BTRFS_I(inode)->flags & BTRFS_INODE_PREALLOC && !force_cow) {
1611                 ret = run_delalloc_nocow(inode, locked_page, start, end,
1612                                          page_started, 0, nr_written);
1613         } else if (!inode_need_compress(inode, start, end)) {
1614                 ret = cow_file_range(inode, locked_page, start, end, end,
1615                                       page_started, nr_written, 1, NULL);
1616         } else {
1617                 set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
1618                         &BTRFS_I(inode)->runtime_flags);
1619                 ret = cow_file_range_async(inode, locked_page, start, end,
1620                                            page_started, nr_written,
1621                                            write_flags);
1622         }
1623         if (ret)
1624                 btrfs_cleanup_ordered_extents(inode, start, end - start + 1);
1625         return ret;
1626 }
1627
1628 static void btrfs_split_extent_hook(void *private_data,
1629                                     struct extent_state *orig, u64 split)
1630 {
1631         struct inode *inode = private_data;
1632         u64 size;
1633
1634         /* not delalloc, ignore it */
1635         if (!(orig->state & EXTENT_DELALLOC))
1636                 return;
1637
1638         size = orig->end - orig->start + 1;
1639         if (size > BTRFS_MAX_EXTENT_SIZE) {
1640                 u32 num_extents;
1641                 u64 new_size;
1642
1643                 /*
1644                  * See the explanation in btrfs_merge_extent_hook, the same
1645                  * applies here, just in reverse.
1646                  */
1647                 new_size = orig->end - split + 1;
1648                 num_extents = count_max_extents(new_size);
1649                 new_size = split - orig->start;
1650                 num_extents += count_max_extents(new_size);
1651                 if (count_max_extents(size) >= num_extents)
1652                         return;
1653         }
1654
1655         spin_lock(&BTRFS_I(inode)->lock);
1656         btrfs_mod_outstanding_extents(BTRFS_I(inode), 1);
1657         spin_unlock(&BTRFS_I(inode)->lock);
1658 }
1659
1660 /*
1661  * extent_io.c merge_extent_hook, used to track merged delayed allocation
1662  * extents so we can keep track of new extents that are just merged onto old
1663  * extents, such as when we are doing sequential writes, so we can properly
1664  * account for the metadata space we'll need.
1665  */
1666 static void btrfs_merge_extent_hook(void *private_data,
1667                                     struct extent_state *new,
1668                                     struct extent_state *other)
1669 {
1670         struct inode *inode = private_data;
1671         u64 new_size, old_size;
1672         u32 num_extents;
1673
1674         /* not delalloc, ignore it */
1675         if (!(other->state & EXTENT_DELALLOC))
1676                 return;
1677
1678         if (new->start > other->start)
1679                 new_size = new->end - other->start + 1;
1680         else
1681                 new_size = other->end - new->start + 1;
1682
1683         /* we're not bigger than the max, unreserve the space and go */
1684         if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
1685                 spin_lock(&BTRFS_I(inode)->lock);
1686                 btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1687                 spin_unlock(&BTRFS_I(inode)->lock);
1688                 return;
1689         }
1690
1691         /*
1692          * We have to add up either side to figure out how many extents were
1693          * accounted for before we merged into one big extent.  If the number of
1694          * extents we accounted for is <= the amount we need for the new range
1695          * then we can return, otherwise drop.  Think of it like this
1696          *
1697          * [ 4k][MAX_SIZE]
1698          *
1699          * So we've grown the extent by a MAX_SIZE extent, this would mean we
1700          * need 2 outstanding extents, on one side we have 1 and the other side
1701          * we have 1 so they are == and we can return.  But in this case
1702          *
1703          * [MAX_SIZE+4k][MAX_SIZE+4k]
1704          *
1705          * Each range on their own accounts for 2 extents, but merged together
1706          * they are only 3 extents worth of accounting, so we need to drop in
1707          * this case.
1708          */
1709         old_size = other->end - other->start + 1;
1710         num_extents = count_max_extents(old_size);
1711         old_size = new->end - new->start + 1;
1712         num_extents += count_max_extents(old_size);
1713         if (count_max_extents(new_size) >= num_extents)
1714                 return;
1715
1716         spin_lock(&BTRFS_I(inode)->lock);
1717         btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
1718         spin_unlock(&BTRFS_I(inode)->lock);
1719 }
1720
1721 static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
1722                                       struct inode *inode)
1723 {
1724         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1725
1726         spin_lock(&root->delalloc_lock);
1727         if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
1728                 list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
1729                               &root->delalloc_inodes);
1730                 set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1731                         &BTRFS_I(inode)->runtime_flags);
1732                 root->nr_delalloc_inodes++;
1733                 if (root->nr_delalloc_inodes == 1) {
1734                         spin_lock(&fs_info->delalloc_root_lock);
1735                         BUG_ON(!list_empty(&root->delalloc_root));
1736                         list_add_tail(&root->delalloc_root,
1737                                       &fs_info->delalloc_roots);
1738                         spin_unlock(&fs_info->delalloc_root_lock);
1739                 }
1740         }
1741         spin_unlock(&root->delalloc_lock);
1742 }
1743
1744 static void btrfs_del_delalloc_inode(struct btrfs_root *root,
1745                                      struct btrfs_inode *inode)
1746 {
1747         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1748
1749         spin_lock(&root->delalloc_lock);
1750         if (!list_empty(&inode->delalloc_inodes)) {
1751                 list_del_init(&inode->delalloc_inodes);
1752                 clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1753                           &inode->runtime_flags);
1754                 root->nr_delalloc_inodes--;
1755                 if (!root->nr_delalloc_inodes) {
1756                         spin_lock(&fs_info->delalloc_root_lock);
1757                         BUG_ON(list_empty(&root->delalloc_root));
1758                         list_del_init(&root->delalloc_root);
1759                         spin_unlock(&fs_info->delalloc_root_lock);
1760                 }
1761         }
1762         spin_unlock(&root->delalloc_lock);
1763 }
1764
1765 /*
1766  * extent_io.c set_bit_hook, used to track delayed allocation
1767  * bytes in this file, and to maintain the list of inodes that
1768  * have pending delalloc work to be done.
1769  */
1770 static void btrfs_set_bit_hook(void *private_data,
1771                                struct extent_state *state, unsigned *bits)
1772 {
1773         struct inode *inode = private_data;
1774
1775         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1776
1777         if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
1778                 WARN_ON(1);
1779         /*
1780          * set_bit and clear bit hooks normally require _irqsave/restore
1781          * but in this case, we are only testing for the DELALLOC
1782          * bit, which is only set or cleared with irqs on
1783          */
1784         if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1785                 struct btrfs_root *root = BTRFS_I(inode)->root;
1786                 u64 len = state->end + 1 - state->start;
1787                 u32 num_extents = count_max_extents(len);
1788                 bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
1789
1790                 spin_lock(&BTRFS_I(inode)->lock);
1791                 btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents);
1792                 spin_unlock(&BTRFS_I(inode)->lock);
1793
1794                 /* For sanity tests */
1795                 if (btrfs_is_testing(fs_info))
1796                         return;
1797
1798                 percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
1799                                          fs_info->delalloc_batch);
1800                 spin_lock(&BTRFS_I(inode)->lock);
1801                 BTRFS_I(inode)->delalloc_bytes += len;
1802                 if (*bits & EXTENT_DEFRAG)
1803                         BTRFS_I(inode)->defrag_bytes += len;
1804                 if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1805                                          &BTRFS_I(inode)->runtime_flags))
1806                         btrfs_add_delalloc_inodes(root, inode);
1807                 spin_unlock(&BTRFS_I(inode)->lock);
1808         }
1809
1810         if (!(state->state & EXTENT_DELALLOC_NEW) &&
1811             (*bits & EXTENT_DELALLOC_NEW)) {
1812                 spin_lock(&BTRFS_I(inode)->lock);
1813                 BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
1814                         state->start;
1815                 spin_unlock(&BTRFS_I(inode)->lock);
1816         }
1817 }
1818
1819 /*
1820  * extent_io.c clear_bit_hook, see set_bit_hook for why
1821  */
1822 static void btrfs_clear_bit_hook(void *private_data,
1823                                  struct extent_state *state,
1824                                  unsigned *bits)
1825 {
1826         struct btrfs_inode *inode = BTRFS_I((struct inode *)private_data);
1827         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
1828         u64 len = state->end + 1 - state->start;
1829         u32 num_extents = count_max_extents(len);
1830
1831         if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {
1832                 spin_lock(&inode->lock);
1833                 inode->defrag_bytes -= len;
1834                 spin_unlock(&inode->lock);
1835         }
1836
1837         /*
1838          * set_bit and clear bit hooks normally require _irqsave/restore
1839          * but in this case, we are only testing for the DELALLOC
1840          * bit, which is only set or cleared with irqs on
1841          */
1842         if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
1843                 struct btrfs_root *root = inode->root;
1844                 bool do_list = !btrfs_is_free_space_inode(inode);
1845
1846                 spin_lock(&inode->lock);
1847                 btrfs_mod_outstanding_extents(inode, -num_extents);
1848                 spin_unlock(&inode->lock);
1849
1850                 /*
1851                  * We don't reserve metadata space for space cache inodes so we
1852                  * don't need to call dellalloc_release_metadata if there is an
1853                  * error.
1854                  */
1855                 if (*bits & EXTENT_CLEAR_META_RESV &&
1856                     root != fs_info->tree_root)
1857                         btrfs_delalloc_release_metadata(inode, len, false);
1858
1859                 /* For sanity tests. */
1860                 if (btrfs_is_testing(fs_info))
1861                         return;
1862
1863                 if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
1864                     do_list && !(state->state & EXTENT_NORESERVE) &&
1865                     (*bits & EXTENT_CLEAR_DATA_RESV))
1866                         btrfs_free_reserved_data_space_noquota(
1867                                         &inode->vfs_inode,
1868                                         state->start, len);
1869
1870                 percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
1871                                          fs_info->delalloc_batch);
1872                 spin_lock(&inode->lock);
1873                 inode->delalloc_bytes -= len;
1874                 if (do_list && inode->delalloc_bytes == 0 &&
1875                     test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
1876                                         &inode->runtime_flags))
1877                         btrfs_del_delalloc_inode(root, inode);
1878                 spin_unlock(&inode->lock);
1879         }
1880
1881         if ((state->state & EXTENT_DELALLOC_NEW) &&
1882             (*bits & EXTENT_DELALLOC_NEW)) {
1883                 spin_lock(&inode->lock);
1884                 ASSERT(inode->new_delalloc_bytes >= len);
1885                 inode->new_delalloc_bytes -= len;
1886                 spin_unlock(&inode->lock);
1887         }
1888 }
1889
1890 /*
1891  * extent_io.c merge_bio_hook, this must check the chunk tree to make sure
1892  * we don't create bios that span stripes or chunks
1893  *
1894  * return 1 if page cannot be merged to bio
1895  * return 0 if page can be merged to bio
1896  * return error otherwise
1897  */
1898 int btrfs_merge_bio_hook(struct page *page, unsigned long offset,
1899                          size_t size, struct bio *bio,
1900                          unsigned long bio_flags)
1901 {
1902         struct inode *inode = page->mapping->host;
1903         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1904         u64 logical = (u64)bio->bi_iter.bi_sector << 9;
1905         u64 length = 0;
1906         u64 map_length;
1907         int ret;
1908
1909         if (bio_flags & EXTENT_BIO_COMPRESSED)
1910                 return 0;
1911
1912         length = bio->bi_iter.bi_size;
1913         map_length = length;
1914         ret = btrfs_map_block(fs_info, btrfs_op(bio), logical, &map_length,
1915                               NULL, 0);
1916         if (ret < 0)
1917                 return ret;
1918         if (map_length < length + size)
1919                 return 1;
1920         return 0;
1921 }
1922
1923 /*
1924  * in order to insert checksums into the metadata in large chunks,
1925  * we wait until bio submission time.   All the pages in the bio are
1926  * checksummed and sums are attached onto the ordered extent record.
1927  *
1928  * At IO completion time the cums attached on the ordered extent record
1929  * are inserted into the btree
1930  */
1931 static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio,
1932                                     u64 bio_offset)
1933 {
1934         struct inode *inode = private_data;
1935         blk_status_t ret = 0;
1936
1937         ret = btrfs_csum_one_bio(inode, bio, 0, 0);
1938         BUG_ON(ret); /* -ENOMEM */
1939         return 0;
1940 }
1941
1942 /*
1943  * in order to insert checksums into the metadata in large chunks,
1944  * we wait until bio submission time.   All the pages in the bio are
1945  * checksummed and sums are attached onto the ordered extent record.
1946  *
1947  * At IO completion time the cums attached on the ordered extent record
1948  * are inserted into the btree
1949  */
1950 static blk_status_t btrfs_submit_bio_done(void *private_data, struct bio *bio,
1951                           int mirror_num)
1952 {
1953         struct inode *inode = private_data;
1954         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1955         blk_status_t ret;
1956
1957         ret = btrfs_map_bio(fs_info, bio, mirror_num, 1);
1958         if (ret) {
1959                 bio->bi_status = ret;
1960                 bio_endio(bio);
1961         }
1962         return ret;
1963 }
1964
1965 /*
1966  * extent_io.c submission hook. This does the right thing for csum calculation
1967  * on write, or reading the csums from the tree before a read.
1968  *
1969  * Rules about async/sync submit,
1970  * a) read:                             sync submit
1971  *
1972  * b) write without checksum:           sync submit
1973  *
1974  * c) write with checksum:
1975  *    c-1) if bio is issued by fsync:   sync submit
1976  *         (sync_writers != 0)
1977  *
1978  *    c-2) if root is reloc root:       sync submit
1979  *         (only in case of buffered IO)
1980  *
1981  *    c-3) otherwise:                   async submit
1982  */
1983 static blk_status_t btrfs_submit_bio_hook(void *private_data, struct bio *bio,
1984                                  int mirror_num, unsigned long bio_flags,
1985                                  u64 bio_offset)
1986 {
1987         struct inode *inode = private_data;
1988         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1989         struct btrfs_root *root = BTRFS_I(inode)->root;
1990         enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
1991         blk_status_t ret = 0;
1992         int skip_sum;
1993         int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
1994
1995         skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
1996
1997         if (btrfs_is_free_space_inode(BTRFS_I(inode)))
1998                 metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
1999
2000         if (bio_op(bio) != REQ_OP_WRITE) {
2001                 ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
2002                 if (ret)
2003                         goto out;
2004
2005                 if (bio_flags & EXTENT_BIO_COMPRESSED) {
2006                         ret = btrfs_submit_compressed_read(inode, bio,
2007                                                            mirror_num,
2008                                                            bio_flags);
2009                         goto out;
2010                 } else if (!skip_sum) {
2011                         ret = btrfs_lookup_bio_sums(inode, bio, NULL);
2012                         if (ret)
2013                                 goto out;
2014                 }
2015                 goto mapit;
2016         } else if (async && !skip_sum) {
2017                 /* csum items have already been cloned */
2018                 if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
2019                         goto mapit;
2020                 /* we're doing a write, do the async checksumming */
2021                 ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
2022                                           bio_offset, inode,
2023                                           btrfs_submit_bio_start,
2024                                           btrfs_submit_bio_done);
2025                 goto out;
2026         } else if (!skip_sum) {
2027                 ret = btrfs_csum_one_bio(inode, bio, 0, 0);
2028                 if (ret)
2029                         goto out;
2030         }
2031
2032 mapit:
2033         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
2034
2035 out:
2036         if (ret) {
2037                 bio->bi_status = ret;
2038                 bio_endio(bio);
2039         }
2040         return ret;
2041 }
2042
2043 /*
2044  * given a list of ordered sums record them in the inode.  This happens
2045  * at IO completion time based on sums calculated at bio submission time.
2046  */
2047 static noinline int add_pending_csums(struct btrfs_trans_handle *trans,
2048                              struct inode *inode, struct list_head *list)
2049 {
2050         struct btrfs_ordered_sum *sum;
2051         int ret;
2052
2053         list_for_each_entry(sum, list, list) {
2054                 trans->adding_csums = true;
2055                 ret = btrfs_csum_file_blocks(trans,
2056                        BTRFS_I(inode)->root->fs_info->csum_root, sum);
2057                 trans->adding_csums = false;
2058                 if (ret)
2059                         return ret;
2060         }
2061         return 0;
2062 }
2063
2064 int btrfs_set_extent_delalloc(struct inode *inode, u64 start, u64 end,
2065                               unsigned int extra_bits,
2066                               struct extent_state **cached_state, int dedupe)
2067 {
2068         WARN_ON((end & (PAGE_SIZE - 1)) == 0);
2069         return set_extent_delalloc(&BTRFS_I(inode)->io_tree, start, end,
2070                                    extra_bits, cached_state);
2071 }
2072
2073 /* see btrfs_writepage_start_hook for details on why this is required */
2074 struct btrfs_writepage_fixup {
2075         struct page *page;
2076         struct btrfs_work work;
2077 };
2078
2079 static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
2080 {
2081         struct btrfs_writepage_fixup *fixup;
2082         struct btrfs_ordered_extent *ordered;
2083         struct extent_state *cached_state = NULL;
2084         struct extent_changeset *data_reserved = NULL;
2085         struct page *page;
2086         struct inode *inode;
2087         u64 page_start;
2088         u64 page_end;
2089         int ret;
2090
2091         fixup = container_of(work, struct btrfs_writepage_fixup, work);
2092         page = fixup->page;
2093 again:
2094         lock_page(page);
2095         if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
2096                 ClearPageChecked(page);
2097                 goto out_page;
2098         }
2099
2100         inode = page->mapping->host;
2101         page_start = page_offset(page);
2102         page_end = page_offset(page) + PAGE_SIZE - 1;
2103
2104         lock_extent_bits(&BTRFS_I(inode)->io_tree, page_start, page_end,
2105                          &cached_state);
2106
2107         /* already ordered? We're done */
2108         if (PagePrivate2(page))
2109                 goto out;
2110
2111         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
2112                                         PAGE_SIZE);
2113         if (ordered) {
2114                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start,
2115                                      page_end, &cached_state);
2116                 unlock_page(page);
2117                 btrfs_start_ordered_extent(inode, ordered, 1);
2118                 btrfs_put_ordered_extent(ordered);
2119                 goto again;
2120         }
2121
2122         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
2123                                            PAGE_SIZE);
2124         if (ret) {
2125                 mapping_set_error(page->mapping, ret);
2126                 end_extent_writepage(page, ret, page_start, page_end);
2127                 ClearPageChecked(page);
2128                 goto out;
2129          }
2130
2131         ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,
2132                                         &cached_state, 0);
2133         if (ret) {
2134                 mapping_set_error(page->mapping, ret);
2135                 end_extent_writepage(page, ret, page_start, page_end);
2136                 ClearPageChecked(page);
2137                 goto out;
2138         }
2139
2140         ClearPageChecked(page);
2141         set_page_dirty(page);
2142         btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, false);
2143 out:
2144         unlock_extent_cached(&BTRFS_I(inode)->io_tree, page_start, page_end,
2145                              &cached_state);
2146 out_page:
2147         unlock_page(page);
2148         put_page(page);
2149         kfree(fixup);
2150         extent_changeset_free(data_reserved);
2151 }
2152
2153 /*
2154  * There are a few paths in the higher layers of the kernel that directly
2155  * set the page dirty bit without asking the filesystem if it is a
2156  * good idea.  This causes problems because we want to make sure COW
2157  * properly happens and the data=ordered rules are followed.
2158  *
2159  * In our case any range that doesn't have the ORDERED bit set
2160  * hasn't been properly setup for IO.  We kick off an async process
2161  * to fix it up.  The async helper will wait for ordered extents, set
2162  * the delalloc bit and make it safe to write the page.
2163  */
2164 static int btrfs_writepage_start_hook(struct page *page, u64 start, u64 end)
2165 {
2166         struct inode *inode = page->mapping->host;
2167         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2168         struct btrfs_writepage_fixup *fixup;
2169
2170         /* this page is properly in the ordered list */
2171         if (TestClearPagePrivate2(page))
2172                 return 0;
2173
2174         if (PageChecked(page))
2175                 return -EAGAIN;
2176
2177         fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
2178         if (!fixup)
2179                 return -EAGAIN;
2180
2181         SetPageChecked(page);
2182         get_page(page);
2183         btrfs_init_work(&fixup->work, btrfs_fixup_helper,
2184                         btrfs_writepage_fixup_worker, NULL, NULL);
2185         fixup->page = page;
2186         btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
2187         return -EBUSY;
2188 }
2189
2190 static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
2191                                        struct inode *inode, u64 file_pos,
2192                                        u64 disk_bytenr, u64 disk_num_bytes,
2193                                        u64 num_bytes, u64 ram_bytes,
2194                                        u8 compression, u8 encryption,
2195                                        u16 other_encoding, int extent_type)
2196 {
2197         struct btrfs_root *root = BTRFS_I(inode)->root;
2198         struct btrfs_file_extent_item *fi;
2199         struct btrfs_path *path;
2200         struct extent_buffer *leaf;
2201         struct btrfs_key ins;
2202         u64 qg_released;
2203         int extent_inserted = 0;
2204         int ret;
2205
2206         path = btrfs_alloc_path();
2207         if (!path)
2208                 return -ENOMEM;
2209
2210         /*
2211          * we may be replacing one extent in the tree with another.
2212          * The new extent is pinned in the extent map, and we don't want
2213          * to drop it from the cache until it is completely in the btree.
2214          *
2215          * So, tell btrfs_drop_extents to leave this extent in the cache.
2216          * the caller is expected to unpin it and allow it to be merged
2217          * with the others.
2218          */
2219         ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
2220                                    file_pos + num_bytes, NULL, 0,
2221                                    1, sizeof(*fi), &extent_inserted);
2222         if (ret)
2223                 goto out;
2224
2225         if (!extent_inserted) {
2226                 ins.objectid = btrfs_ino(BTRFS_I(inode));
2227                 ins.offset = file_pos;
2228                 ins.type = BTRFS_EXTENT_DATA_KEY;
2229
2230                 path->leave_spinning = 1;
2231                 ret = btrfs_insert_empty_item(trans, root, path, &ins,
2232                                               sizeof(*fi));
2233                 if (ret)
2234                         goto out;
2235         }
2236         leaf = path->nodes[0];
2237         fi = btrfs_item_ptr(leaf, path->slots[0],
2238                             struct btrfs_file_extent_item);
2239         btrfs_set_file_extent_generation(leaf, fi, trans->transid);
2240         btrfs_set_file_extent_type(leaf, fi, extent_type);
2241         btrfs_set_file_extent_disk_bytenr(leaf, fi, disk_bytenr);
2242         btrfs_set_file_extent_disk_num_bytes(leaf, fi, disk_num_bytes);
2243         btrfs_set_file_extent_offset(leaf, fi, 0);
2244         btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2245         btrfs_set_file_extent_ram_bytes(leaf, fi, ram_bytes);
2246         btrfs_set_file_extent_compression(leaf, fi, compression);
2247         btrfs_set_file_extent_encryption(leaf, fi, encryption);
2248         btrfs_set_file_extent_other_encoding(leaf, fi, other_encoding);
2249
2250         btrfs_mark_buffer_dirty(leaf);
2251         btrfs_release_path(path);
2252
2253         inode_add_bytes(inode, num_bytes);
2254
2255         ins.objectid = disk_bytenr;
2256         ins.offset = disk_num_bytes;
2257         ins.type = BTRFS_EXTENT_ITEM_KEY;
2258
2259         /*
2260          * Release the reserved range from inode dirty range map, as it is
2261          * already moved into delayed_ref_head
2262          */
2263         ret = btrfs_qgroup_release_data(inode, file_pos, ram_bytes);
2264         if (ret < 0)
2265                 goto out;
2266         qg_released = ret;
2267         ret = btrfs_alloc_reserved_file_extent(trans, root,
2268                                                btrfs_ino(BTRFS_I(inode)),
2269                                                file_pos, qg_released, &ins);
2270 out:
2271         btrfs_free_path(path);
2272
2273         return ret;
2274 }
2275
2276 /* snapshot-aware defrag */
2277 struct sa_defrag_extent_backref {
2278         struct rb_node node;
2279         struct old_sa_defrag_extent *old;
2280         u64 root_id;
2281         u64 inum;
2282         u64 file_pos;
2283         u64 extent_offset;
2284         u64 num_bytes;
2285         u64 generation;
2286 };
2287
2288 struct old_sa_defrag_extent {
2289         struct list_head list;
2290         struct new_sa_defrag_extent *new;
2291
2292         u64 extent_offset;
2293         u64 bytenr;
2294         u64 offset;
2295         u64 len;
2296         int count;
2297 };
2298
2299 struct new_sa_defrag_extent {
2300         struct rb_root root;
2301         struct list_head head;
2302         struct btrfs_path *path;
2303         struct inode *inode;
2304         u64 file_pos;
2305         u64 len;
2306         u64 bytenr;
2307         u64 disk_len;
2308         u8 compress_type;
2309 };
2310
2311 static int backref_comp(struct sa_defrag_extent_backref *b1,
2312                         struct sa_defrag_extent_backref *b2)
2313 {
2314         if (b1->root_id < b2->root_id)
2315                 return -1;
2316         else if (b1->root_id > b2->root_id)
2317                 return 1;
2318
2319         if (b1->inum < b2->inum)
2320                 return -1;
2321         else if (b1->inum > b2->inum)
2322                 return 1;
2323
2324         if (b1->file_pos < b2->file_pos)
2325                 return -1;
2326         else if (b1->file_pos > b2->file_pos)
2327                 return 1;
2328
2329         /*
2330          * [------------------------------] ===> (a range of space)
2331          *     |<--->|   |<---->| =============> (fs/file tree A)
2332          * |<---------------------------->| ===> (fs/file tree B)
2333          *
2334          * A range of space can refer to two file extents in one tree while
2335          * refer to only one file extent in another tree.
2336          *
2337          * So we may process a disk offset more than one time(two extents in A)
2338          * and locate at the same extent(one extent in B), then insert two same
2339          * backrefs(both refer to the extent in B).
2340          */
2341         return 0;
2342 }
2343
2344 static void backref_insert(struct rb_root *root,
2345                            struct sa_defrag_extent_backref *backref)
2346 {
2347         struct rb_node **p = &root->rb_node;
2348         struct rb_node *parent = NULL;
2349         struct sa_defrag_extent_backref *entry;
2350         int ret;
2351
2352         while (*p) {
2353                 parent = *p;
2354                 entry = rb_entry(parent, struct sa_defrag_extent_backref, node);
2355
2356                 ret = backref_comp(backref, entry);
2357                 if (ret < 0)
2358                         p = &(*p)->rb_left;
2359                 else
2360                         p = &(*p)->rb_right;
2361         }
2362
2363         rb_link_node(&backref->node, parent, p);
2364         rb_insert_color(&backref->node, root);
2365 }
2366
2367 /*
2368  * Note the backref might has changed, and in this case we just return 0.
2369  */
2370 static noinline int record_one_backref(u64 inum, u64 offset, u64 root_id,
2371                                        void *ctx)
2372 {
2373         struct btrfs_file_extent_item *extent;
2374         struct old_sa_defrag_extent *old = ctx;
2375         struct new_sa_defrag_extent *new = old->new;
2376         struct btrfs_path *path = new->path;
2377         struct btrfs_key key;
2378         struct btrfs_root *root;
2379         struct sa_defrag_extent_backref *backref;
2380         struct extent_buffer *leaf;
2381         struct inode *inode = new->inode;
2382         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2383         int slot;
2384         int ret;
2385         u64 extent_offset;
2386         u64 num_bytes;
2387
2388         if (BTRFS_I(inode)->root->root_key.objectid == root_id &&
2389             inum == btrfs_ino(BTRFS_I(inode)))
2390                 return 0;
2391
2392         key.objectid = root_id;
2393         key.type = BTRFS_ROOT_ITEM_KEY;
2394         key.offset = (u64)-1;
2395
2396         root = btrfs_read_fs_root_no_name(fs_info, &key);
2397         if (IS_ERR(root)) {
2398                 if (PTR_ERR(root) == -ENOENT)
2399                         return 0;
2400                 WARN_ON(1);
2401                 btrfs_debug(fs_info, "inum=%llu, offset=%llu, root_id=%llu",
2402                          inum, offset, root_id);
2403                 return PTR_ERR(root);
2404         }
2405
2406         key.objectid = inum;
2407         key.type = BTRFS_EXTENT_DATA_KEY;
2408         if (offset > (u64)-1 << 32)
2409                 key.offset = 0;
2410         else
2411                 key.offset = offset;
2412
2413         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2414         if (WARN_ON(ret < 0))
2415                 return ret;
2416         ret = 0;
2417
2418         while (1) {
2419                 cond_resched();
2420
2421                 leaf = path->nodes[0];
2422                 slot = path->slots[0];
2423
2424                 if (slot >= btrfs_header_nritems(leaf)) {
2425                         ret = btrfs_next_leaf(root, path);
2426                         if (ret < 0) {
2427                                 goto out;
2428                         } else if (ret > 0) {
2429                                 ret = 0;
2430                                 goto out;
2431                         }
2432                         continue;
2433                 }
2434
2435                 path->slots[0]++;
2436
2437                 btrfs_item_key_to_cpu(leaf, &key, slot);
2438
2439                 if (key.objectid > inum)
2440                         goto out;
2441
2442                 if (key.objectid < inum || key.type != BTRFS_EXTENT_DATA_KEY)
2443                         continue;
2444
2445                 extent = btrfs_item_ptr(leaf, slot,
2446                                         struct btrfs_file_extent_item);
2447
2448                 if (btrfs_file_extent_disk_bytenr(leaf, extent) != old->bytenr)
2449                         continue;
2450
2451                 /*
2452                  * 'offset' refers to the exact key.offset,
2453                  * NOT the 'offset' field in btrfs_extent_data_ref, ie.
2454                  * (key.offset - extent_offset).
2455                  */
2456                 if (key.offset != offset)
2457                         continue;
2458
2459                 extent_offset = btrfs_file_extent_offset(leaf, extent);
2460                 num_bytes = btrfs_file_extent_num_bytes(leaf, extent);
2461
2462                 if (extent_offset >= old->extent_offset + old->offset +
2463                     old->len || extent_offset + num_bytes <=
2464                     old->extent_offset + old->offset)
2465                         continue;
2466                 break;
2467         }
2468
2469         backref = kmalloc(sizeof(*backref), GFP_NOFS);
2470         if (!backref) {
2471                 ret = -ENOENT;
2472                 goto out;
2473         }
2474
2475         backref->root_id = root_id;
2476         backref->inum = inum;
2477         backref->file_pos = offset;
2478         backref->num_bytes = num_bytes;
2479         backref->extent_offset = extent_offset;
2480         backref->generation = btrfs_file_extent_generation(leaf, extent);
2481         backref->old = old;
2482         backref_insert(&new->root, backref);
2483         old->count++;
2484 out:
2485         btrfs_release_path(path);
2486         WARN_ON(ret);
2487         return ret;
2488 }
2489
2490 static noinline bool record_extent_backrefs(struct btrfs_path *path,
2491                                    struct new_sa_defrag_extent *new)
2492 {
2493         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2494         struct old_sa_defrag_extent *old, *tmp;
2495         int ret;
2496
2497         new->path = path;
2498
2499         list_for_each_entry_safe(old, tmp, &new->head, list) {
2500                 ret = iterate_inodes_from_logical(old->bytenr +
2501                                                   old->extent_offset, fs_info,
2502                                                   path, record_one_backref,
2503                                                   old, false);
2504                 if (ret < 0 && ret != -ENOENT)
2505                         return false;
2506
2507                 /* no backref to be processed for this extent */
2508                 if (!old->count) {
2509                         list_del(&old->list);
2510                         kfree(old);
2511                 }
2512         }
2513
2514         if (list_empty(&new->head))
2515                 return false;
2516
2517         return true;
2518 }
2519
2520 static int relink_is_mergable(struct extent_buffer *leaf,
2521                               struct btrfs_file_extent_item *fi,
2522                               struct new_sa_defrag_extent *new)
2523 {
2524         if (btrfs_file_extent_disk_bytenr(leaf, fi) != new->bytenr)
2525                 return 0;
2526
2527         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2528                 return 0;
2529
2530         if (btrfs_file_extent_compression(leaf, fi) != new->compress_type)
2531                 return 0;
2532
2533         if (btrfs_file_extent_encryption(leaf, fi) ||
2534             btrfs_file_extent_other_encoding(leaf, fi))
2535                 return 0;
2536
2537         return 1;
2538 }
2539
2540 /*
2541  * Note the backref might has changed, and in this case we just return 0.
2542  */
2543 static noinline int relink_extent_backref(struct btrfs_path *path,
2544                                  struct sa_defrag_extent_backref *prev,
2545                                  struct sa_defrag_extent_backref *backref)
2546 {
2547         struct btrfs_file_extent_item *extent;
2548         struct btrfs_file_extent_item *item;
2549         struct btrfs_ordered_extent *ordered;
2550         struct btrfs_trans_handle *trans;
2551         struct btrfs_root *root;
2552         struct btrfs_key key;
2553         struct extent_buffer *leaf;
2554         struct old_sa_defrag_extent *old = backref->old;
2555         struct new_sa_defrag_extent *new = old->new;
2556         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2557         struct inode *inode;
2558         struct extent_state *cached = NULL;
2559         int ret = 0;
2560         u64 start;
2561         u64 len;
2562         u64 lock_start;
2563         u64 lock_end;
2564         bool merge = false;
2565         int index;
2566
2567         if (prev && prev->root_id == backref->root_id &&
2568             prev->inum == backref->inum &&
2569             prev->file_pos + prev->num_bytes == backref->file_pos)
2570                 merge = true;
2571
2572         /* step 1: get root */
2573         key.objectid = backref->root_id;
2574         key.type = BTRFS_ROOT_ITEM_KEY;
2575         key.offset = (u64)-1;
2576
2577         index = srcu_read_lock(&fs_info->subvol_srcu);
2578
2579         root = btrfs_read_fs_root_no_name(fs_info, &key);
2580         if (IS_ERR(root)) {
2581                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2582                 if (PTR_ERR(root) == -ENOENT)
2583                         return 0;
2584                 return PTR_ERR(root);
2585         }
2586
2587         if (btrfs_root_readonly(root)) {
2588                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2589                 return 0;
2590         }
2591
2592         /* step 2: get inode */
2593         key.objectid = backref->inum;
2594         key.type = BTRFS_INODE_ITEM_KEY;
2595         key.offset = 0;
2596
2597         inode = btrfs_iget(fs_info->sb, &key, root, NULL);
2598         if (IS_ERR(inode)) {
2599                 srcu_read_unlock(&fs_info->subvol_srcu, index);
2600                 return 0;
2601         }
2602
2603         srcu_read_unlock(&fs_info->subvol_srcu, index);
2604
2605         /* step 3: relink backref */
2606         lock_start = backref->file_pos;
2607         lock_end = backref->file_pos + backref->num_bytes - 1;
2608         lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2609                          &cached);
2610
2611         ordered = btrfs_lookup_first_ordered_extent(inode, lock_end);
2612         if (ordered) {
2613                 btrfs_put_ordered_extent(ordered);
2614                 goto out_unlock;
2615         }
2616
2617         trans = btrfs_join_transaction(root);
2618         if (IS_ERR(trans)) {
2619                 ret = PTR_ERR(trans);
2620                 goto out_unlock;
2621         }
2622
2623         key.objectid = backref->inum;
2624         key.type = BTRFS_EXTENT_DATA_KEY;
2625         key.offset = backref->file_pos;
2626
2627         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2628         if (ret < 0) {
2629                 goto out_free_path;
2630         } else if (ret > 0) {
2631                 ret = 0;
2632                 goto out_free_path;
2633         }
2634
2635         extent = btrfs_item_ptr(path->nodes[0], path->slots[0],
2636                                 struct btrfs_file_extent_item);
2637
2638         if (btrfs_file_extent_generation(path->nodes[0], extent) !=
2639             backref->generation)
2640                 goto out_free_path;
2641
2642         btrfs_release_path(path);
2643
2644         start = backref->file_pos;
2645         if (backref->extent_offset < old->extent_offset + old->offset)
2646                 start += old->extent_offset + old->offset -
2647                          backref->extent_offset;
2648
2649         len = min(backref->extent_offset + backref->num_bytes,
2650                   old->extent_offset + old->offset + old->len);
2651         len -= max(backref->extent_offset, old->extent_offset + old->offset);
2652
2653         ret = btrfs_drop_extents(trans, root, inode, start,
2654                                  start + len, 1);
2655         if (ret)
2656                 goto out_free_path;
2657 again:
2658         key.objectid = btrfs_ino(BTRFS_I(inode));
2659         key.type = BTRFS_EXTENT_DATA_KEY;
2660         key.offset = start;
2661
2662         path->leave_spinning = 1;
2663         if (merge) {
2664                 struct btrfs_file_extent_item *fi;
2665                 u64 extent_len;
2666                 struct btrfs_key found_key;
2667
2668                 ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2669                 if (ret < 0)
2670                         goto out_free_path;
2671
2672                 path->slots[0]--;
2673                 leaf = path->nodes[0];
2674                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
2675
2676                 fi = btrfs_item_ptr(leaf, path->slots[0],
2677                                     struct btrfs_file_extent_item);
2678                 extent_len = btrfs_file_extent_num_bytes(leaf, fi);
2679
2680                 if (extent_len + found_key.offset == start &&
2681                     relink_is_mergable(leaf, fi, new)) {
2682                         btrfs_set_file_extent_num_bytes(leaf, fi,
2683                                                         extent_len + len);
2684                         btrfs_mark_buffer_dirty(leaf);
2685                         inode_add_bytes(inode, len);
2686
2687                         ret = 1;
2688                         goto out_free_path;
2689                 } else {
2690                         merge = false;
2691                         btrfs_release_path(path);
2692                         goto again;
2693                 }
2694         }
2695
2696         ret = btrfs_insert_empty_item(trans, root, path, &key,
2697                                         sizeof(*extent));
2698         if (ret) {
2699                 btrfs_abort_transaction(trans, ret);
2700                 goto out_free_path;
2701         }
2702
2703         leaf = path->nodes[0];
2704         item = btrfs_item_ptr(leaf, path->slots[0],
2705                                 struct btrfs_file_extent_item);
2706         btrfs_set_file_extent_disk_bytenr(leaf, item, new->bytenr);
2707         btrfs_set_file_extent_disk_num_bytes(leaf, item, new->disk_len);
2708         btrfs_set_file_extent_offset(leaf, item, start - new->file_pos);
2709         btrfs_set_file_extent_num_bytes(leaf, item, len);
2710         btrfs_set_file_extent_ram_bytes(leaf, item, new->len);
2711         btrfs_set_file_extent_generation(leaf, item, trans->transid);
2712         btrfs_set_file_extent_type(leaf, item, BTRFS_FILE_EXTENT_REG);
2713         btrfs_set_file_extent_compression(leaf, item, new->compress_type);
2714         btrfs_set_file_extent_encryption(leaf, item, 0);
2715         btrfs_set_file_extent_other_encoding(leaf, item, 0);
2716
2717         btrfs_mark_buffer_dirty(leaf);
2718         inode_add_bytes(inode, len);
2719         btrfs_release_path(path);
2720
2721         ret = btrfs_inc_extent_ref(trans, root, new->bytenr,
2722                         new->disk_len, 0,
2723                         backref->root_id, backref->inum,
2724                         new->file_pos); /* start - extent_offset */
2725         if (ret) {
2726                 btrfs_abort_transaction(trans, ret);
2727                 goto out_free_path;
2728         }
2729
2730         ret = 1;
2731 out_free_path:
2732         btrfs_release_path(path);
2733         path->leave_spinning = 0;
2734         btrfs_end_transaction(trans);
2735 out_unlock:
2736         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start, lock_end,
2737                              &cached);
2738         iput(inode);
2739         return ret;
2740 }
2741
2742 static void free_sa_defrag_extent(struct new_sa_defrag_extent *new)
2743 {
2744         struct old_sa_defrag_extent *old, *tmp;
2745
2746         if (!new)
2747                 return;
2748
2749         list_for_each_entry_safe(old, tmp, &new->head, list) {
2750                 kfree(old);
2751         }
2752         kfree(new);
2753 }
2754
2755 static void relink_file_extents(struct new_sa_defrag_extent *new)
2756 {
2757         struct btrfs_fs_info *fs_info = btrfs_sb(new->inode->i_sb);
2758         struct btrfs_path *path;
2759         struct sa_defrag_extent_backref *backref;
2760         struct sa_defrag_extent_backref *prev = NULL;
2761         struct inode *inode;
2762         struct rb_node *node;
2763         int ret;
2764
2765         inode = new->inode;
2766
2767         path = btrfs_alloc_path();
2768         if (!path)
2769                 return;
2770
2771         if (!record_extent_backrefs(path, new)) {
2772                 btrfs_free_path(path);
2773                 goto out;
2774         }
2775         btrfs_release_path(path);
2776
2777         while (1) {
2778                 node = rb_first(&new->root);
2779                 if (!node)
2780                         break;
2781                 rb_erase(node, &new->root);
2782
2783                 backref = rb_entry(node, struct sa_defrag_extent_backref, node);
2784
2785                 ret = relink_extent_backref(path, prev, backref);
2786                 WARN_ON(ret < 0);
2787
2788                 kfree(prev);
2789
2790                 if (ret == 1)
2791                         prev = backref;
2792                 else
2793                         prev = NULL;
2794                 cond_resched();
2795         }
2796         kfree(prev);
2797
2798         btrfs_free_path(path);
2799 out:
2800         free_sa_defrag_extent(new);
2801
2802         atomic_dec(&fs_info->defrag_running);
2803         wake_up(&fs_info->transaction_wait);
2804 }
2805
2806 static struct new_sa_defrag_extent *
2807 record_old_file_extents(struct inode *inode,
2808                         struct btrfs_ordered_extent *ordered)
2809 {
2810         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2811         struct btrfs_root *root = BTRFS_I(inode)->root;
2812         struct btrfs_path *path;
2813         struct btrfs_key key;
2814         struct old_sa_defrag_extent *old;
2815         struct new_sa_defrag_extent *new;
2816         int ret;
2817
2818         new = kmalloc(sizeof(*new), GFP_NOFS);
2819         if (!new)
2820                 return NULL;
2821
2822         new->inode = inode;
2823         new->file_pos = ordered->file_offset;
2824         new->len = ordered->len;
2825         new->bytenr = ordered->start;
2826         new->disk_len = ordered->disk_len;
2827         new->compress_type = ordered->compress_type;
2828         new->root = RB_ROOT;
2829         INIT_LIST_HEAD(&new->head);
2830
2831         path = btrfs_alloc_path();
2832         if (!path)
2833                 goto out_kfree;
2834
2835         key.objectid = btrfs_ino(BTRFS_I(inode));
2836         key.type = BTRFS_EXTENT_DATA_KEY;
2837         key.offset = new->file_pos;
2838
2839         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
2840         if (ret < 0)
2841                 goto out_free_path;
2842         if (ret > 0 && path->slots[0] > 0)
2843                 path->slots[0]--;
2844
2845         /* find out all the old extents for the file range */
2846         while (1) {
2847                 struct btrfs_file_extent_item *extent;
2848                 struct extent_buffer *l;
2849                 int slot;
2850                 u64 num_bytes;
2851                 u64 offset;
2852                 u64 end;
2853                 u64 disk_bytenr;
2854                 u64 extent_offset;
2855
2856                 l = path->nodes[0];
2857                 slot = path->slots[0];
2858
2859                 if (slot >= btrfs_header_nritems(l)) {
2860                         ret = btrfs_next_leaf(root, path);
2861                         if (ret < 0)
2862                                 goto out_free_path;
2863                         else if (ret > 0)
2864                                 break;
2865                         continue;
2866                 }
2867
2868                 btrfs_item_key_to_cpu(l, &key, slot);
2869
2870                 if (key.objectid != btrfs_ino(BTRFS_I(inode)))
2871                         break;
2872                 if (key.type != BTRFS_EXTENT_DATA_KEY)
2873                         break;
2874                 if (key.offset >= new->file_pos + new->len)
2875                         break;
2876
2877                 extent = btrfs_item_ptr(l, slot, struct btrfs_file_extent_item);
2878
2879                 num_bytes = btrfs_file_extent_num_bytes(l, extent);
2880                 if (key.offset + num_bytes < new->file_pos)
2881                         goto next;
2882
2883                 disk_bytenr = btrfs_file_extent_disk_bytenr(l, extent);
2884                 if (!disk_bytenr)
2885                         goto next;
2886
2887                 extent_offset = btrfs_file_extent_offset(l, extent);
2888
2889                 old = kmalloc(sizeof(*old), GFP_NOFS);
2890                 if (!old)
2891                         goto out_free_path;
2892
2893                 offset = max(new->file_pos, key.offset);
2894                 end = min(new->file_pos + new->len, key.offset + num_bytes);
2895
2896                 old->bytenr = disk_bytenr;
2897                 old->extent_offset = extent_offset;
2898                 old->offset = offset - key.offset;
2899                 old->len = end - offset;
2900                 old->new = new;
2901                 old->count = 0;
2902                 list_add_tail(&old->list, &new->head);
2903 next:
2904                 path->slots[0]++;
2905                 cond_resched();
2906         }
2907
2908         btrfs_free_path(path);
2909         atomic_inc(&fs_info->defrag_running);
2910
2911         return new;
2912
2913 out_free_path:
2914         btrfs_free_path(path);
2915 out_kfree:
2916         free_sa_defrag_extent(new);
2917         return NULL;
2918 }
2919
2920 static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
2921                                          u64 start, u64 len)
2922 {
2923         struct btrfs_block_group_cache *cache;
2924
2925         cache = btrfs_lookup_block_group(fs_info, start);
2926         ASSERT(cache);
2927
2928         spin_lock(&cache->lock);
2929         cache->delalloc_bytes -= len;
2930         spin_unlock(&cache->lock);
2931
2932         btrfs_put_block_group(cache);
2933 }
2934
2935 /* as ordered data IO finishes, this gets called so we can finish
2936  * an ordered extent if the range of bytes in the file it covers are
2937  * fully written.
2938  */
2939 static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
2940 {
2941         struct inode *inode = ordered_extent->inode;
2942         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2943         struct btrfs_root *root = BTRFS_I(inode)->root;
2944         struct btrfs_trans_handle *trans = NULL;
2945         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
2946         struct extent_state *cached_state = NULL;
2947         struct new_sa_defrag_extent *new = NULL;
2948         int compress_type = 0;
2949         int ret = 0;
2950         u64 logical_len = ordered_extent->len;
2951         bool nolock;
2952         bool truncated = false;
2953         bool range_locked = false;
2954         bool clear_new_delalloc_bytes = false;
2955
2956         if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
2957             !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
2958             !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
2959                 clear_new_delalloc_bytes = true;
2960
2961         nolock = btrfs_is_free_space_inode(BTRFS_I(inode));
2962
2963         if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
2964                 ret = -EIO;
2965                 goto out;
2966         }
2967
2968         btrfs_free_io_failure_record(BTRFS_I(inode),
2969                         ordered_extent->file_offset,
2970                         ordered_extent->file_offset +
2971                         ordered_extent->len - 1);
2972
2973         if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
2974                 truncated = true;
2975                 logical_len = ordered_extent->truncated_len;
2976                 /* Truncated the entire extent, don't bother adding */
2977                 if (!logical_len)
2978                         goto out;
2979         }
2980
2981         if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
2982                 BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
2983
2984                 /*
2985                  * For mwrite(mmap + memset to write) case, we still reserve
2986                  * space for NOCOW range.
2987                  * As NOCOW won't cause a new delayed ref, just free the space
2988                  */
2989                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
2990                                        ordered_extent->len);
2991                 btrfs_ordered_update_i_size(inode, 0, ordered_extent);
2992                 if (nolock)
2993                         trans = btrfs_join_transaction_nolock(root);
2994                 else
2995                         trans = btrfs_join_transaction(root);
2996                 if (IS_ERR(trans)) {
2997                         ret = PTR_ERR(trans);
2998                         trans = NULL;
2999                         goto out;
3000                 }
3001                 trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3002                 ret = btrfs_update_inode_fallback(trans, root, inode);
3003                 if (ret) /* -ENOMEM or corruption */
3004                         btrfs_abort_transaction(trans, ret);
3005                 goto out;
3006         }
3007
3008         range_locked = true;
3009         lock_extent_bits(io_tree, ordered_extent->file_offset,
3010                          ordered_extent->file_offset + ordered_extent->len - 1,
3011                          &cached_state);
3012
3013         ret = test_range_bit(io_tree, ordered_extent->file_offset,
3014                         ordered_extent->file_offset + ordered_extent->len - 1,
3015                         EXTENT_DEFRAG, 0, cached_state);
3016         if (ret) {
3017                 u64 last_snapshot = btrfs_root_last_snapshot(&root->root_item);
3018                 if (0 && last_snapshot >= BTRFS_I(inode)->generation)
3019                         /* the inode is shared */
3020                         new = record_old_file_extents(inode, ordered_extent);
3021
3022                 clear_extent_bit(io_tree, ordered_extent->file_offset,
3023                         ordered_extent->file_offset + ordered_extent->len - 1,
3024                         EXTENT_DEFRAG, 0, 0, &cached_state);
3025         }
3026
3027         if (nolock)
3028                 trans = btrfs_join_transaction_nolock(root);
3029         else
3030                 trans = btrfs_join_transaction(root);
3031         if (IS_ERR(trans)) {
3032                 ret = PTR_ERR(trans);
3033                 trans = NULL;
3034                 goto out;
3035         }
3036
3037         trans->block_rsv = &BTRFS_I(inode)->block_rsv;
3038
3039         if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
3040                 compress_type = ordered_extent->compress_type;
3041         if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
3042                 BUG_ON(compress_type);
3043                 btrfs_qgroup_free_data(inode, NULL, ordered_extent->file_offset,
3044                                        ordered_extent->len);
3045                 ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
3046                                                 ordered_extent->file_offset,
3047                                                 ordered_extent->file_offset +
3048                                                 logical_len);
3049         } else {
3050                 BUG_ON(root == fs_info->tree_root);
3051                 ret = insert_reserved_file_extent(trans, inode,
3052                                                 ordered_extent->file_offset,
3053                                                 ordered_extent->start,
3054                                                 ordered_extent->disk_len,
3055                                                 logical_len, logical_len,
3056                                                 compress_type, 0, 0,
3057                                                 BTRFS_FILE_EXTENT_REG);
3058                 if (!ret)
3059                         btrfs_release_delalloc_bytes(fs_info,
3060                                                      ordered_extent->start,
3061                                                      ordered_extent->disk_len);
3062         }
3063         unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
3064                            ordered_extent->file_offset, ordered_extent->len,
3065                            trans->transid);
3066         if (ret < 0) {
3067                 btrfs_abort_transaction(trans, ret);
3068                 goto out;
3069         }
3070
3071         ret = add_pending_csums(trans, inode, &ordered_extent->list);
3072         if (ret) {
3073                 btrfs_abort_transaction(trans, ret);
3074                 goto out;
3075         }
3076
3077         btrfs_ordered_update_i_size(inode, 0, ordered_extent);
3078         ret = btrfs_update_inode_fallback(trans, root, inode);
3079         if (ret) { /* -ENOMEM or corruption */
3080                 btrfs_abort_transaction(trans, ret);
3081                 goto out;
3082         }
3083         ret = 0;
3084 out:
3085         if (range_locked || clear_new_delalloc_bytes) {
3086                 unsigned int clear_bits = 0;
3087
3088                 if (range_locked)
3089                         clear_bits |= EXTENT_LOCKED;
3090                 if (clear_new_delalloc_bytes)
3091                         clear_bits |= EXTENT_DELALLOC_NEW;
3092                 clear_extent_bit(&BTRFS_I(inode)->io_tree,
3093                                  ordered_extent->file_offset,
3094                                  ordered_extent->file_offset +
3095                                  ordered_extent->len - 1,
3096                                  clear_bits,
3097                                  (clear_bits & EXTENT_LOCKED) ? 1 : 0,
3098                                  0, &cached_state);
3099         }
3100
3101         if (trans)
3102                 btrfs_end_transaction(trans);
3103
3104         if (ret || truncated) {
3105                 u64 start, end;
3106
3107                 if (truncated)
3108                         start = ordered_extent->file_offset + logical_len;
3109                 else
3110                         start = ordered_extent->file_offset;
3111                 end = ordered_extent->file_offset + ordered_extent->len - 1;
3112                 clear_extent_uptodate(io_tree, start, end, NULL);
3113
3114                 /* Drop the cache for the part of the extent we didn't write. */
3115                 btrfs_drop_extent_cache(BTRFS_I(inode), start, end, 0);
3116
3117                 /*
3118                  * If the ordered extent had an IOERR or something else went
3119                  * wrong we need to return the space for this ordered extent
3120                  * back to the allocator.  We only free the extent in the
3121                  * truncated case if we didn't write out the extent at all.
3122                  */
3123                 if ((ret || !logical_len) &&
3124                     !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
3125                     !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags))
3126                         btrfs_free_reserved_extent(fs_info,
3127                                                    ordered_extent->start,
3128                                                    ordered_extent->disk_len, 1);
3129         }
3130
3131
3132         /*
3133          * This needs to be done to make sure anybody waiting knows we are done
3134          * updating everything for this ordered extent.
3135          */
3136         btrfs_remove_ordered_extent(inode, ordered_extent);
3137
3138         /* for snapshot-aware defrag */
3139         if (new) {
3140                 if (ret) {
3141                         free_sa_defrag_extent(new);
3142                         atomic_dec(&fs_info->defrag_running);
3143                 } else {
3144                         relink_file_extents(new);
3145                 }
3146         }
3147
3148         /* once for us */
3149         btrfs_put_ordered_extent(ordered_extent);
3150         /* once for the tree */
3151         btrfs_put_ordered_extent(ordered_extent);
3152
3153         return ret;
3154 }
3155
3156 static void finish_ordered_fn(struct btrfs_work *work)
3157 {
3158         struct btrfs_ordered_extent *ordered_extent;
3159         ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
3160         btrfs_finish_ordered_io(ordered_extent);
3161 }
3162
3163 static void btrfs_writepage_end_io_hook(struct page *page, u64 start, u64 end,
3164                                 struct extent_state *state, int uptodate)
3165 {
3166         struct inode *inode = page->mapping->host;
3167         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3168         struct btrfs_ordered_extent *ordered_extent = NULL;
3169         struct btrfs_workqueue *wq;
3170         btrfs_work_func_t func;
3171
3172         trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
3173
3174         ClearPagePrivate2(page);
3175         if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
3176                                             end - start + 1, uptodate))
3177                 return;
3178
3179         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
3180                 wq = fs_info->endio_freespace_worker;
3181                 func = btrfs_freespace_write_helper;
3182         } else {
3183                 wq = fs_info->endio_write_workers;
3184                 func = btrfs_endio_write_helper;
3185         }
3186
3187         btrfs_init_work(&ordered_extent->work, func, finish_ordered_fn, NULL,
3188                         NULL);
3189         btrfs_queue_work(wq, &ordered_extent->work);
3190 }
3191
3192 static int __readpage_endio_check(struct inode *inode,
3193                                   struct btrfs_io_bio *io_bio,
3194                                   int icsum, struct page *page,
3195                                   int pgoff, u64 start, size_t len)
3196 {
3197         char *kaddr;
3198         u32 csum_expected;
3199         u32 csum = ~(u32)0;
3200
3201         csum_expected = *(((u32 *)io_bio->csum) + icsum);
3202
3203         kaddr = kmap_atomic(page);
3204         csum = btrfs_csum_data(kaddr + pgoff, csum,  len);
3205         btrfs_csum_final(csum, (u8 *)&csum);
3206         if (csum != csum_expected)
3207                 goto zeroit;
3208
3209         kunmap_atomic(kaddr);
3210         return 0;
3211 zeroit:
3212         btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
3213                                     io_bio->mirror_num);
3214         memset(kaddr + pgoff, 1, len);
3215         flush_dcache_page(page);
3216         kunmap_atomic(kaddr);
3217         return -EIO;
3218 }
3219
3220 /*
3221  * when reads are done, we need to check csums to verify the data is correct
3222  * if there's a match, we allow the bio to finish.  If not, the code in
3223  * extent_io.c will try to find good copies for us.
3224  */
3225 static int btrfs_readpage_end_io_hook(struct btrfs_io_bio *io_bio,
3226                                       u64 phy_offset, struct page *page,
3227                                       u64 start, u64 end, int mirror)
3228 {
3229         size_t offset = start - page_offset(page);
3230         struct inode *inode = page->mapping->host;
3231         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
3232         struct btrfs_root *root = BTRFS_I(inode)->root;
3233
3234         if (PageChecked(page)) {
3235                 ClearPageChecked(page);
3236                 return 0;
3237         }
3238
3239         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
3240                 return 0;
3241
3242         if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
3243             test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
3244                 clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
3245                 return 0;
3246         }
3247
3248         phy_offset >>= inode->i_sb->s_blocksize_bits;
3249         return __readpage_endio_check(inode, io_bio, phy_offset, page, offset,
3250                                       start, (size_t)(end - start + 1));
3251 }
3252
3253 /*
3254  * btrfs_add_delayed_iput - perform a delayed iput on @inode
3255  *
3256  * @inode: The inode we want to perform iput on
3257  *
3258  * This function uses the generic vfs_inode::i_count to track whether we should
3259  * just decrement it (in case it's > 1) or if this is the last iput then link
3260  * the inode to the delayed iput machinery. Delayed iputs are processed at
3261  * transaction commit time/superblock commit/cleaner kthread.
3262  */
3263 void btrfs_add_delayed_iput(struct inode *inode)
3264 {
3265         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3266         struct btrfs_inode *binode = BTRFS_I(inode);
3267
3268         if (atomic_add_unless(&inode->i_count, -1, 1))
3269                 return;
3270
3271         spin_lock(&fs_info->delayed_iput_lock);
3272         ASSERT(list_empty(&binode->delayed_iput));
3273         list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
3274         spin_unlock(&fs_info->delayed_iput_lock);
3275 }
3276
3277 void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
3278 {
3279
3280         spin_lock(&fs_info->delayed_iput_lock);
3281         while (!list_empty(&fs_info->delayed_iputs)) {
3282                 struct btrfs_inode *inode;
3283
3284                 inode = list_first_entry(&fs_info->delayed_iputs,
3285                                 struct btrfs_inode, delayed_iput);
3286                 list_del_init(&inode->delayed_iput);
3287                 spin_unlock(&fs_info->delayed_iput_lock);
3288                 iput(&inode->vfs_inode);
3289                 spin_lock(&fs_info->delayed_iput_lock);
3290         }
3291         spin_unlock(&fs_info->delayed_iput_lock);
3292 }
3293
3294 /*
3295  * This is called in transaction commit time. If there are no orphan
3296  * files in the subvolume, it removes orphan item and frees block_rsv
3297  * structure.
3298  */
3299 void btrfs_orphan_commit_root(struct btrfs_trans_handle *trans,
3300                               struct btrfs_root *root)
3301 {
3302         struct btrfs_fs_info *fs_info = root->fs_info;
3303         struct btrfs_block_rsv *block_rsv;
3304         int ret;
3305
3306         if (atomic_read(&root->orphan_inodes) ||
3307             root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE)
3308                 return;
3309
3310         spin_lock(&root->orphan_lock);
3311         if (atomic_read(&root->orphan_inodes)) {
3312                 spin_unlock(&root->orphan_lock);
3313                 return;
3314         }
3315
3316         if (root->orphan_cleanup_state != ORPHAN_CLEANUP_DONE) {
3317                 spin_unlock(&root->orphan_lock);
3318                 return;
3319         }
3320
3321         block_rsv = root->orphan_block_rsv;
3322         root->orphan_block_rsv = NULL;
3323         spin_unlock(&root->orphan_lock);
3324
3325         if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state) &&
3326             btrfs_root_refs(&root->root_item) > 0) {
3327                 ret = btrfs_del_orphan_item(trans, fs_info->tree_root,
3328                                             root->root_key.objectid);
3329                 if (ret)
3330                         btrfs_abort_transaction(trans, ret);
3331                 else
3332                         clear_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED,
3333                                   &root->state);
3334         }
3335
3336         if (block_rsv) {
3337                 WARN_ON(block_rsv->size > 0);
3338                 btrfs_free_block_rsv(fs_info, block_rsv);
3339         }
3340 }
3341
3342 /*
3343  * This creates an orphan entry for the given inode in case something goes
3344  * wrong in the middle of an unlink/truncate.
3345  *
3346  * NOTE: caller of this function should reserve 5 units of metadata for
3347  *       this function.
3348  */
3349 int btrfs_orphan_add(struct btrfs_trans_handle *trans,
3350                 struct btrfs_inode *inode)
3351 {
3352         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
3353         struct btrfs_root *root = inode->root;
3354         struct btrfs_block_rsv *block_rsv = NULL;
3355         int reserve = 0;
3356         bool insert = false;
3357         int ret;
3358
3359         if (!root->orphan_block_rsv) {
3360                 block_rsv = btrfs_alloc_block_rsv(fs_info,
3361                                                   BTRFS_BLOCK_RSV_TEMP);
3362                 if (!block_rsv)
3363                         return -ENOMEM;
3364         }
3365
3366         if (!test_and_set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3367                               &inode->runtime_flags))
3368                 insert = true;
3369
3370         if (!test_and_set_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3371                               &inode->runtime_flags))
3372                 reserve = 1;
3373
3374         spin_lock(&root->orphan_lock);
3375         /* If someone has created ->orphan_block_rsv, be happy to use it. */
3376         if (!root->orphan_block_rsv) {
3377                 root->orphan_block_rsv = block_rsv;
3378         } else if (block_rsv) {
3379                 btrfs_free_block_rsv(fs_info, block_rsv);
3380                 block_rsv = NULL;
3381         }
3382
3383         if (insert)
3384                 atomic_inc(&root->orphan_inodes);
3385         spin_unlock(&root->orphan_lock);
3386
3387         /* grab metadata reservation from transaction handle */
3388         if (reserve) {
3389                 ret = btrfs_orphan_reserve_metadata(trans, inode);
3390                 ASSERT(!ret);
3391                 if (ret) {
3392                         /*
3393                          * dec doesn't need spin_lock as ->orphan_block_rsv
3394                          * would be released only if ->orphan_inodes is
3395                          * zero.
3396                          */
3397                         atomic_dec(&root->orphan_inodes);
3398                         clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3399                                   &inode->runtime_flags);
3400                         if (insert)
3401                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3402                                           &inode->runtime_flags);
3403                         return ret;
3404                 }
3405         }
3406
3407         /* insert an orphan item to track this unlinked/truncated file */
3408         if (insert) {
3409                 ret = btrfs_insert_orphan_item(trans, root, btrfs_ino(inode));
3410                 if (ret) {
3411                         if (reserve) {
3412                                 clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3413                                           &inode->runtime_flags);
3414                                 btrfs_orphan_release_metadata(inode);
3415                         }
3416                         /*
3417                          * btrfs_orphan_commit_root may race with us and set
3418                          * ->orphan_block_rsv to zero, in order to avoid that,
3419                          * decrease ->orphan_inodes after everything is done.
3420                          */
3421                         atomic_dec(&root->orphan_inodes);
3422                         if (ret != -EEXIST) {
3423                                 clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3424                                           &inode->runtime_flags);
3425                                 btrfs_abort_transaction(trans, ret);
3426                                 return ret;
3427                         }
3428                 }
3429                 ret = 0;
3430         }
3431
3432         return 0;
3433 }
3434
3435 /*
3436  * We have done the truncate/delete so we can go ahead and remove the orphan
3437  * item for this particular inode.
3438  */
3439 static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
3440                             struct btrfs_inode *inode)
3441 {
3442         struct btrfs_root *root = inode->root;
3443         int delete_item = 0;
3444         int ret = 0;
3445
3446         if (test_and_clear_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3447                                &inode->runtime_flags))
3448                 delete_item = 1;
3449
3450         if (delete_item && trans)
3451                 ret = btrfs_del_orphan_item(trans, root, btrfs_ino(inode));
3452
3453         if (test_and_clear_bit(BTRFS_INODE_ORPHAN_META_RESERVED,
3454                                &inode->runtime_flags))
3455                 btrfs_orphan_release_metadata(inode);
3456
3457         /*
3458          * btrfs_orphan_commit_root may race with us and set ->orphan_block_rsv
3459          * to zero, in order to avoid that, decrease ->orphan_inodes after
3460          * everything is done.
3461          */
3462         if (delete_item)
3463                 atomic_dec(&root->orphan_inodes);
3464
3465         return ret;
3466 }
3467
3468 /*
3469  * this cleans up any orphans that may be left on the list from the last use
3470  * of this root.
3471  */
3472 int btrfs_orphan_cleanup(struct btrfs_root *root)
3473 {
3474         struct btrfs_fs_info *fs_info = root->fs_info;
3475         struct btrfs_path *path;
3476         struct extent_buffer *leaf;
3477         struct btrfs_key key, found_key;
3478         struct btrfs_trans_handle *trans;
3479         struct inode *inode;
3480         u64 last_objectid = 0;
3481         int ret = 0, nr_unlink = 0, nr_truncate = 0;
3482
3483         if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
3484                 return 0;
3485
3486         path = btrfs_alloc_path();
3487         if (!path) {
3488                 ret = -ENOMEM;
3489                 goto out;
3490         }
3491         path->reada = READA_BACK;
3492
3493         key.objectid = BTRFS_ORPHAN_OBJECTID;
3494         key.type = BTRFS_ORPHAN_ITEM_KEY;
3495         key.offset = (u64)-1;
3496
3497         while (1) {
3498                 ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
3499                 if (ret < 0)
3500                         goto out;
3501
3502                 /*
3503                  * if ret == 0 means we found what we were searching for, which
3504                  * is weird, but possible, so only screw with path if we didn't
3505                  * find the key and see if we have stuff that matches
3506                  */
3507                 if (ret > 0) {
3508                         ret = 0;
3509                         if (path->slots[0] == 0)
3510                                 break;
3511                         path->slots[0]--;
3512                 }
3513
3514                 /* pull out the item */
3515                 leaf = path->nodes[0];
3516                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
3517
3518                 /* make sure the item matches what we want */
3519                 if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
3520                         break;
3521                 if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
3522                         break;
3523
3524                 /* release the path since we're done with it */
3525                 btrfs_release_path(path);
3526
3527                 /*
3528                  * this is where we are basically btrfs_lookup, without the
3529                  * crossing root thing.  we store the inode number in the
3530                  * offset of the orphan item.
3531                  */
3532
3533                 if (found_key.offset == last_objectid) {
3534                         btrfs_err(fs_info,
3535                                   "Error removing orphan entry, stopping orphan cleanup");
3536                         ret = -EINVAL;
3537                         goto out;
3538                 }
3539
3540                 last_objectid = found_key.offset;
3541
3542                 found_key.objectid = found_key.offset;
3543                 found_key.type = BTRFS_INODE_ITEM_KEY;
3544                 found_key.offset = 0;
3545                 inode = btrfs_iget(fs_info->sb, &found_key, root, NULL);
3546                 ret = PTR_ERR_OR_ZERO(inode);
3547                 if (ret && ret != -ENOENT)
3548                         goto out;
3549
3550                 if (ret == -ENOENT && root == fs_info->tree_root) {
3551                         struct btrfs_root *dead_root;
3552                         struct btrfs_fs_info *fs_info = root->fs_info;
3553                         int is_dead_root = 0;
3554
3555                         /*
3556                          * this is an orphan in the tree root. Currently these
3557                          * could come from 2 sources:
3558                          *  a) a snapshot deletion in progress
3559                          *  b) a free space cache inode
3560                          * We need to distinguish those two, as the snapshot
3561                          * orphan must not get deleted.
3562                          * find_dead_roots already ran before us, so if this
3563                          * is a snapshot deletion, we should find the root
3564                          * in the dead_roots list
3565                          */
3566                         spin_lock(&fs_info->trans_lock);
3567                         list_for_each_entry(dead_root, &fs_info->dead_roots,
3568                                             root_list) {
3569                                 if (dead_root->root_key.objectid ==
3570                                     found_key.objectid) {
3571                                         is_dead_root = 1;
3572                                         break;
3573                                 }
3574                         }
3575                         spin_unlock(&fs_info->trans_lock);
3576                         if (is_dead_root) {
3577                                 /* prevent this orphan from being found again */
3578                                 key.offset = found_key.objectid - 1;
3579                                 continue;
3580                         }
3581                 }
3582                 /*
3583                  * Inode is already gone but the orphan item is still there,
3584                  * kill the orphan item.
3585                  */
3586                 if (ret == -ENOENT) {
3587                         trans = btrfs_start_transaction(root, 1);
3588                         if (IS_ERR(trans)) {
3589                                 ret = PTR_ERR(trans);
3590                                 goto out;
3591                         }
3592                         btrfs_debug(fs_info, "auto deleting %Lu",
3593                                     found_key.objectid);
3594                         ret = btrfs_del_orphan_item(trans, root,
3595                                                     found_key.objectid);
3596                         btrfs_end_transaction(trans);
3597                         if (ret)
3598                                 goto out;
3599                         continue;
3600                 }
3601
3602                 /*
3603                  * add this inode to the orphan list so btrfs_orphan_del does
3604                  * the proper thing when we hit it
3605                  */
3606                 set_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
3607                         &BTRFS_I(inode)->runtime_flags);
3608                 atomic_inc(&root->orphan_inodes);
3609
3610                 /* if we have links, this was a truncate, lets do that */
3611                 if (inode->i_nlink) {
3612                         if (WARN_ON(!S_ISREG(inode->i_mode))) {
3613                                 iput(inode);
3614                                 continue;
3615                         }
3616                         nr_truncate++;
3617
3618                         /* 1 for the orphan item deletion. */
3619                         trans = btrfs_start_transaction(root, 1);
3620                         if (IS_ERR(trans)) {
3621                                 iput(inode);
3622                                 ret = PTR_ERR(trans);
3623                                 goto out;
3624                         }
3625                         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
3626                         btrfs_end_transaction(trans);
3627                         if (ret) {
3628                                 iput(inode);
3629                                 goto out;
3630                         }
3631
3632                         ret = btrfs_truncate(inode, false);
3633                         if (ret)
3634                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
3635                 } else {
3636                         nr_unlink++;
3637                 }
3638
3639                 /* this will do delete_inode and everything for us */
3640                 iput(inode);
3641                 if (ret)
3642                         goto out;
3643         }
3644         /* release the path since we're done with it */
3645         btrfs_release_path(path);
3646
3647         root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
3648
3649         if (root->orphan_block_rsv)
3650                 btrfs_block_rsv_release(fs_info, root->orphan_block_rsv,
3651                                         (u64)-1);
3652
3653         if (root->orphan_block_rsv ||
3654             test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
3655                 trans = btrfs_join_transaction(root);
3656                 if (!IS_ERR(trans))
3657                         btrfs_end_transaction(trans);
3658         }
3659
3660         if (nr_unlink)
3661                 btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
3662         if (nr_truncate)
3663                 btrfs_debug(fs_info, "truncated %d orphans", nr_truncate);
3664
3665 out:
3666         if (ret)
3667                 btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
3668         btrfs_free_path(path);
3669         return ret;
3670 }
3671
3672 /*
3673  * very simple check to peek ahead in the leaf looking for xattrs.  If we
3674  * don't find any xattrs, we know there can't be any acls.
3675  *
3676  * slot is the slot the inode is in, objectid is the objectid of the inode
3677  */
3678 static noinline int acls_after_inode_item(struct extent_buffer *leaf,
3679                                           int slot, u64 objectid,
3680                                           int *first_xattr_slot)
3681 {
3682         u32 nritems = btrfs_header_nritems(leaf);
3683         struct btrfs_key found_key;
3684         static u64 xattr_access = 0;
3685         static u64 xattr_default = 0;
3686         int scanned = 0;
3687
3688         if (!xattr_access) {
3689                 xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
3690                                         strlen(XATTR_NAME_POSIX_ACL_ACCESS));
3691                 xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
3692                                         strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
3693         }
3694
3695         slot++;
3696         *first_xattr_slot = -1;
3697         while (slot < nritems) {
3698                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
3699
3700                 /* we found a different objectid, there must not be acls */
3701                 if (found_key.objectid != objectid)
3702                         return 0;
3703
3704                 /* we found an xattr, assume we've got an acl */
3705                 if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
3706                         if (*first_xattr_slot == -1)
3707                                 *first_xattr_slot = slot;
3708                         if (found_key.offset == xattr_access ||
3709                             found_key.offset == xattr_default)
3710                                 return 1;
3711                 }
3712
3713                 /*
3714                  * we found a key greater than an xattr key, there can't
3715                  * be any acls later on
3716                  */
3717                 if (found_key.type > BTRFS_XATTR_ITEM_KEY)
3718                         return 0;
3719
3720                 slot++;
3721                 scanned++;
3722
3723                 /*
3724                  * it goes inode, inode backrefs, xattrs, extents,
3725                  * so if there are a ton of hard links to an inode there can
3726                  * be a lot of backrefs.  Don't waste time searching too hard,
3727                  * this is just an optimization
3728                  */
3729                 if (scanned >= 8)
3730                         break;
3731         }
3732         /* we hit the end of the leaf before we found an xattr or
3733          * something larger than an xattr.  We have to assume the inode
3734          * has acls
3735          */
3736         if (*first_xattr_slot == -1)
3737                 *first_xattr_slot = slot;
3738         return 1;
3739 }
3740
3741 /*
3742  * read an inode from the btree into the in-memory inode
3743  */
3744 static int btrfs_read_locked_inode(struct inode *inode)
3745 {
3746         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
3747         struct btrfs_path *path;
3748         struct extent_buffer *leaf;
3749         struct btrfs_inode_item *inode_item;
3750         struct btrfs_root *root = BTRFS_I(inode)->root;
3751         struct btrfs_key location;
3752         unsigned long ptr;
3753         int maybe_acls;
3754         u32 rdev;
3755         int ret;
3756         bool filled = false;
3757         int first_xattr_slot;
3758
3759         ret = btrfs_fill_inode(inode, &rdev);
3760         if (!ret)
3761                 filled = true;
3762
3763         path = btrfs_alloc_path();
3764         if (!path) {
3765                 ret = -ENOMEM;
3766                 goto make_bad;
3767         }
3768
3769         memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
3770
3771         ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
3772         if (ret) {
3773                 if (ret > 0)
3774                         ret = -ENOENT;
3775                 goto make_bad;
3776         }
3777
3778         leaf = path->nodes[0];
3779
3780         if (filled)
3781                 goto cache_index;
3782
3783         inode_item = btrfs_item_ptr(leaf, path->slots[0],
3784                                     struct btrfs_inode_item);
3785         inode->i_mode = btrfs_inode_mode(leaf, inode_item);
3786         set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
3787         i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
3788         i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
3789         btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
3790
3791         inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
3792         inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
3793
3794         inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
3795         inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
3796
3797         inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
3798         inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
3799
3800         BTRFS_I(inode)->i_otime.tv_sec =
3801                 btrfs_timespec_sec(leaf, &inode_item->otime);
3802         BTRFS_I(inode)->i_otime.tv_nsec =
3803                 btrfs_timespec_nsec(leaf, &inode_item->otime);
3804
3805         inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
3806         BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
3807         BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
3808
3809         inode_set_iversion_queried(inode,
3810                                    btrfs_inode_sequence(leaf, inode_item));
3811         inode->i_generation = BTRFS_I(inode)->generation;
3812         inode->i_rdev = 0;
3813         rdev = btrfs_inode_rdev(leaf, inode_item);
3814
3815         BTRFS_I(inode)->index_cnt = (u64)-1;
3816         BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
3817
3818 cache_index:
3819         /*
3820          * If we were modified in the current generation and evicted from memory
3821          * and then re-read we need to do a full sync since we don't have any
3822          * idea about which extents were modified before we were evicted from
3823          * cache.
3824          *
3825          * This is required for both inode re-read from disk and delayed inode
3826          * in delayed_nodes_tree.
3827          */
3828         if (BTRFS_I(inode)->last_trans == fs_info->generation)
3829                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
3830                         &BTRFS_I(inode)->runtime_flags);
3831
3832         /*
3833          * We don't persist the id of the transaction where an unlink operation
3834          * against the inode was last made. So here we assume the inode might
3835          * have been evicted, and therefore the exact value of last_unlink_trans
3836          * lost, and set it to last_trans to avoid metadata inconsistencies
3837          * between the inode and its parent if the inode is fsync'ed and the log
3838          * replayed. For example, in the scenario:
3839          *
3840          * touch mydir/foo
3841          * ln mydir/foo mydir/bar
3842          * sync
3843          * unlink mydir/bar
3844          * echo 2 > /proc/sys/vm/drop_caches   # evicts inode
3845          * xfs_io -c fsync mydir/foo
3846          * <power failure>
3847          * mount fs, triggers fsync log replay
3848          *
3849          * We must make sure that when we fsync our inode foo we also log its
3850          * parent inode, otherwise after log replay the parent still has the
3851          * dentry with the "bar" name but our inode foo has a link count of 1
3852          * and doesn't have an inode ref with the name "bar" anymore.
3853          *
3854          * Setting last_unlink_trans to last_trans is a pessimistic approach,
3855          * but it guarantees correctness at the expense of occasional full
3856          * transaction commits on fsync if our inode is a directory, or if our
3857          * inode is not a directory, logging its parent unnecessarily.
3858          */
3859         BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
3860
3861         path->slots[0]++;
3862         if (inode->i_nlink != 1 ||
3863             path->slots[0] >= btrfs_header_nritems(leaf))
3864                 goto cache_acl;
3865
3866         btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
3867         if (location.objectid != btrfs_ino(BTRFS_I(inode)))
3868                 goto cache_acl;
3869
3870         ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
3871         if (location.type == BTRFS_INODE_REF_KEY) {
3872                 struct btrfs_inode_ref *ref;
3873
3874                 ref = (struct btrfs_inode_ref *)ptr;
3875                 BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
3876         } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
3877                 struct btrfs_inode_extref *extref;
3878
3879                 extref = (struct btrfs_inode_extref *)ptr;
3880                 BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
3881                                                                      extref);
3882         }
3883 cache_acl:
3884         /*
3885          * try to precache a NULL acl entry for files that don't have
3886          * any xattrs or acls
3887          */
3888         maybe_acls = acls_after_inode_item(leaf, path->slots[0],
3889                         btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
3890         if (first_xattr_slot != -1) {
3891                 path->slots[0] = first_xattr_slot;
3892                 ret = btrfs_load_inode_props(inode, path);
3893                 if (ret)
3894                         btrfs_err(fs_info,
3895                                   "error loading props for ino %llu (root %llu): %d",
3896                                   btrfs_ino(BTRFS_I(inode)),
3897                                   root->root_key.objectid, ret);
3898         }
3899         btrfs_free_path(path);
3900
3901         if (!maybe_acls)
3902                 cache_no_acl(inode);
3903
3904         switch (inode->i_mode & S_IFMT) {
3905         case S_IFREG:
3906                 inode->i_mapping->a_ops = &btrfs_aops;
3907                 BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
3908                 inode->i_fop = &btrfs_file_operations;
3909                 inode->i_op = &btrfs_file_inode_operations;
3910                 break;
3911         case S_IFDIR:
3912                 inode->i_fop = &btrfs_dir_file_operations;
3913                 inode->i_op = &btrfs_dir_inode_operations;
3914                 break;
3915         case S_IFLNK:
3916                 inode->i_op = &btrfs_symlink_inode_operations;
3917                 inode_nohighmem(inode);
3918                 inode->i_mapping->a_ops = &btrfs_symlink_aops;
3919                 break;
3920         default:
3921                 inode->i_op = &btrfs_special_inode_operations;
3922                 init_special_inode(inode, inode->i_mode, rdev);
3923                 break;
3924         }
3925
3926         btrfs_update_iflags(inode);
3927         return 0;
3928
3929 make_bad:
3930         btrfs_free_path(path);
3931         make_bad_inode(inode);
3932         return ret;
3933 }
3934
3935 /*
3936  * given a leaf and an inode, copy the inode fields into the leaf
3937  */
3938 static void fill_inode_item(struct btrfs_trans_handle *trans,
3939                             struct extent_buffer *leaf,
3940                             struct btrfs_inode_item *item,
3941                             struct inode *inode)
3942 {
3943         struct btrfs_map_token token;
3944
3945         btrfs_init_map_token(&token);
3946
3947         btrfs_set_token_inode_uid(leaf, item, i_uid_read(inode), &token);
3948         btrfs_set_token_inode_gid(leaf, item, i_gid_read(inode), &token);
3949         btrfs_set_token_inode_size(leaf, item, BTRFS_I(inode)->disk_i_size,
3950                                    &token);
3951         btrfs_set_token_inode_mode(leaf, item, inode->i_mode, &token);
3952         btrfs_set_token_inode_nlink(leaf, item, inode->i_nlink, &token);
3953
3954         btrfs_set_token_timespec_sec(leaf, &item->atime,
3955                                      inode->i_atime.tv_sec, &token);
3956         btrfs_set_token_timespec_nsec(leaf, &item->atime,
3957                                       inode->i_atime.tv_nsec, &token);
3958
3959         btrfs_set_token_timespec_sec(leaf, &item->mtime,
3960                                      inode->i_mtime.tv_sec, &token);
3961         btrfs_set_token_timespec_nsec(leaf, &item->mtime,
3962                                       inode->i_mtime.tv_nsec, &token);
3963
3964         btrfs_set_token_timespec_sec(leaf, &item->ctime,
3965                                      inode->i_ctime.tv_sec, &token);
3966         btrfs_set_token_timespec_nsec(leaf, &item->ctime,
3967                                       inode->i_ctime.tv_nsec, &token);
3968
3969         btrfs_set_token_timespec_sec(leaf, &item->otime,
3970                                      BTRFS_I(inode)->i_otime.tv_sec, &token);
3971         btrfs_set_token_timespec_nsec(leaf, &item->otime,
3972                                       BTRFS_I(inode)->i_otime.tv_nsec, &token);
3973
3974         btrfs_set_token_inode_nbytes(leaf, item, inode_get_bytes(inode),
3975                                      &token);
3976         btrfs_set_token_inode_generation(leaf, item, BTRFS_I(inode)->generation,
3977                                          &token);
3978         btrfs_set_token_inode_sequence(leaf, item, inode_peek_iversion(inode),
3979                                        &token);
3980         btrfs_set_token_inode_transid(leaf, item, trans->transid, &token);
3981         btrfs_set_token_inode_rdev(leaf, item, inode->i_rdev, &token);
3982         btrfs_set_token_inode_flags(leaf, item, BTRFS_I(inode)->flags, &token);
3983         btrfs_set_token_inode_block_group(leaf, item, 0, &token);
3984 }
3985
3986 /*
3987  * copy everything in the in-memory inode into the btree.
3988  */
3989 static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
3990                                 struct btrfs_root *root, struct inode *inode)
3991 {
3992         struct btrfs_inode_item *inode_item;
3993         struct btrfs_path *path;
3994         struct extent_buffer *leaf;
3995         int ret;
3996
3997         path = btrfs_alloc_path();
3998         if (!path)
3999                 return -ENOMEM;
4000
4001         path->leave_spinning = 1;
4002         ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
4003                                  1);
4004         if (ret) {
4005                 if (ret > 0)
4006                         ret = -ENOENT;
4007                 goto failed;
4008         }
4009
4010         leaf = path->nodes[0];
4011         inode_item = btrfs_item_ptr(leaf, path->slots[0],
4012                                     struct btrfs_inode_item);
4013
4014         fill_inode_item(trans, leaf, inode_item, inode);
4015         btrfs_mark_buffer_dirty(leaf);
4016         btrfs_set_inode_last_trans(trans, inode);
4017         ret = 0;
4018 failed:
4019         btrfs_free_path(path);
4020         return ret;
4021 }
4022
4023 /*
4024  * copy everything in the in-memory inode into the btree.
4025  */
4026 noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
4027                                 struct btrfs_root *root, struct inode *inode)
4028 {
4029         struct btrfs_fs_info *fs_info = root->fs_info;
4030         int ret;
4031
4032         /*
4033          * If the inode is a free space inode, we can deadlock during commit
4034          * if we put it into the delayed code.
4035          *
4036          * The data relocation inode should also be directly updated
4037          * without delay
4038          */
4039         if (!btrfs_is_free_space_inode(BTRFS_I(inode))
4040             && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
4041             && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
4042                 btrfs_update_root_times(trans, root);
4043
4044                 ret = btrfs_delayed_update_inode(trans, root, inode);
4045                 if (!ret)
4046                         btrfs_set_inode_last_trans(trans, inode);
4047                 return ret;
4048         }
4049
4050         return btrfs_update_inode_item(trans, root, inode);
4051 }
4052
4053 noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
4054                                          struct btrfs_root *root,
4055                                          struct inode *inode)
4056 {
4057         int ret;
4058
4059         ret = btrfs_update_inode(trans, root, inode);
4060         if (ret == -ENOSPC)
4061                 return btrfs_update_inode_item(trans, root, inode);
4062         return ret;
4063 }
4064
4065 /*
4066  * unlink helper that gets used here in inode.c and in the tree logging
4067  * recovery code.  It remove a link in a directory with a given name, and
4068  * also drops the back refs in the inode to the directory
4069  */
4070 static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4071                                 struct btrfs_root *root,
4072                                 struct btrfs_inode *dir,
4073                                 struct btrfs_inode *inode,
4074                                 const char *name, int name_len)
4075 {
4076         struct btrfs_fs_info *fs_info = root->fs_info;
4077         struct btrfs_path *path;
4078         int ret = 0;
4079         struct extent_buffer *leaf;
4080         struct btrfs_dir_item *di;
4081         struct btrfs_key key;
4082         u64 index;
4083         u64 ino = btrfs_ino(inode);
4084         u64 dir_ino = btrfs_ino(dir);
4085
4086         path = btrfs_alloc_path();
4087         if (!path) {
4088                 ret = -ENOMEM;
4089                 goto out;
4090         }
4091
4092         path->leave_spinning = 1;
4093         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4094                                     name, name_len, -1);
4095         if (IS_ERR(di)) {
4096                 ret = PTR_ERR(di);
4097                 goto err;
4098         }
4099         if (!di) {
4100                 ret = -ENOENT;
4101                 goto err;
4102         }
4103         leaf = path->nodes[0];
4104         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4105         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4106         if (ret)
4107                 goto err;
4108         btrfs_release_path(path);
4109
4110         /*
4111          * If we don't have dir index, we have to get it by looking up
4112          * the inode ref, since we get the inode ref, remove it directly,
4113          * it is unnecessary to do delayed deletion.
4114          *
4115          * But if we have dir index, needn't search inode ref to get it.
4116          * Since the inode ref is close to the inode item, it is better
4117          * that we delay to delete it, and just do this deletion when
4118          * we update the inode item.
4119          */
4120         if (inode->dir_index) {
4121                 ret = btrfs_delayed_delete_inode_ref(inode);
4122                 if (!ret) {
4123                         index = inode->dir_index;
4124                         goto skip_backref;
4125                 }
4126         }
4127
4128         ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
4129                                   dir_ino, &index);
4130         if (ret) {
4131                 btrfs_info(fs_info,
4132                         "failed to delete reference to %.*s, inode %llu parent %llu",
4133                         name_len, name, ino, dir_ino);
4134                 btrfs_abort_transaction(trans, ret);
4135                 goto err;
4136         }
4137 skip_backref:
4138         ret = btrfs_delete_delayed_dir_index(trans, fs_info, dir, index);
4139         if (ret) {
4140                 btrfs_abort_transaction(trans, ret);
4141                 goto err;
4142         }
4143
4144         ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
4145                         dir_ino);
4146         if (ret != 0 && ret != -ENOENT) {
4147                 btrfs_abort_transaction(trans, ret);
4148                 goto err;
4149         }
4150
4151         ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
4152                         index);
4153         if (ret == -ENOENT)
4154                 ret = 0;
4155         else if (ret)
4156                 btrfs_abort_transaction(trans, ret);
4157 err:
4158         btrfs_free_path(path);
4159         if (ret)
4160                 goto out;
4161
4162         btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
4163         inode_inc_iversion(&inode->vfs_inode);
4164         inode_inc_iversion(&dir->vfs_inode);
4165         inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
4166                 dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
4167         ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
4168 out:
4169         return ret;
4170 }
4171
4172 int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
4173                        struct btrfs_root *root,
4174                        struct btrfs_inode *dir, struct btrfs_inode *inode,
4175                        const char *name, int name_len)
4176 {
4177         int ret;
4178         ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
4179         if (!ret) {
4180                 drop_nlink(&inode->vfs_inode);
4181                 ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
4182         }
4183         return ret;
4184 }
4185
4186 /*
4187  * helper to start transaction for unlink and rmdir.
4188  *
4189  * unlink and rmdir are special in btrfs, they do not always free space, so
4190  * if we cannot make our reservations the normal way try and see if there is
4191  * plenty of slack room in the global reserve to migrate, otherwise we cannot
4192  * allow the unlink to occur.
4193  */
4194 static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
4195 {
4196         struct btrfs_root *root = BTRFS_I(dir)->root;
4197
4198         /*
4199          * 1 for the possible orphan item
4200          * 1 for the dir item
4201          * 1 for the dir index
4202          * 1 for the inode ref
4203          * 1 for the inode
4204          */
4205         return btrfs_start_transaction_fallback_global_rsv(root, 5, 5);
4206 }
4207
4208 static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
4209 {
4210         struct btrfs_root *root = BTRFS_I(dir)->root;
4211         struct btrfs_trans_handle *trans;
4212         struct inode *inode = d_inode(dentry);
4213         int ret;
4214
4215         trans = __unlink_start_trans(dir);
4216         if (IS_ERR(trans))
4217                 return PTR_ERR(trans);
4218
4219         btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
4220                         0);
4221
4222         ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4223                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4224                         dentry->d_name.len);
4225         if (ret)
4226                 goto out;
4227
4228         if (inode->i_nlink == 0) {
4229                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
4230                 if (ret)
4231                         goto out;
4232         }
4233
4234 out:
4235         btrfs_end_transaction(trans);
4236         btrfs_btree_balance_dirty(root->fs_info);
4237         return ret;
4238 }
4239
4240 int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
4241                         struct btrfs_root *root,
4242                         struct inode *dir, u64 objectid,
4243                         const char *name, int name_len)
4244 {
4245         struct btrfs_fs_info *fs_info = root->fs_info;
4246         struct btrfs_path *path;
4247         struct extent_buffer *leaf;
4248         struct btrfs_dir_item *di;
4249         struct btrfs_key key;
4250         u64 index;
4251         int ret;
4252         u64 dir_ino = btrfs_ino(BTRFS_I(dir));
4253
4254         path = btrfs_alloc_path();
4255         if (!path)
4256                 return -ENOMEM;
4257
4258         di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
4259                                    name, name_len, -1);
4260         if (IS_ERR_OR_NULL(di)) {
4261                 if (!di)
4262                         ret = -ENOENT;
4263                 else
4264                         ret = PTR_ERR(di);
4265                 goto out;
4266         }
4267
4268         leaf = path->nodes[0];
4269         btrfs_dir_item_key_to_cpu(leaf, di, &key);
4270         WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
4271         ret = btrfs_delete_one_dir_name(trans, root, path, di);
4272         if (ret) {
4273                 btrfs_abort_transaction(trans, ret);
4274                 goto out;
4275         }
4276         btrfs_release_path(path);
4277
4278         ret = btrfs_del_root_ref(trans, fs_info, objectid,
4279                                  root->root_key.objectid, dir_ino,
4280                                  &index, name, name_len);
4281         if (ret < 0) {
4282                 if (ret != -ENOENT) {
4283                         btrfs_abort_transaction(trans, ret);
4284                         goto out;
4285                 }
4286                 di = btrfs_search_dir_index_item(root, path, dir_ino,
4287                                                  name, name_len);
4288                 if (IS_ERR_OR_NULL(di)) {
4289                         if (!di)
4290                                 ret = -ENOENT;
4291                         else
4292                                 ret = PTR_ERR(di);
4293                         btrfs_abort_transaction(trans, ret);
4294                         goto out;
4295                 }
4296
4297                 leaf = path->nodes[0];
4298                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
4299                 btrfs_release_path(path);
4300                 index = key.offset;
4301         }
4302         btrfs_release_path(path);
4303
4304         ret = btrfs_delete_delayed_dir_index(trans, fs_info, BTRFS_I(dir), index);
4305         if (ret) {
4306                 btrfs_abort_transaction(trans, ret);
4307                 goto out;
4308         }
4309
4310         btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
4311         inode_inc_iversion(dir);
4312         dir->i_mtime = dir->i_ctime = current_time(dir);
4313         ret = btrfs_update_inode_fallback(trans, root, dir);
4314         if (ret)
4315                 btrfs_abort_transaction(trans, ret);
4316 out:
4317         btrfs_free_path(path);
4318         return ret;
4319 }
4320
4321 static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
4322 {
4323         struct inode *inode = d_inode(dentry);
4324         int err = 0;
4325         struct btrfs_root *root = BTRFS_I(dir)->root;
4326         struct btrfs_trans_handle *trans;
4327         u64 last_unlink_trans;
4328
4329         if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
4330                 return -ENOTEMPTY;
4331         if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
4332                 return -EPERM;
4333
4334         trans = __unlink_start_trans(dir);
4335         if (IS_ERR(trans))
4336                 return PTR_ERR(trans);
4337
4338         if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
4339                 err = btrfs_unlink_subvol(trans, root, dir,
4340                                           BTRFS_I(inode)->location.objectid,
4341                                           dentry->d_name.name,
4342                                           dentry->d_name.len);
4343                 goto out;
4344         }
4345
4346         err = btrfs_orphan_add(trans, BTRFS_I(inode));
4347         if (err)
4348                 goto out;
4349
4350         last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
4351
4352         /* now the directory is empty */
4353         err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
4354                         BTRFS_I(d_inode(dentry)), dentry->d_name.name,
4355                         dentry->d_name.len);
4356         if (!err) {
4357                 btrfs_i_size_write(BTRFS_I(inode), 0);
4358                 /*
4359                  * Propagate the last_unlink_trans value of the deleted dir to
4360                  * its parent directory. This is to prevent an unrecoverable
4361                  * log tree in the case we do something like this:
4362                  * 1) create dir foo
4363                  * 2) create snapshot under dir foo
4364                  * 3) delete the snapshot
4365                  * 4) rmdir foo
4366                  * 5) mkdir foo
4367                  * 6) fsync foo or some file inside foo
4368                  */
4369                 if (last_unlink_trans >= trans->transid)
4370                         BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
4371         }
4372 out:
4373         btrfs_end_transaction(trans);
4374         btrfs_btree_balance_dirty(root->fs_info);
4375
4376         return err;
4377 }
4378
4379 static int truncate_space_check(struct btrfs_trans_handle *trans,
4380                                 struct btrfs_root *root,
4381                                 u64 bytes_deleted)
4382 {
4383         struct btrfs_fs_info *fs_info = root->fs_info;
4384         int ret;
4385
4386         /*
4387          * This is only used to apply pressure to the enospc system, we don't
4388          * intend to use this reservation at all.
4389          */
4390         bytes_deleted = btrfs_csum_bytes_to_leaves(fs_info, bytes_deleted);
4391         bytes_deleted *= fs_info->nodesize;
4392         ret = btrfs_block_rsv_add(root, &fs_info->trans_block_rsv,
4393                                   bytes_deleted, BTRFS_RESERVE_NO_FLUSH);
4394         if (!ret) {
4395                 trace_btrfs_space_reservation(fs_info, "transaction",
4396                                               trans->transid,
4397                                               bytes_deleted, 1);
4398                 trans->bytes_reserved += bytes_deleted;
4399         }
4400         return ret;
4401
4402 }
4403
4404 /*
4405  * Return this if we need to call truncate_block for the last bit of the
4406  * truncate.
4407  */
4408 #define NEED_TRUNCATE_BLOCK 1
4409
4410 /*
4411  * this can truncate away extent items, csum items and directory items.
4412  * It starts at a high offset and removes keys until it can't find
4413  * any higher than new_size
4414  *
4415  * csum items that cross the new i_size are truncated to the new size
4416  * as well.
4417  *
4418  * min_type is the minimum key type to truncate down to.  If set to 0, this
4419  * will kill all the items on this inode, including the INODE_ITEM_KEY.
4420  */
4421 int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
4422                                struct btrfs_root *root,
4423                                struct inode *inode,
4424                                u64 new_size, u32 min_type)
4425 {
4426         struct btrfs_fs_info *fs_info = root->fs_info;
4427         struct btrfs_path *path;
4428         struct extent_buffer *leaf;
4429         struct btrfs_file_extent_item *fi;
4430         struct btrfs_key key;
4431         struct btrfs_key found_key;
4432         u64 extent_start = 0;
4433         u64 extent_num_bytes = 0;
4434         u64 extent_offset = 0;
4435         u64 item_end = 0;
4436         u64 last_size = new_size;
4437         u32 found_type = (u8)-1;
4438         int found_extent;
4439         int del_item;
4440         int pending_del_nr = 0;
4441         int pending_del_slot = 0;
4442         int extent_type = -1;
4443         int ret;
4444         int err = 0;
4445         u64 ino = btrfs_ino(BTRFS_I(inode));
4446         u64 bytes_deleted = 0;
4447         bool be_nice = false;
4448         bool should_throttle = false;
4449         bool should_end = false;
4450
4451         BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
4452
4453         /*
4454          * for non-free space inodes and ref cows, we want to back off from
4455          * time to time
4456          */
4457         if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
4458             test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4459                 be_nice = true;
4460
4461         path = btrfs_alloc_path();
4462         if (!path)
4463                 return -ENOMEM;
4464         path->reada = READA_BACK;
4465
4466         /*
4467          * We want to drop from the next block forward in case this new size is
4468          * not block aligned since we will be keeping the last block of the
4469          * extent just the way it is.
4470          */
4471         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4472             root == fs_info->tree_root)
4473                 btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
4474                                         fs_info->sectorsize),
4475                                         (u64)-1, 0);
4476
4477         /*
4478          * This function is also used to drop the items in the log tree before
4479          * we relog the inode, so if root != BTRFS_I(inode)->root, it means
4480          * it is used to drop the loged items. So we shouldn't kill the delayed
4481          * items.
4482          */
4483         if (min_type == 0 && root == BTRFS_I(inode)->root)
4484                 btrfs_kill_delayed_inode_items(BTRFS_I(inode));
4485
4486         key.objectid = ino;
4487         key.offset = (u64)-1;
4488         key.type = (u8)-1;
4489
4490 search_again:
4491         /*
4492          * with a 16K leaf size and 128MB extents, you can actually queue
4493          * up a huge file in a single leaf.  Most of the time that
4494          * bytes_deleted is > 0, it will be huge by the time we get here
4495          */
4496         if (be_nice && bytes_deleted > SZ_32M) {
4497                 if (btrfs_should_end_transaction(trans)) {
4498                         err = -EAGAIN;
4499                         goto error;
4500                 }
4501         }
4502
4503
4504         path->leave_spinning = 1;
4505         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
4506         if (ret < 0) {
4507                 err = ret;
4508                 goto out;
4509         }
4510
4511         if (ret > 0) {
4512                 /* there are no items in the tree for us to truncate, we're
4513                  * done
4514                  */
4515                 if (path->slots[0] == 0)
4516                         goto out;
4517                 path->slots[0]--;
4518         }
4519
4520         while (1) {
4521                 fi = NULL;
4522                 leaf = path->nodes[0];
4523                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
4524                 found_type = found_key.type;
4525
4526                 if (found_key.objectid != ino)
4527                         break;
4528
4529                 if (found_type < min_type)
4530                         break;
4531
4532                 item_end = found_key.offset;
4533                 if (found_type == BTRFS_EXTENT_DATA_KEY) {
4534                         fi = btrfs_item_ptr(leaf, path->slots[0],
4535                                             struct btrfs_file_extent_item);
4536                         extent_type = btrfs_file_extent_type(leaf, fi);
4537                         if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4538                                 item_end +=
4539                                     btrfs_file_extent_num_bytes(leaf, fi);
4540
4541                                 trace_btrfs_truncate_show_fi_regular(
4542                                         BTRFS_I(inode), leaf, fi,
4543                                         found_key.offset);
4544                         } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4545                                 item_end += btrfs_file_extent_inline_len(leaf,
4546                                                          path->slots[0], fi);
4547
4548                                 trace_btrfs_truncate_show_fi_inline(
4549                                         BTRFS_I(inode), leaf, fi, path->slots[0],
4550                                         found_key.offset);
4551                         }
4552                         item_end--;
4553                 }
4554                 if (found_type > min_type) {
4555                         del_item = 1;
4556                 } else {
4557                         if (item_end < new_size)
4558                                 break;
4559                         if (found_key.offset >= new_size)
4560                                 del_item = 1;
4561                         else
4562                                 del_item = 0;
4563                 }
4564                 found_extent = 0;
4565                 /* FIXME, shrink the extent if the ref count is only 1 */
4566                 if (found_type != BTRFS_EXTENT_DATA_KEY)
4567                         goto delete;
4568
4569                 if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
4570                         u64 num_dec;
4571                         extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
4572                         if (!del_item) {
4573                                 u64 orig_num_bytes =
4574                                         btrfs_file_extent_num_bytes(leaf, fi);
4575                                 extent_num_bytes = ALIGN(new_size -
4576                                                 found_key.offset,
4577                                                 fs_info->sectorsize);
4578                                 btrfs_set_file_extent_num_bytes(leaf, fi,
4579                                                          extent_num_bytes);
4580                                 num_dec = (orig_num_bytes -
4581                                            extent_num_bytes);
4582                                 if (test_bit(BTRFS_ROOT_REF_COWS,
4583                                              &root->state) &&
4584                                     extent_start != 0)
4585                                         inode_sub_bytes(inode, num_dec);
4586                                 btrfs_mark_buffer_dirty(leaf);
4587                         } else {
4588                                 extent_num_bytes =
4589                                         btrfs_file_extent_disk_num_bytes(leaf,
4590                                                                          fi);
4591                                 extent_offset = found_key.offset -
4592                                         btrfs_file_extent_offset(leaf, fi);
4593
4594                                 /* FIXME blocksize != 4096 */
4595                                 num_dec = btrfs_file_extent_num_bytes(leaf, fi);
4596                                 if (extent_start != 0) {
4597                                         found_extent = 1;
4598                                         if (test_bit(BTRFS_ROOT_REF_COWS,
4599                                                      &root->state))
4600                                                 inode_sub_bytes(inode, num_dec);
4601                                 }
4602                         }
4603                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
4604                         /*
4605                          * we can't truncate inline items that have had
4606                          * special encodings
4607                          */
4608                         if (!del_item &&
4609                             btrfs_file_extent_encryption(leaf, fi) == 0 &&
4610                             btrfs_file_extent_other_encoding(leaf, fi) == 0 &&
4611                             btrfs_file_extent_compression(leaf, fi) == 0) {
4612                                 u32 size = (u32)(new_size - found_key.offset);
4613
4614                                 btrfs_set_file_extent_ram_bytes(leaf, fi, size);
4615                                 size = btrfs_file_extent_calc_inline_size(size);
4616                                 btrfs_truncate_item(root->fs_info, path, size, 1);
4617                         } else if (!del_item) {
4618                                 /*
4619                                  * We have to bail so the last_size is set to
4620                                  * just before this extent.
4621                                  */
4622                                 err = NEED_TRUNCATE_BLOCK;
4623                                 break;
4624                         }
4625
4626                         if (test_bit(BTRFS_ROOT_REF_COWS, &root->state))
4627                                 inode_sub_bytes(inode, item_end + 1 - new_size);
4628                 }
4629 delete:
4630                 if (del_item)
4631                         last_size = found_key.offset;
4632                 else
4633                         last_size = new_size;
4634                 if (del_item) {
4635                         if (!pending_del_nr) {
4636                                 /* no pending yet, add ourselves */
4637                                 pending_del_slot = path->slots[0];
4638                                 pending_del_nr = 1;
4639                         } else if (pending_del_nr &&
4640                                    path->slots[0] + 1 == pending_del_slot) {
4641                                 /* hop on the pending chunk */
4642                                 pending_del_nr++;
4643                                 pending_del_slot = path->slots[0];
4644                         } else {
4645                                 BUG();
4646                         }
4647                 } else {
4648                         break;
4649                 }
4650                 should_throttle = false;
4651
4652                 if (found_extent &&
4653                     (test_bit(BTRFS_ROOT_REF_COWS, &root->state) ||
4654                      root == fs_info->tree_root)) {
4655                         btrfs_set_path_blocking(path);
4656                         bytes_deleted += extent_num_bytes;
4657                         ret = btrfs_free_extent(trans, root, extent_start,
4658                                                 extent_num_bytes, 0,
4659                                                 btrfs_header_owner(leaf),
4660                                                 ino, extent_offset);
4661                         BUG_ON(ret);
4662                         if (btrfs_should_throttle_delayed_refs(trans, fs_info))
4663                                 btrfs_async_run_delayed_refs(fs_info,
4664                                         trans->delayed_ref_updates * 2,
4665                                         trans->transid, 0);
4666                         if (be_nice) {
4667                                 if (truncate_space_check(trans, root,
4668                                                          extent_num_bytes)) {
4669                                         should_end = true;
4670                                 }
4671                                 if (btrfs_should_throttle_delayed_refs(trans,
4672                                                                        fs_info))
4673                                         should_throttle = true;
4674                         }
4675                 }
4676
4677                 if (found_type == BTRFS_INODE_ITEM_KEY)
4678                         break;
4679
4680                 if (path->slots[0] == 0 ||
4681                     path->slots[0] != pending_del_slot ||
4682                     should_throttle || should_end) {
4683                         if (pending_del_nr) {
4684                                 ret = btrfs_del_items(trans, root, path,
4685                                                 pending_del_slot,
4686                                                 pending_del_nr);
4687                                 if (ret) {
4688                                         btrfs_abort_transaction(trans, ret);
4689                                         goto error;
4690                                 }
4691                                 pending_del_nr = 0;
4692                         }
4693                         btrfs_release_path(path);
4694                         if (should_throttle) {
4695                                 unsigned long updates = trans->delayed_ref_updates;
4696                                 if (updates) {
4697                                         trans->delayed_ref_updates = 0;
4698                                         ret = btrfs_run_delayed_refs(trans,
4699                                                                    updates * 2);
4700                                         if (ret && !err)
4701                                                 err = ret;
4702                                 }
4703                         }
4704                         /*
4705                          * if we failed to refill our space rsv, bail out
4706                          * and let the transaction restart
4707                          */
4708                         if (should_end) {
4709                                 err = -EAGAIN;
4710                                 goto error;
4711                         }
4712                         goto search_again;
4713                 } else {
4714                         path->slots[0]--;
4715                 }
4716         }
4717 out:
4718         if (pending_del_nr) {
4719                 ret = btrfs_del_items(trans, root, path, pending_del_slot,
4720                                       pending_del_nr);
4721                 if (ret)
4722                         btrfs_abort_transaction(trans, ret);
4723         }
4724 error:
4725         if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
4726                 ASSERT(last_size >= new_size);
4727                 if (!err && last_size > new_size)
4728                         last_size = new_size;
4729                 btrfs_ordered_update_i_size(inode, last_size, NULL);
4730         }
4731
4732         btrfs_free_path(path);
4733
4734         if (be_nice && bytes_deleted > SZ_32M) {
4735                 unsigned long updates = trans->delayed_ref_updates;
4736                 if (updates) {
4737                         trans->delayed_ref_updates = 0;
4738                         ret = btrfs_run_delayed_refs(trans, updates * 2);
4739                         if (ret && !err)
4740                                 err = ret;
4741                 }
4742         }
4743         return err;
4744 }
4745
4746 /*
4747  * btrfs_truncate_block - read, zero a chunk and write a block
4748  * @inode - inode that we're zeroing
4749  * @from - the offset to start zeroing
4750  * @len - the length to zero, 0 to zero the entire range respective to the
4751  *      offset
4752  * @front - zero up to the offset instead of from the offset on
4753  *
4754  * This will find the block for the "from" offset and cow the block and zero the
4755  * part we want to zero.  This is used with truncate and hole punching.
4756  */
4757 int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
4758                         int front)
4759 {
4760         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4761         struct address_space *mapping = inode->i_mapping;
4762         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4763         struct btrfs_ordered_extent *ordered;
4764         struct extent_state *cached_state = NULL;
4765         struct extent_changeset *data_reserved = NULL;
4766         char *kaddr;
4767         u32 blocksize = fs_info->sectorsize;
4768         pgoff_t index = from >> PAGE_SHIFT;
4769         unsigned offset = from & (blocksize - 1);
4770         struct page *page;
4771         gfp_t mask = btrfs_alloc_write_mask(mapping);
4772         int ret = 0;
4773         u64 block_start;
4774         u64 block_end;
4775
4776         if (IS_ALIGNED(offset, blocksize) &&
4777             (!len || IS_ALIGNED(len, blocksize)))
4778                 goto out;
4779
4780         block_start = round_down(from, blocksize);
4781         block_end = block_start + blocksize - 1;
4782
4783         ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
4784                                            block_start, blocksize);
4785         if (ret)
4786                 goto out;
4787
4788 again:
4789         page = find_or_create_page(mapping, index, mask);
4790         if (!page) {
4791                 btrfs_delalloc_release_space(inode, data_reserved,
4792                                              block_start, blocksize, true);
4793                 btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, true);
4794                 ret = -ENOMEM;
4795                 goto out;
4796         }
4797
4798         if (!PageUptodate(page)) {
4799                 ret = btrfs_readpage(NULL, page);
4800                 lock_page(page);
4801                 if (page->mapping != mapping) {
4802                         unlock_page(page);
4803                         put_page(page);
4804                         goto again;
4805                 }
4806                 if (!PageUptodate(page)) {
4807                         ret = -EIO;
4808                         goto out_unlock;
4809                 }
4810         }
4811         wait_on_page_writeback(page);
4812
4813         lock_extent_bits(io_tree, block_start, block_end, &cached_state);
4814         set_page_extent_mapped(page);
4815
4816         ordered = btrfs_lookup_ordered_extent(inode, block_start);
4817         if (ordered) {
4818                 unlock_extent_cached(io_tree, block_start, block_end,
4819                                      &cached_state);
4820                 unlock_page(page);
4821                 put_page(page);
4822                 btrfs_start_ordered_extent(inode, ordered, 1);
4823                 btrfs_put_ordered_extent(ordered);
4824                 goto again;
4825         }
4826
4827         clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
4828                           EXTENT_DIRTY | EXTENT_DELALLOC |
4829                           EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
4830                           0, 0, &cached_state);
4831
4832         ret = btrfs_set_extent_delalloc(inode, block_start, block_end, 0,
4833                                         &cached_state, 0);
4834         if (ret) {
4835                 unlock_extent_cached(io_tree, block_start, block_end,
4836                                      &cached_state);
4837                 goto out_unlock;
4838         }
4839
4840         if (offset != blocksize) {
4841                 if (!len)
4842                         len = blocksize - offset;
4843                 kaddr = kmap(page);
4844                 if (front)
4845                         memset(kaddr + (block_start - page_offset(page)),
4846                                 0, offset);
4847                 else
4848                         memset(kaddr + (block_start - page_offset(page)) +  offset,
4849                                 0, len);
4850                 flush_dcache_page(page);
4851                 kunmap(page);
4852         }
4853         ClearPageChecked(page);
4854         set_page_dirty(page);
4855         unlock_extent_cached(io_tree, block_start, block_end, &cached_state);
4856
4857 out_unlock:
4858         if (ret)
4859                 btrfs_delalloc_release_space(inode, data_reserved, block_start,
4860                                              blocksize, true);
4861         btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize, (ret != 0));
4862         unlock_page(page);
4863         put_page(page);
4864 out:
4865         extent_changeset_free(data_reserved);
4866         return ret;
4867 }
4868
4869 static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
4870                              u64 offset, u64 len)
4871 {
4872         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4873         struct btrfs_trans_handle *trans;
4874         int ret;
4875
4876         /*
4877          * Still need to make sure the inode looks like it's been updated so
4878          * that any holes get logged if we fsync.
4879          */
4880         if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
4881                 BTRFS_I(inode)->last_trans = fs_info->generation;
4882                 BTRFS_I(inode)->last_sub_trans = root->log_transid;
4883                 BTRFS_I(inode)->last_log_commit = root->last_log_commit;
4884                 return 0;
4885         }
4886
4887         /*
4888          * 1 - for the one we're dropping
4889          * 1 - for the one we're adding
4890          * 1 - for updating the inode.
4891          */
4892         trans = btrfs_start_transaction(root, 3);
4893         if (IS_ERR(trans))
4894                 return PTR_ERR(trans);
4895
4896         ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
4897         if (ret) {
4898                 btrfs_abort_transaction(trans, ret);
4899                 btrfs_end_transaction(trans);
4900                 return ret;
4901         }
4902
4903         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
4904                         offset, 0, 0, len, 0, len, 0, 0, 0);
4905         if (ret)
4906                 btrfs_abort_transaction(trans, ret);
4907         else
4908                 btrfs_update_inode(trans, root, inode);
4909         btrfs_end_transaction(trans);
4910         return ret;
4911 }
4912
4913 /*
4914  * This function puts in dummy file extents for the area we're creating a hole
4915  * for.  So if we are truncating this file to a larger size we need to insert
4916  * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
4917  * the range between oldsize and size
4918  */
4919 int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
4920 {
4921         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
4922         struct btrfs_root *root = BTRFS_I(inode)->root;
4923         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
4924         struct extent_map *em = NULL;
4925         struct extent_state *cached_state = NULL;
4926         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
4927         u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
4928         u64 block_end = ALIGN(size, fs_info->sectorsize);
4929         u64 last_byte;
4930         u64 cur_offset;
4931         u64 hole_size;
4932         int err = 0;
4933
4934         /*
4935          * If our size started in the middle of a block we need to zero out the
4936          * rest of the block before we expand the i_size, otherwise we could
4937          * expose stale data.
4938          */
4939         err = btrfs_truncate_block(inode, oldsize, 0, 0);
4940         if (err)
4941                 return err;
4942
4943         if (size <= hole_start)
4944                 return 0;
4945
4946         while (1) {
4947                 struct btrfs_ordered_extent *ordered;
4948
4949                 lock_extent_bits(io_tree, hole_start, block_end - 1,
4950                                  &cached_state);
4951                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), hole_start,
4952                                                      block_end - hole_start);
4953                 if (!ordered)
4954                         break;
4955                 unlock_extent_cached(io_tree, hole_start, block_end - 1,
4956                                      &cached_state);
4957                 btrfs_start_ordered_extent(inode, ordered, 1);
4958                 btrfs_put_ordered_extent(ordered);
4959         }
4960
4961         cur_offset = hole_start;
4962         while (1) {
4963                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
4964                                 block_end - cur_offset, 0);
4965                 if (IS_ERR(em)) {
4966                         err = PTR_ERR(em);
4967                         em = NULL;
4968                         break;
4969                 }
4970                 last_byte = min(extent_map_end(em), block_end);
4971                 last_byte = ALIGN(last_byte, fs_info->sectorsize);
4972                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
4973                         struct extent_map *hole_em;
4974                         hole_size = last_byte - cur_offset;
4975
4976                         err = maybe_insert_hole(root, inode, cur_offset,
4977                                                 hole_size);
4978                         if (err)
4979                                 break;
4980                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
4981                                                 cur_offset + hole_size - 1, 0);
4982                         hole_em = alloc_extent_map();
4983                         if (!hole_em) {
4984                                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
4985                                         &BTRFS_I(inode)->runtime_flags);
4986                                 goto next;
4987                         }
4988                         hole_em->start = cur_offset;
4989                         hole_em->len = hole_size;
4990                         hole_em->orig_start = cur_offset;
4991
4992                         hole_em->block_start = EXTENT_MAP_HOLE;
4993                         hole_em->block_len = 0;
4994                         hole_em->orig_block_len = 0;
4995                         hole_em->ram_bytes = hole_size;
4996                         hole_em->bdev = fs_info->fs_devices->latest_bdev;
4997                         hole_em->compress_type = BTRFS_COMPRESS_NONE;
4998                         hole_em->generation = fs_info->generation;
4999
5000                         while (1) {
5001                                 write_lock(&em_tree->lock);
5002                                 err = add_extent_mapping(em_tree, hole_em, 1);
5003                                 write_unlock(&em_tree->lock);
5004                                 if (err != -EEXIST)
5005                                         break;
5006                                 btrfs_drop_extent_cache(BTRFS_I(inode),
5007                                                         cur_offset,
5008                                                         cur_offset +
5009                                                         hole_size - 1, 0);
5010                         }
5011                         free_extent_map(hole_em);
5012                 }
5013 next:
5014                 free_extent_map(em);
5015                 em = NULL;
5016                 cur_offset = last_byte;
5017                 if (cur_offset >= block_end)
5018                         break;
5019         }
5020         free_extent_map(em);
5021         unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state);
5022         return err;
5023 }
5024
5025 static int btrfs_setsize(struct inode *inode, struct iattr *attr)
5026 {
5027         struct btrfs_root *root = BTRFS_I(inode)->root;
5028         struct btrfs_trans_handle *trans;
5029         loff_t oldsize = i_size_read(inode);
5030         loff_t newsize = attr->ia_size;
5031         int mask = attr->ia_valid;
5032         int ret;
5033
5034         /*
5035          * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
5036          * special case where we need to update the times despite not having
5037          * these flags set.  For all other operations the VFS set these flags
5038          * explicitly if it wants a timestamp update.
5039          */
5040         if (newsize != oldsize) {
5041                 inode_inc_iversion(inode);
5042                 if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
5043                         inode->i_ctime = inode->i_mtime =
5044                                 current_time(inode);
5045         }
5046
5047         if (newsize > oldsize) {
5048                 /*
5049                  * Don't do an expanding truncate while snapshotting is ongoing.
5050                  * This is to ensure the snapshot captures a fully consistent
5051                  * state of this file - if the snapshot captures this expanding
5052                  * truncation, it must capture all writes that happened before
5053                  * this truncation.
5054                  */
5055                 btrfs_wait_for_snapshot_creation(root);
5056                 ret = btrfs_cont_expand(inode, oldsize, newsize);
5057                 if (ret) {
5058                         btrfs_end_write_no_snapshotting(root);
5059                         return ret;
5060                 }
5061
5062                 trans = btrfs_start_transaction(root, 1);
5063                 if (IS_ERR(trans)) {
5064                         btrfs_end_write_no_snapshotting(root);
5065                         return PTR_ERR(trans);
5066                 }
5067
5068                 i_size_write(inode, newsize);
5069                 btrfs_ordered_update_i_size(inode, i_size_read(inode), NULL);
5070                 pagecache_isize_extended(inode, oldsize, newsize);
5071                 ret = btrfs_update_inode(trans, root, inode);
5072                 btrfs_end_write_no_snapshotting(root);
5073                 btrfs_end_transaction(trans);
5074         } else {
5075
5076                 /*
5077                  * We're truncating a file that used to have good data down to
5078                  * zero. Make sure it gets into the ordered flush list so that
5079                  * any new writes get down to disk quickly.
5080                  */
5081                 if (newsize == 0)
5082                         set_bit(BTRFS_INODE_ORDERED_DATA_CLOSE,
5083                                 &BTRFS_I(inode)->runtime_flags);
5084
5085                 /*
5086                  * 1 for the orphan item we're going to add
5087                  * 1 for the orphan item deletion.
5088                  */
5089                 trans = btrfs_start_transaction(root, 2);
5090                 if (IS_ERR(trans))
5091                         return PTR_ERR(trans);
5092
5093                 /*
5094                  * We need to do this in case we fail at _any_ point during the
5095                  * actual truncate.  Once we do the truncate_setsize we could
5096                  * invalidate pages which forces any outstanding ordered io to
5097                  * be instantly completed which will give us extents that need
5098                  * to be truncated.  If we fail to get an orphan inode down we
5099                  * could have left over extents that were never meant to live,
5100                  * so we need to guarantee from this point on that everything
5101                  * will be consistent.
5102                  */
5103                 ret = btrfs_orphan_add(trans, BTRFS_I(inode));
5104                 btrfs_end_transaction(trans);
5105                 if (ret)
5106                         return ret;
5107
5108                 /* we don't support swapfiles, so vmtruncate shouldn't fail */
5109                 truncate_setsize(inode, newsize);
5110
5111                 /* Disable nonlocked read DIO to avoid the end less truncate */
5112                 btrfs_inode_block_unlocked_dio(BTRFS_I(inode));
5113                 inode_dio_wait(inode);
5114                 btrfs_inode_resume_unlocked_dio(BTRFS_I(inode));
5115
5116                 ret = btrfs_truncate(inode, newsize == oldsize);
5117                 if (ret && inode->i_nlink) {
5118                         int err;
5119
5120                         /* To get a stable disk_i_size */
5121                         err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
5122                         if (err) {
5123                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5124                                 return err;
5125                         }
5126
5127                         /*
5128                          * failed to truncate, disk_i_size is only adjusted down
5129                          * as we remove extents, so it should represent the true
5130                          * size of the inode, so reset the in memory size and
5131                          * delete our orphan entry.
5132                          */
5133                         trans = btrfs_join_transaction(root);
5134                         if (IS_ERR(trans)) {
5135                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5136                                 return ret;
5137                         }
5138                         i_size_write(inode, BTRFS_I(inode)->disk_i_size);
5139                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
5140                         if (err)
5141                                 btrfs_abort_transaction(trans, err);
5142                         btrfs_end_transaction(trans);
5143                 }
5144         }
5145
5146         return ret;
5147 }
5148
5149 static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
5150 {
5151         struct inode *inode = d_inode(dentry);
5152         struct btrfs_root *root = BTRFS_I(inode)->root;
5153         int err;
5154
5155         if (btrfs_root_readonly(root))
5156                 return -EROFS;
5157
5158         err = setattr_prepare(dentry, attr);
5159         if (err)
5160                 return err;
5161
5162         if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
5163                 err = btrfs_setsize(inode, attr);
5164                 if (err)
5165                         return err;
5166         }
5167
5168         if (attr->ia_valid) {
5169                 setattr_copy(inode, attr);
5170                 inode_inc_iversion(inode);
5171                 err = btrfs_dirty_inode(inode);
5172
5173                 if (!err && attr->ia_valid & ATTR_MODE)
5174                         err = posix_acl_chmod(inode, inode->i_mode);
5175         }
5176
5177         return err;
5178 }
5179
5180 /*
5181  * While truncating the inode pages during eviction, we get the VFS calling
5182  * btrfs_invalidatepage() against each page of the inode. This is slow because
5183  * the calls to btrfs_invalidatepage() result in a huge amount of calls to
5184  * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
5185  * extent_state structures over and over, wasting lots of time.
5186  *
5187  * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
5188  * those expensive operations on a per page basis and do only the ordered io
5189  * finishing, while we release here the extent_map and extent_state structures,
5190  * without the excessive merging and splitting.
5191  */
5192 static void evict_inode_truncate_pages(struct inode *inode)
5193 {
5194         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
5195         struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
5196         struct rb_node *node;
5197
5198         ASSERT(inode->i_state & I_FREEING);
5199         truncate_inode_pages_final(&inode->i_data);
5200
5201         write_lock(&map_tree->lock);
5202         while (!RB_EMPTY_ROOT(&map_tree->map)) {
5203                 struct extent_map *em;
5204
5205                 node = rb_first(&map_tree->map);
5206                 em = rb_entry(node, struct extent_map, rb_node);
5207                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
5208                 clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
5209                 remove_extent_mapping(map_tree, em);
5210                 free_extent_map(em);
5211                 if (need_resched()) {
5212                         write_unlock(&map_tree->lock);
5213                         cond_resched();
5214                         write_lock(&map_tree->lock);
5215                 }
5216         }
5217         write_unlock(&map_tree->lock);
5218
5219         /*
5220          * Keep looping until we have no more ranges in the io tree.
5221          * We can have ongoing bios started by readpages (called from readahead)
5222          * that have their endio callback (extent_io.c:end_bio_extent_readpage)
5223          * still in progress (unlocked the pages in the bio but did not yet
5224          * unlocked the ranges in the io tree). Therefore this means some
5225          * ranges can still be locked and eviction started because before
5226          * submitting those bios, which are executed by a separate task (work
5227          * queue kthread), inode references (inode->i_count) were not taken
5228          * (which would be dropped in the end io callback of each bio).
5229          * Therefore here we effectively end up waiting for those bios and
5230          * anyone else holding locked ranges without having bumped the inode's
5231          * reference count - if we don't do it, when they access the inode's
5232          * io_tree to unlock a range it may be too late, leading to an
5233          * use-after-free issue.
5234          */
5235         spin_lock(&io_tree->lock);
5236         while (!RB_EMPTY_ROOT(&io_tree->state)) {
5237                 struct extent_state *state;
5238                 struct extent_state *cached_state = NULL;
5239                 u64 start;
5240                 u64 end;
5241
5242                 node = rb_first(&io_tree->state);
5243                 state = rb_entry(node, struct extent_state, rb_node);
5244                 start = state->start;
5245                 end = state->end;
5246                 spin_unlock(&io_tree->lock);
5247
5248                 lock_extent_bits(io_tree, start, end, &cached_state);
5249
5250                 /*
5251                  * If still has DELALLOC flag, the extent didn't reach disk,
5252                  * and its reserved space won't be freed by delayed_ref.
5253                  * So we need to free its reserved space here.
5254                  * (Refer to comment in btrfs_invalidatepage, case 2)
5255                  *
5256                  * Note, end is the bytenr of last byte, so we need + 1 here.
5257                  */
5258                 if (state->state & EXTENT_DELALLOC)
5259                         btrfs_qgroup_free_data(inode, NULL, start, end - start + 1);
5260
5261                 clear_extent_bit(io_tree, start, end,
5262                                  EXTENT_LOCKED | EXTENT_DIRTY |
5263                                  EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
5264                                  EXTENT_DEFRAG, 1, 1, &cached_state);
5265
5266                 cond_resched();
5267                 spin_lock(&io_tree->lock);
5268         }
5269         spin_unlock(&io_tree->lock);
5270 }
5271
5272 void btrfs_evict_inode(struct inode *inode)
5273 {
5274         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5275         struct btrfs_trans_handle *trans;
5276         struct btrfs_root *root = BTRFS_I(inode)->root;
5277         struct btrfs_block_rsv *rsv, *global_rsv;
5278         int steal_from_global = 0;
5279         u64 min_size;
5280         int ret;
5281
5282         trace_btrfs_inode_evict(inode);
5283
5284         if (!root) {
5285                 clear_inode(inode);
5286                 return;
5287         }
5288
5289         min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
5290
5291         evict_inode_truncate_pages(inode);
5292
5293         if (inode->i_nlink &&
5294             ((btrfs_root_refs(&root->root_item) != 0 &&
5295               root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
5296              btrfs_is_free_space_inode(BTRFS_I(inode))))
5297                 goto no_delete;
5298
5299         if (is_bad_inode(inode)) {
5300                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5301                 goto no_delete;
5302         }
5303         /* do we really want it for ->i_nlink > 0 and zero btrfs_root_refs? */
5304         if (!special_file(inode->i_mode))
5305                 btrfs_wait_ordered_range(inode, 0, (u64)-1);
5306
5307         btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
5308
5309         if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
5310                 BUG_ON(test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
5311                                  &BTRFS_I(inode)->runtime_flags));
5312                 goto no_delete;
5313         }
5314
5315         if (inode->i_nlink > 0) {
5316                 BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
5317                        root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
5318                 goto no_delete;
5319         }
5320
5321         ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
5322         if (ret) {
5323                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5324                 goto no_delete;
5325         }
5326
5327         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
5328         if (!rsv) {
5329                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5330                 goto no_delete;
5331         }
5332         rsv->size = min_size;
5333         rsv->failfast = 1;
5334         global_rsv = &fs_info->global_block_rsv;
5335
5336         btrfs_i_size_write(BTRFS_I(inode), 0);
5337
5338         /*
5339          * This is a bit simpler than btrfs_truncate since we've already
5340          * reserved our space for our orphan item in the unlink, so we just
5341          * need to reserve some slack space in case we add bytes and update
5342          * inode item when doing the truncate.
5343          */
5344         while (1) {
5345                 ret = btrfs_block_rsv_refill(root, rsv, min_size,
5346                                              BTRFS_RESERVE_FLUSH_LIMIT);
5347
5348                 /*
5349                  * Try and steal from the global reserve since we will
5350                  * likely not use this space anyway, we want to try as
5351                  * hard as possible to get this to work.
5352                  */
5353                 if (ret)
5354                         steal_from_global++;
5355                 else
5356                         steal_from_global = 0;
5357                 ret = 0;
5358
5359                 /*
5360                  * steal_from_global == 0: we reserved stuff, hooray!
5361                  * steal_from_global == 1: we didn't reserve stuff, boo!
5362                  * steal_from_global == 2: we've committed, still not a lot of
5363                  * room but maybe we'll have room in the global reserve this
5364                  * time.
5365                  * steal_from_global == 3: abandon all hope!
5366                  */
5367                 if (steal_from_global > 2) {
5368                         btrfs_warn(fs_info,
5369                                    "Could not get space for a delete, will truncate on mount %d",
5370                                    ret);
5371                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5372                         btrfs_free_block_rsv(fs_info, rsv);
5373                         goto no_delete;
5374                 }
5375
5376                 trans = btrfs_join_transaction(root);
5377                 if (IS_ERR(trans)) {
5378                         btrfs_orphan_del(NULL, BTRFS_I(inode));
5379                         btrfs_free_block_rsv(fs_info, rsv);
5380                         goto no_delete;
5381                 }
5382
5383                 /*
5384                  * We can't just steal from the global reserve, we need to make
5385                  * sure there is room to do it, if not we need to commit and try
5386                  * again.
5387                  */
5388                 if (steal_from_global) {
5389                         if (!btrfs_check_space_for_delayed_refs(trans, fs_info))
5390                                 ret = btrfs_block_rsv_migrate(global_rsv, rsv,
5391                                                               min_size, 0);
5392                         else
5393                                 ret = -ENOSPC;
5394                 }
5395
5396                 /*
5397                  * Couldn't steal from the global reserve, we have too much
5398                  * pending stuff built up, commit the transaction and try it
5399                  * again.
5400                  */
5401                 if (ret) {
5402                         ret = btrfs_commit_transaction(trans);
5403                         if (ret) {
5404                                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5405                                 btrfs_free_block_rsv(fs_info, rsv);
5406                                 goto no_delete;
5407                         }
5408                         continue;
5409                 } else {
5410                         steal_from_global = 0;
5411                 }
5412
5413                 trans->block_rsv = rsv;
5414
5415                 ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
5416                 if (ret != -ENOSPC && ret != -EAGAIN)
5417                         break;
5418
5419                 trans->block_rsv = &fs_info->trans_block_rsv;
5420                 btrfs_end_transaction(trans);
5421                 trans = NULL;
5422                 btrfs_btree_balance_dirty(fs_info);
5423         }
5424
5425         btrfs_free_block_rsv(fs_info, rsv);
5426
5427         /*
5428          * Errors here aren't a big deal, it just means we leave orphan items
5429          * in the tree.  They will be cleaned up on the next mount.
5430          */
5431         if (ret == 0) {
5432                 trans->block_rsv = root->orphan_block_rsv;
5433                 btrfs_orphan_del(trans, BTRFS_I(inode));
5434         } else {
5435                 btrfs_orphan_del(NULL, BTRFS_I(inode));
5436         }
5437
5438         trans->block_rsv = &fs_info->trans_block_rsv;
5439         if (!(root == fs_info->tree_root ||
5440               root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
5441                 btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
5442
5443         btrfs_end_transaction(trans);
5444         btrfs_btree_balance_dirty(fs_info);
5445 no_delete:
5446         btrfs_remove_delayed_node(BTRFS_I(inode));
5447         clear_inode(inode);
5448 }
5449
5450 /*
5451  * this returns the key found in the dir entry in the location pointer.
5452  * If no dir entries were found, returns -ENOENT.
5453  * If found a corrupted location in dir entry, returns -EUCLEAN.
5454  */
5455 static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
5456                                struct btrfs_key *location)
5457 {
5458         const char *name = dentry->d_name.name;
5459         int namelen = dentry->d_name.len;
5460         struct btrfs_dir_item *di;
5461         struct btrfs_path *path;
5462         struct btrfs_root *root = BTRFS_I(dir)->root;
5463         int ret = 0;
5464
5465         path = btrfs_alloc_path();
5466         if (!path)
5467                 return -ENOMEM;
5468
5469         di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
5470                         name, namelen, 0);
5471         if (!di) {
5472                 ret = -ENOENT;
5473                 goto out;
5474         }
5475         if (IS_ERR(di)) {
5476                 ret = PTR_ERR(di);
5477                 goto out;
5478         }
5479
5480         btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
5481         if (location->type != BTRFS_INODE_ITEM_KEY &&
5482             location->type != BTRFS_ROOT_ITEM_KEY) {
5483                 ret = -EUCLEAN;
5484                 btrfs_warn(root->fs_info,
5485 "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
5486                            __func__, name, btrfs_ino(BTRFS_I(dir)),
5487                            location->objectid, location->type, location->offset);
5488         }
5489 out:
5490         btrfs_free_path(path);
5491         return ret;
5492 }
5493
5494 /*
5495  * when we hit a tree root in a directory, the btrfs part of the inode
5496  * needs to be changed to reflect the root directory of the tree root.  This
5497  * is kind of like crossing a mount point.
5498  */
5499 static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
5500                                     struct inode *dir,
5501                                     struct dentry *dentry,
5502                                     struct btrfs_key *location,
5503                                     struct btrfs_root **sub_root)
5504 {
5505         struct btrfs_path *path;
5506         struct btrfs_root *new_root;
5507         struct btrfs_root_ref *ref;
5508         struct extent_buffer *leaf;
5509         struct btrfs_key key;
5510         int ret;
5511         int err = 0;
5512
5513         path = btrfs_alloc_path();
5514         if (!path) {
5515                 err = -ENOMEM;
5516                 goto out;
5517         }
5518
5519         err = -ENOENT;
5520         key.objectid = BTRFS_I(dir)->root->root_key.objectid;
5521         key.type = BTRFS_ROOT_REF_KEY;
5522         key.offset = location->objectid;
5523
5524         ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
5525         if (ret) {
5526                 if (ret < 0)
5527                         err = ret;
5528                 goto out;
5529         }
5530
5531         leaf = path->nodes[0];
5532         ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
5533         if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
5534             btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
5535                 goto out;
5536
5537         ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
5538                                    (unsigned long)(ref + 1),
5539                                    dentry->d_name.len);
5540         if (ret)
5541                 goto out;
5542
5543         btrfs_release_path(path);
5544
5545         new_root = btrfs_read_fs_root_no_name(fs_info, location);
5546         if (IS_ERR(new_root)) {
5547                 err = PTR_ERR(new_root);
5548                 goto out;
5549         }
5550
5551         *sub_root = new_root;
5552         location->objectid = btrfs_root_dirid(&new_root->root_item);
5553         location->type = BTRFS_INODE_ITEM_KEY;
5554         location->offset = 0;
5555         err = 0;
5556 out:
5557         btrfs_free_path(path);
5558         return err;
5559 }
5560
5561 static void inode_tree_add(struct inode *inode)
5562 {
5563         struct btrfs_root *root = BTRFS_I(inode)->root;
5564         struct btrfs_inode *entry;
5565         struct rb_node **p;
5566         struct rb_node *parent;
5567         struct rb_node *new = &BTRFS_I(inode)->rb_node;
5568         u64 ino = btrfs_ino(BTRFS_I(inode));
5569
5570         if (inode_unhashed(inode))
5571                 return;
5572         parent = NULL;
5573         spin_lock(&root->inode_lock);
5574         p = &root->inode_tree.rb_node;
5575         while (*p) {
5576                 parent = *p;
5577                 entry = rb_entry(parent, struct btrfs_inode, rb_node);
5578
5579                 if (ino < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5580                         p = &parent->rb_left;
5581                 else if (ino > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5582                         p = &parent->rb_right;
5583                 else {
5584                         WARN_ON(!(entry->vfs_inode.i_state &
5585                                   (I_WILL_FREE | I_FREEING)));
5586                         rb_replace_node(parent, new, &root->inode_tree);
5587                         RB_CLEAR_NODE(parent);
5588                         spin_unlock(&root->inode_lock);
5589                         return;
5590                 }
5591         }
5592         rb_link_node(new, parent, p);
5593         rb_insert_color(new, &root->inode_tree);
5594         spin_unlock(&root->inode_lock);
5595 }
5596
5597 static void inode_tree_del(struct inode *inode)
5598 {
5599         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
5600         struct btrfs_root *root = BTRFS_I(inode)->root;
5601         int empty = 0;
5602
5603         spin_lock(&root->inode_lock);
5604         if (!RB_EMPTY_NODE(&BTRFS_I(inode)->rb_node)) {
5605                 rb_erase(&BTRFS_I(inode)->rb_node, &root->inode_tree);
5606                 RB_CLEAR_NODE(&BTRFS_I(inode)->rb_node);
5607                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5608         }
5609         spin_unlock(&root->inode_lock);
5610
5611         if (empty && btrfs_root_refs(&root->root_item) == 0) {
5612                 synchronize_srcu(&fs_info->subvol_srcu);
5613                 spin_lock(&root->inode_lock);
5614                 empty = RB_EMPTY_ROOT(&root->inode_tree);
5615                 spin_unlock(&root->inode_lock);
5616                 if (empty)
5617                         btrfs_add_dead_root(root);
5618         }
5619 }
5620
5621 void btrfs_invalidate_inodes(struct btrfs_root *root)
5622 {
5623         struct btrfs_fs_info *fs_info = root->fs_info;
5624         struct rb_node *node;
5625         struct rb_node *prev;
5626         struct btrfs_inode *entry;
5627         struct inode *inode;
5628         u64 objectid = 0;
5629
5630         if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
5631                 WARN_ON(btrfs_root_refs(&root->root_item) != 0);
5632
5633         spin_lock(&root->inode_lock);
5634 again:
5635         node = root->inode_tree.rb_node;
5636         prev = NULL;
5637         while (node) {
5638                 prev = node;
5639                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5640
5641                 if (objectid < btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5642                         node = node->rb_left;
5643                 else if (objectid > btrfs_ino(BTRFS_I(&entry->vfs_inode)))
5644                         node = node->rb_right;
5645                 else
5646                         break;
5647         }
5648         if (!node) {
5649                 while (prev) {
5650                         entry = rb_entry(prev, struct btrfs_inode, rb_node);
5651                         if (objectid <= btrfs_ino(BTRFS_I(&entry->vfs_inode))) {
5652                                 node = prev;
5653                                 break;
5654                         }
5655                         prev = rb_next(prev);
5656                 }
5657         }
5658         while (node) {
5659                 entry = rb_entry(node, struct btrfs_inode, rb_node);
5660                 objectid = btrfs_ino(BTRFS_I(&entry->vfs_inode)) + 1;
5661                 inode = igrab(&entry->vfs_inode);
5662                 if (inode) {
5663                         spin_unlock(&root->inode_lock);
5664                         if (atomic_read(&inode->i_count) > 1)
5665                                 d_prune_aliases(inode);
5666                         /*
5667                          * btrfs_drop_inode will have it removed from
5668                          * the inode cache when its usage count
5669                          * hits zero.
5670                          */
5671                         iput(inode);
5672                         cond_resched();
5673                         spin_lock(&root->inode_lock);
5674                         goto again;
5675                 }
5676
5677                 if (cond_resched_lock(&root->inode_lock))
5678                         goto again;
5679
5680                 node = rb_next(node);
5681         }
5682         spin_unlock(&root->inode_lock);
5683 }
5684
5685 static int btrfs_init_locked_inode(struct inode *inode, void *p)
5686 {
5687         struct btrfs_iget_args *args = p;
5688         inode->i_ino = args->location->objectid;
5689         memcpy(&BTRFS_I(inode)->location, args->location,
5690                sizeof(*args->location));
5691         BTRFS_I(inode)->root = args->root;
5692         return 0;
5693 }
5694
5695 static int btrfs_find_actor(struct inode *inode, void *opaque)
5696 {
5697         struct btrfs_iget_args *args = opaque;
5698         return args->location->objectid == BTRFS_I(inode)->location.objectid &&
5699                 args->root == BTRFS_I(inode)->root;
5700 }
5701
5702 static struct inode *btrfs_iget_locked(struct super_block *s,
5703                                        struct btrfs_key *location,
5704                                        struct btrfs_root *root)
5705 {
5706         struct inode *inode;
5707         struct btrfs_iget_args args;
5708         unsigned long hashval = btrfs_inode_hash(location->objectid, root);
5709
5710         args.location = location;
5711         args.root = root;
5712
5713         inode = iget5_locked(s, hashval, btrfs_find_actor,
5714                              btrfs_init_locked_inode,
5715                              (void *)&args);
5716         return inode;
5717 }
5718
5719 /* Get an inode object given its location and corresponding root.
5720  * Returns in *is_new if the inode was read from disk
5721  */
5722 struct inode *btrfs_iget(struct super_block *s, struct btrfs_key *location,
5723                          struct btrfs_root *root, int *new)
5724 {
5725         struct inode *inode;
5726
5727         inode = btrfs_iget_locked(s, location, root);
5728         if (!inode)
5729                 return ERR_PTR(-ENOMEM);
5730
5731         if (inode->i_state & I_NEW) {
5732                 int ret;
5733
5734                 ret = btrfs_read_locked_inode(inode);
5735                 if (!is_bad_inode(inode)) {
5736                         inode_tree_add(inode);
5737                         unlock_new_inode(inode);
5738                         if (new)
5739                                 *new = 1;
5740                 } else {
5741                         unlock_new_inode(inode);
5742                         iput(inode);
5743                         ASSERT(ret < 0);
5744                         inode = ERR_PTR(ret < 0 ? ret : -ESTALE);
5745                 }
5746         }
5747
5748         return inode;
5749 }
5750
5751 static struct inode *new_simple_dir(struct super_block *s,
5752                                     struct btrfs_key *key,
5753                                     struct btrfs_root *root)
5754 {
5755         struct inode *inode = new_inode(s);
5756
5757         if (!inode)
5758                 return ERR_PTR(-ENOMEM);
5759
5760         BTRFS_I(inode)->root = root;
5761         memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
5762         set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
5763
5764         inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
5765         inode->i_op = &btrfs_dir_ro_inode_operations;
5766         inode->i_opflags &= ~IOP_XATTR;
5767         inode->i_fop = &simple_dir_operations;
5768         inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
5769         inode->i_mtime = current_time(inode);
5770         inode->i_atime = inode->i_mtime;
5771         inode->i_ctime = inode->i_mtime;
5772         BTRFS_I(inode)->i_otime = inode->i_mtime;
5773
5774         return inode;
5775 }
5776
5777 struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
5778 {
5779         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
5780         struct inode *inode;
5781         struct btrfs_root *root = BTRFS_I(dir)->root;
5782         struct btrfs_root *sub_root = root;
5783         struct btrfs_key location;
5784         int index;
5785         int ret = 0;
5786
5787         if (dentry->d_name.len > BTRFS_NAME_LEN)
5788                 return ERR_PTR(-ENAMETOOLONG);
5789
5790         ret = btrfs_inode_by_name(dir, dentry, &location);
5791         if (ret < 0)
5792                 return ERR_PTR(ret);
5793
5794         if (location.type == BTRFS_INODE_ITEM_KEY) {
5795                 inode = btrfs_iget(dir->i_sb, &location, root, NULL);
5796                 return inode;
5797         }
5798
5799         index = srcu_read_lock(&fs_info->subvol_srcu);
5800         ret = fixup_tree_root_location(fs_info, dir, dentry,
5801                                        &location, &sub_root);
5802         if (ret < 0) {
5803                 if (ret != -ENOENT)
5804                         inode = ERR_PTR(ret);
5805                 else
5806                         inode = new_simple_dir(dir->i_sb, &location, sub_root);
5807         } else {
5808                 inode = btrfs_iget(dir->i_sb, &location, sub_root, NULL);
5809         }
5810         srcu_read_unlock(&fs_info->subvol_srcu, index);
5811
5812         if (!IS_ERR(inode) && root != sub_root) {
5813                 down_read(&fs_info->cleanup_work_sem);
5814                 if (!sb_rdonly(inode->i_sb))
5815                         ret = btrfs_orphan_cleanup(sub_root);
5816                 up_read(&fs_info->cleanup_work_sem);
5817                 if (ret) {
5818                         iput(inode);
5819                         inode = ERR_PTR(ret);
5820                 }
5821         }
5822
5823         return inode;
5824 }
5825
5826 static int btrfs_dentry_delete(const struct dentry *dentry)
5827 {
5828         struct btrfs_root *root;
5829         struct inode *inode = d_inode(dentry);
5830
5831         if (!inode && !IS_ROOT(dentry))
5832                 inode = d_inode(dentry->d_parent);
5833
5834         if (inode) {
5835                 root = BTRFS_I(inode)->root;
5836                 if (btrfs_root_refs(&root->root_item) == 0)
5837                         return 1;
5838
5839                 if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
5840                         return 1;
5841         }
5842         return 0;
5843 }
5844
5845 static void btrfs_dentry_release(struct dentry *dentry)
5846 {
5847         kfree(dentry->d_fsdata);
5848 }
5849
5850 static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
5851                                    unsigned int flags)
5852 {
5853         struct inode *inode;
5854
5855         inode = btrfs_lookup_dentry(dir, dentry);
5856         if (IS_ERR(inode)) {
5857                 if (PTR_ERR(inode) == -ENOENT)
5858                         inode = NULL;
5859                 else
5860                         return ERR_CAST(inode);
5861         }
5862
5863         return d_splice_alias(inode, dentry);
5864 }
5865
5866 unsigned char btrfs_filetype_table[] = {
5867         DT_UNKNOWN, DT_REG, DT_DIR, DT_CHR, DT_BLK, DT_FIFO, DT_SOCK, DT_LNK
5868 };
5869
5870 /*
5871  * All this infrastructure exists because dir_emit can fault, and we are holding
5872  * the tree lock when doing readdir.  For now just allocate a buffer and copy
5873  * our information into that, and then dir_emit from the buffer.  This is
5874  * similar to what NFS does, only we don't keep the buffer around in pagecache
5875  * because I'm afraid I'll mess that up.  Long term we need to make filldir do
5876  * copy_to_user_inatomic so we don't have to worry about page faulting under the
5877  * tree lock.
5878  */
5879 static int btrfs_opendir(struct inode *inode, struct file *file)
5880 {
5881         struct btrfs_file_private *private;
5882
5883         private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
5884         if (!private)
5885                 return -ENOMEM;
5886         private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
5887         if (!private->filldir_buf) {
5888                 kfree(private);
5889                 return -ENOMEM;
5890         }
5891         file->private_data = private;
5892         return 0;
5893 }
5894
5895 struct dir_entry {
5896         u64 ino;
5897         u64 offset;
5898         unsigned type;
5899         int name_len;
5900 };
5901
5902 static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
5903 {
5904         while (entries--) {
5905                 struct dir_entry *entry = addr;
5906                 char *name = (char *)(entry + 1);
5907
5908                 ctx->pos = entry->offset;
5909                 if (!dir_emit(ctx, name, entry->name_len, entry->ino,
5910                               entry->type))
5911                         return 1;
5912                 addr += sizeof(struct dir_entry) + entry->name_len;
5913                 ctx->pos++;
5914         }
5915         return 0;
5916 }
5917
5918 static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
5919 {
5920         struct inode *inode = file_inode(file);
5921         struct btrfs_root *root = BTRFS_I(inode)->root;
5922         struct btrfs_file_private *private = file->private_data;
5923         struct btrfs_dir_item *di;
5924         struct btrfs_key key;
5925         struct btrfs_key found_key;
5926         struct btrfs_path *path;
5927         void *addr;
5928         struct list_head ins_list;
5929         struct list_head del_list;
5930         int ret;
5931         struct extent_buffer *leaf;
5932         int slot;
5933         char *name_ptr;
5934         int name_len;
5935         int entries = 0;
5936         int total_len = 0;
5937         bool put = false;
5938         struct btrfs_key location;
5939
5940         if (!dir_emit_dots(file, ctx))
5941                 return 0;
5942
5943         path = btrfs_alloc_path();
5944         if (!path)
5945                 return -ENOMEM;
5946
5947         addr = private->filldir_buf;
5948         path->reada = READA_FORWARD;
5949
5950         INIT_LIST_HEAD(&ins_list);
5951         INIT_LIST_HEAD(&del_list);
5952         put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
5953
5954 again:
5955         key.type = BTRFS_DIR_INDEX_KEY;
5956         key.offset = ctx->pos;
5957         key.objectid = btrfs_ino(BTRFS_I(inode));
5958
5959         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
5960         if (ret < 0)
5961                 goto err;
5962
5963         while (1) {
5964                 struct dir_entry *entry;
5965
5966                 leaf = path->nodes[0];
5967                 slot = path->slots[0];
5968                 if (slot >= btrfs_header_nritems(leaf)) {
5969                         ret = btrfs_next_leaf(root, path);
5970                         if (ret < 0)
5971                                 goto err;
5972                         else if (ret > 0)
5973                                 break;
5974                         continue;
5975                 }
5976
5977                 btrfs_item_key_to_cpu(leaf, &found_key, slot);
5978
5979                 if (found_key.objectid != key.objectid)
5980                         break;
5981                 if (found_key.type != BTRFS_DIR_INDEX_KEY)
5982                         break;
5983                 if (found_key.offset < ctx->pos)
5984                         goto next;
5985                 if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
5986                         goto next;
5987                 di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
5988                 name_len = btrfs_dir_name_len(leaf, di);
5989                 if ((total_len + sizeof(struct dir_entry) + name_len) >=
5990                     PAGE_SIZE) {
5991                         btrfs_release_path(path);
5992                         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
5993                         if (ret)
5994                                 goto nopos;
5995                         addr = private->filldir_buf;
5996                         entries = 0;
5997                         total_len = 0;
5998                         goto again;
5999                 }
6000
6001                 entry = addr;
6002                 entry->name_len = name_len;
6003                 name_ptr = (char *)(entry + 1);
6004                 read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
6005                                    name_len);
6006                 entry->type = btrfs_filetype_table[btrfs_dir_type(leaf, di)];
6007                 btrfs_dir_item_key_to_cpu(leaf, di, &location);
6008                 entry->ino = location.objectid;
6009                 entry->offset = found_key.offset;
6010                 entries++;
6011                 addr += sizeof(struct dir_entry) + name_len;
6012                 total_len += sizeof(struct dir_entry) + name_len;
6013 next:
6014                 path->slots[0]++;
6015         }
6016         btrfs_release_path(path);
6017
6018         ret = btrfs_filldir(private->filldir_buf, entries, ctx);
6019         if (ret)
6020                 goto nopos;
6021
6022         ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
6023         if (ret)
6024                 goto nopos;
6025
6026         /*
6027          * Stop new entries from being returned after we return the last
6028          * entry.
6029          *
6030          * New directory entries are assigned a strictly increasing
6031          * offset.  This means that new entries created during readdir
6032          * are *guaranteed* to be seen in the future by that readdir.
6033          * This has broken buggy programs which operate on names as
6034          * they're returned by readdir.  Until we re-use freed offsets
6035          * we have this hack to stop new entries from being returned
6036          * under the assumption that they'll never reach this huge
6037          * offset.
6038          *
6039          * This is being careful not to overflow 32bit loff_t unless the
6040          * last entry requires it because doing so has broken 32bit apps
6041          * in the past.
6042          */
6043         if (ctx->pos >= INT_MAX)
6044                 ctx->pos = LLONG_MAX;
6045         else
6046                 ctx->pos = INT_MAX;
6047 nopos:
6048         ret = 0;
6049 err:
6050         if (put)
6051                 btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
6052         btrfs_free_path(path);
6053         return ret;
6054 }
6055
6056 int btrfs_write_inode(struct inode *inode, struct writeback_control *wbc)
6057 {
6058         struct btrfs_root *root = BTRFS_I(inode)->root;
6059         struct btrfs_trans_handle *trans;
6060         int ret = 0;
6061         bool nolock = false;
6062
6063         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6064                 return 0;
6065
6066         if (btrfs_fs_closing(root->fs_info) &&
6067                         btrfs_is_free_space_inode(BTRFS_I(inode)))
6068                 nolock = true;
6069
6070         if (wbc->sync_mode == WB_SYNC_ALL) {
6071                 if (nolock)
6072                         trans = btrfs_join_transaction_nolock(root);
6073                 else
6074                         trans = btrfs_join_transaction(root);
6075                 if (IS_ERR(trans))
6076                         return PTR_ERR(trans);
6077                 ret = btrfs_commit_transaction(trans);
6078         }
6079         return ret;
6080 }
6081
6082 /*
6083  * This is somewhat expensive, updating the tree every time the
6084  * inode changes.  But, it is most likely to find the inode in cache.
6085  * FIXME, needs more benchmarking...there are no reasons other than performance
6086  * to keep or drop this code.
6087  */
6088 static int btrfs_dirty_inode(struct inode *inode)
6089 {
6090         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6091         struct btrfs_root *root = BTRFS_I(inode)->root;
6092         struct btrfs_trans_handle *trans;
6093         int ret;
6094
6095         if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
6096                 return 0;
6097
6098         trans = btrfs_join_transaction(root);
6099         if (IS_ERR(trans))
6100                 return PTR_ERR(trans);
6101
6102         ret = btrfs_update_inode(trans, root, inode);
6103         if (ret && ret == -ENOSPC) {
6104                 /* whoops, lets try again with the full transaction */
6105                 btrfs_end_transaction(trans);
6106                 trans = btrfs_start_transaction(root, 1);
6107                 if (IS_ERR(trans))
6108                         return PTR_ERR(trans);
6109
6110                 ret = btrfs_update_inode(trans, root, inode);
6111         }
6112         btrfs_end_transaction(trans);
6113         if (BTRFS_I(inode)->delayed_node)
6114                 btrfs_balance_delayed_items(fs_info);
6115
6116         return ret;
6117 }
6118
6119 /*
6120  * This is a copy of file_update_time.  We need this so we can return error on
6121  * ENOSPC for updating the inode in the case of file write and mmap writes.
6122  */
6123 static int btrfs_update_time(struct inode *inode, struct timespec *now,
6124                              int flags)
6125 {
6126         struct btrfs_root *root = BTRFS_I(inode)->root;
6127         bool dirty = flags & ~S_VERSION;
6128
6129         if (btrfs_root_readonly(root))
6130                 return -EROFS;
6131
6132         if (flags & S_VERSION)
6133                 dirty |= inode_maybe_inc_iversion(inode, dirty);
6134         if (flags & S_CTIME)
6135                 inode->i_ctime = *now;
6136         if (flags & S_MTIME)
6137                 inode->i_mtime = *now;
6138         if (flags & S_ATIME)
6139                 inode->i_atime = *now;
6140         return dirty ? btrfs_dirty_inode(inode) : 0;
6141 }
6142
6143 /*
6144  * find the highest existing sequence number in a directory
6145  * and then set the in-memory index_cnt variable to reflect
6146  * free sequence numbers
6147  */
6148 static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
6149 {
6150         struct btrfs_root *root = inode->root;
6151         struct btrfs_key key, found_key;
6152         struct btrfs_path *path;
6153         struct extent_buffer *leaf;
6154         int ret;
6155
6156         key.objectid = btrfs_ino(inode);
6157         key.type = BTRFS_DIR_INDEX_KEY;
6158         key.offset = (u64)-1;
6159
6160         path = btrfs_alloc_path();
6161         if (!path)
6162                 return -ENOMEM;
6163
6164         ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
6165         if (ret < 0)
6166                 goto out;
6167         /* FIXME: we should be able to handle this */
6168         if (ret == 0)
6169                 goto out;
6170         ret = 0;
6171
6172         /*
6173          * MAGIC NUMBER EXPLANATION:
6174          * since we search a directory based on f_pos we have to start at 2
6175          * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
6176          * else has to start at 2
6177          */
6178         if (path->slots[0] == 0) {
6179                 inode->index_cnt = 2;
6180                 goto out;
6181         }
6182
6183         path->slots[0]--;
6184
6185         leaf = path->nodes[0];
6186         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6187
6188         if (found_key.objectid != btrfs_ino(inode) ||
6189             found_key.type != BTRFS_DIR_INDEX_KEY) {
6190                 inode->index_cnt = 2;
6191                 goto out;
6192         }
6193
6194         inode->index_cnt = found_key.offset + 1;
6195 out:
6196         btrfs_free_path(path);
6197         return ret;
6198 }
6199
6200 /*
6201  * helper to find a free sequence number in a given directory.  This current
6202  * code is very simple, later versions will do smarter things in the btree
6203  */
6204 int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
6205 {
6206         int ret = 0;
6207
6208         if (dir->index_cnt == (u64)-1) {
6209                 ret = btrfs_inode_delayed_dir_index_count(dir);
6210                 if (ret) {
6211                         ret = btrfs_set_inode_index_count(dir);
6212                         if (ret)
6213                                 return ret;
6214                 }
6215         }
6216
6217         *index = dir->index_cnt;
6218         dir->index_cnt++;
6219
6220         return ret;
6221 }
6222
6223 static int btrfs_insert_inode_locked(struct inode *inode)
6224 {
6225         struct btrfs_iget_args args;
6226         args.location = &BTRFS_I(inode)->location;
6227         args.root = BTRFS_I(inode)->root;
6228
6229         return insert_inode_locked4(inode,
6230                    btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
6231                    btrfs_find_actor, &args);
6232 }
6233
6234 /*
6235  * Inherit flags from the parent inode.
6236  *
6237  * Currently only the compression flags and the cow flags are inherited.
6238  */
6239 static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
6240 {
6241         unsigned int flags;
6242
6243         if (!dir)
6244                 return;
6245
6246         flags = BTRFS_I(dir)->flags;
6247
6248         if (flags & BTRFS_INODE_NOCOMPRESS) {
6249                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
6250                 BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
6251         } else if (flags & BTRFS_INODE_COMPRESS) {
6252                 BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
6253                 BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
6254         }
6255
6256         if (flags & BTRFS_INODE_NODATACOW) {
6257                 BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
6258                 if (S_ISREG(inode->i_mode))
6259                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6260         }
6261
6262         btrfs_update_iflags(inode);
6263 }
6264
6265 static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
6266                                      struct btrfs_root *root,
6267                                      struct inode *dir,
6268                                      const char *name, int name_len,
6269                                      u64 ref_objectid, u64 objectid,
6270                                      umode_t mode, u64 *index)
6271 {
6272         struct btrfs_fs_info *fs_info = root->fs_info;
6273         struct inode *inode;
6274         struct btrfs_inode_item *inode_item;
6275         struct btrfs_key *location;
6276         struct btrfs_path *path;
6277         struct btrfs_inode_ref *ref;
6278         struct btrfs_key key[2];
6279         u32 sizes[2];
6280         int nitems = name ? 2 : 1;
6281         unsigned long ptr;
6282         int ret;
6283
6284         path = btrfs_alloc_path();
6285         if (!path)
6286                 return ERR_PTR(-ENOMEM);
6287
6288         inode = new_inode(fs_info->sb);
6289         if (!inode) {
6290                 btrfs_free_path(path);
6291                 return ERR_PTR(-ENOMEM);
6292         }
6293
6294         /*
6295          * O_TMPFILE, set link count to 0, so that after this point,
6296          * we fill in an inode item with the correct link count.
6297          */
6298         if (!name)
6299                 set_nlink(inode, 0);
6300
6301         /*
6302          * we have to initialize this early, so we can reclaim the inode
6303          * number if we fail afterwards in this function.
6304          */
6305         inode->i_ino = objectid;
6306
6307         if (dir && name) {
6308                 trace_btrfs_inode_request(dir);
6309
6310                 ret = btrfs_set_inode_index(BTRFS_I(dir), index);
6311                 if (ret) {
6312                         btrfs_free_path(path);
6313                         iput(inode);
6314                         return ERR_PTR(ret);
6315                 }
6316         } else if (dir) {
6317                 *index = 0;
6318         }
6319         /*
6320          * index_cnt is ignored for everything but a dir,
6321          * btrfs_set_inode_index_count has an explanation for the magic
6322          * number
6323          */
6324         BTRFS_I(inode)->index_cnt = 2;
6325         BTRFS_I(inode)->dir_index = *index;
6326         BTRFS_I(inode)->root = root;
6327         BTRFS_I(inode)->generation = trans->transid;
6328         inode->i_generation = BTRFS_I(inode)->generation;
6329
6330         /*
6331          * We could have gotten an inode number from somebody who was fsynced
6332          * and then removed in this same transaction, so let's just set full
6333          * sync since it will be a full sync anyway and this will blow away the
6334          * old info in the log.
6335          */
6336         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
6337
6338         key[0].objectid = objectid;
6339         key[0].type = BTRFS_INODE_ITEM_KEY;
6340         key[0].offset = 0;
6341
6342         sizes[0] = sizeof(struct btrfs_inode_item);
6343
6344         if (name) {
6345                 /*
6346                  * Start new inodes with an inode_ref. This is slightly more
6347                  * efficient for small numbers of hard links since they will
6348                  * be packed into one item. Extended refs will kick in if we
6349                  * add more hard links than can fit in the ref item.
6350                  */
6351                 key[1].objectid = objectid;
6352                 key[1].type = BTRFS_INODE_REF_KEY;
6353                 key[1].offset = ref_objectid;
6354
6355                 sizes[1] = name_len + sizeof(*ref);
6356         }
6357
6358         location = &BTRFS_I(inode)->location;
6359         location->objectid = objectid;
6360         location->offset = 0;
6361         location->type = BTRFS_INODE_ITEM_KEY;
6362
6363         ret = btrfs_insert_inode_locked(inode);
6364         if (ret < 0)
6365                 goto fail;
6366
6367         path->leave_spinning = 1;
6368         ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
6369         if (ret != 0)
6370                 goto fail_unlock;
6371
6372         inode_init_owner(inode, dir, mode);
6373         inode_set_bytes(inode, 0);
6374
6375         inode->i_mtime = current_time(inode);
6376         inode->i_atime = inode->i_mtime;
6377         inode->i_ctime = inode->i_mtime;
6378         BTRFS_I(inode)->i_otime = inode->i_mtime;
6379
6380         inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
6381                                   struct btrfs_inode_item);
6382         memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
6383                              sizeof(*inode_item));
6384         fill_inode_item(trans, path->nodes[0], inode_item, inode);
6385
6386         if (name) {
6387                 ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
6388                                      struct btrfs_inode_ref);
6389                 btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
6390                 btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
6391                 ptr = (unsigned long)(ref + 1);
6392                 write_extent_buffer(path->nodes[0], name, ptr, name_len);
6393         }
6394
6395         btrfs_mark_buffer_dirty(path->nodes[0]);
6396         btrfs_free_path(path);
6397
6398         btrfs_inherit_iflags(inode, dir);
6399
6400         if (S_ISREG(mode)) {
6401                 if (btrfs_test_opt(fs_info, NODATASUM))
6402                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
6403                 if (btrfs_test_opt(fs_info, NODATACOW))
6404                         BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
6405                                 BTRFS_INODE_NODATASUM;
6406         }
6407
6408         inode_tree_add(inode);
6409
6410         trace_btrfs_inode_new(inode);
6411         btrfs_set_inode_last_trans(trans, inode);
6412
6413         btrfs_update_root_times(trans, root);
6414
6415         ret = btrfs_inode_inherit_props(trans, inode, dir);
6416         if (ret)
6417                 btrfs_err(fs_info,
6418                           "error inheriting props for ino %llu (root %llu): %d",
6419                         btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
6420
6421         return inode;
6422
6423 fail_unlock:
6424         unlock_new_inode(inode);
6425 fail:
6426         if (dir && name)
6427                 BTRFS_I(dir)->index_cnt--;
6428         btrfs_free_path(path);
6429         iput(inode);
6430         return ERR_PTR(ret);
6431 }
6432
6433 static inline u8 btrfs_inode_type(struct inode *inode)
6434 {
6435         return btrfs_type_by_mode[(inode->i_mode & S_IFMT) >> S_SHIFT];
6436 }
6437
6438 /*
6439  * utility function to add 'inode' into 'parent_inode' with
6440  * a give name and a given sequence number.
6441  * if 'add_backref' is true, also insert a backref from the
6442  * inode to the parent directory.
6443  */
6444 int btrfs_add_link(struct btrfs_trans_handle *trans,
6445                    struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
6446                    const char *name, int name_len, int add_backref, u64 index)
6447 {
6448         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6449         int ret = 0;
6450         struct btrfs_key key;
6451         struct btrfs_root *root = parent_inode->root;
6452         u64 ino = btrfs_ino(inode);
6453         u64 parent_ino = btrfs_ino(parent_inode);
6454
6455         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6456                 memcpy(&key, &inode->root->root_key, sizeof(key));
6457         } else {
6458                 key.objectid = ino;
6459                 key.type = BTRFS_INODE_ITEM_KEY;
6460                 key.offset = 0;
6461         }
6462
6463         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6464                 ret = btrfs_add_root_ref(trans, fs_info, key.objectid,
6465                                          root->root_key.objectid, parent_ino,
6466                                          index, name, name_len);
6467         } else if (add_backref) {
6468                 ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
6469                                              parent_ino, index);
6470         }
6471
6472         /* Nothing to clean up yet */
6473         if (ret)
6474                 return ret;
6475
6476         ret = btrfs_insert_dir_item(trans, root, name, name_len,
6477                                     parent_inode, &key,
6478                                     btrfs_inode_type(&inode->vfs_inode), index);
6479         if (ret == -EEXIST || ret == -EOVERFLOW)
6480                 goto fail_dir_item;
6481         else if (ret) {
6482                 btrfs_abort_transaction(trans, ret);
6483                 return ret;
6484         }
6485
6486         btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
6487                            name_len * 2);
6488         inode_inc_iversion(&parent_inode->vfs_inode);
6489         parent_inode->vfs_inode.i_mtime = parent_inode->vfs_inode.i_ctime =
6490                 current_time(&parent_inode->vfs_inode);
6491         ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
6492         if (ret)
6493                 btrfs_abort_transaction(trans, ret);
6494         return ret;
6495
6496 fail_dir_item:
6497         if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
6498                 u64 local_index;
6499                 int err;
6500                 err = btrfs_del_root_ref(trans, fs_info, key.objectid,
6501                                          root->root_key.objectid, parent_ino,
6502                                          &local_index, name, name_len);
6503
6504         } else if (add_backref) {
6505                 u64 local_index;
6506                 int err;
6507
6508                 err = btrfs_del_inode_ref(trans, root, name, name_len,
6509                                           ino, parent_ino, &local_index);
6510         }
6511         return ret;
6512 }
6513
6514 static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
6515                             struct btrfs_inode *dir, struct dentry *dentry,
6516                             struct btrfs_inode *inode, int backref, u64 index)
6517 {
6518         int err = btrfs_add_link(trans, dir, inode,
6519                                  dentry->d_name.name, dentry->d_name.len,
6520                                  backref, index);
6521         if (err > 0)
6522                 err = -EEXIST;
6523         return err;
6524 }
6525
6526 static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
6527                         umode_t mode, dev_t rdev)
6528 {
6529         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6530         struct btrfs_trans_handle *trans;
6531         struct btrfs_root *root = BTRFS_I(dir)->root;
6532         struct inode *inode = NULL;
6533         int err;
6534         int drop_inode = 0;
6535         u64 objectid;
6536         u64 index = 0;
6537
6538         /*
6539          * 2 for inode item and ref
6540          * 2 for dir items
6541          * 1 for xattr if selinux is on
6542          */
6543         trans = btrfs_start_transaction(root, 5);
6544         if (IS_ERR(trans))
6545                 return PTR_ERR(trans);
6546
6547         err = btrfs_find_free_ino(root, &objectid);
6548         if (err)
6549                 goto out_unlock;
6550
6551         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6552                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6553                         mode, &index);
6554         if (IS_ERR(inode)) {
6555                 err = PTR_ERR(inode);
6556                 goto out_unlock;
6557         }
6558
6559         /*
6560         * If the active LSM wants to access the inode during
6561         * d_instantiate it needs these. Smack checks to see
6562         * if the filesystem supports xattrs by looking at the
6563         * ops vector.
6564         */
6565         inode->i_op = &btrfs_special_inode_operations;
6566         init_special_inode(inode, inode->i_mode, rdev);
6567
6568         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6569         if (err)
6570                 goto out_unlock_inode;
6571
6572         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6573                         0, index);
6574         if (err) {
6575                 goto out_unlock_inode;
6576         } else {
6577                 btrfs_update_inode(trans, root, inode);
6578                 unlock_new_inode(inode);
6579                 d_instantiate(dentry, inode);
6580         }
6581
6582 out_unlock:
6583         btrfs_end_transaction(trans);
6584         btrfs_btree_balance_dirty(fs_info);
6585         if (drop_inode) {
6586                 inode_dec_link_count(inode);
6587                 iput(inode);
6588         }
6589         return err;
6590
6591 out_unlock_inode:
6592         drop_inode = 1;
6593         unlock_new_inode(inode);
6594         goto out_unlock;
6595
6596 }
6597
6598 static int btrfs_create(struct inode *dir, struct dentry *dentry,
6599                         umode_t mode, bool excl)
6600 {
6601         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6602         struct btrfs_trans_handle *trans;
6603         struct btrfs_root *root = BTRFS_I(dir)->root;
6604         struct inode *inode = NULL;
6605         int drop_inode_on_err = 0;
6606         int err;
6607         u64 objectid;
6608         u64 index = 0;
6609
6610         /*
6611          * 2 for inode item and ref
6612          * 2 for dir items
6613          * 1 for xattr if selinux is on
6614          */
6615         trans = btrfs_start_transaction(root, 5);
6616         if (IS_ERR(trans))
6617                 return PTR_ERR(trans);
6618
6619         err = btrfs_find_free_ino(root, &objectid);
6620         if (err)
6621                 goto out_unlock;
6622
6623         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6624                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6625                         mode, &index);
6626         if (IS_ERR(inode)) {
6627                 err = PTR_ERR(inode);
6628                 goto out_unlock;
6629         }
6630         drop_inode_on_err = 1;
6631         /*
6632         * If the active LSM wants to access the inode during
6633         * d_instantiate it needs these. Smack checks to see
6634         * if the filesystem supports xattrs by looking at the
6635         * ops vector.
6636         */
6637         inode->i_fop = &btrfs_file_operations;
6638         inode->i_op = &btrfs_file_inode_operations;
6639         inode->i_mapping->a_ops = &btrfs_aops;
6640
6641         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6642         if (err)
6643                 goto out_unlock_inode;
6644
6645         err = btrfs_update_inode(trans, root, inode);
6646         if (err)
6647                 goto out_unlock_inode;
6648
6649         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6650                         0, index);
6651         if (err)
6652                 goto out_unlock_inode;
6653
6654         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
6655         unlock_new_inode(inode);
6656         d_instantiate(dentry, inode);
6657
6658 out_unlock:
6659         btrfs_end_transaction(trans);
6660         if (err && drop_inode_on_err) {
6661                 inode_dec_link_count(inode);
6662                 iput(inode);
6663         }
6664         btrfs_btree_balance_dirty(fs_info);
6665         return err;
6666
6667 out_unlock_inode:
6668         unlock_new_inode(inode);
6669         goto out_unlock;
6670
6671 }
6672
6673 static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
6674                       struct dentry *dentry)
6675 {
6676         struct btrfs_trans_handle *trans = NULL;
6677         struct btrfs_root *root = BTRFS_I(dir)->root;
6678         struct inode *inode = d_inode(old_dentry);
6679         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
6680         u64 index;
6681         int err;
6682         int drop_inode = 0;
6683
6684         /* do not allow sys_link's with other subvols of the same device */
6685         if (root->objectid != BTRFS_I(inode)->root->objectid)
6686                 return -EXDEV;
6687
6688         if (inode->i_nlink >= BTRFS_LINK_MAX)
6689                 return -EMLINK;
6690
6691         err = btrfs_set_inode_index(BTRFS_I(dir), &index);
6692         if (err)
6693                 goto fail;
6694
6695         /*
6696          * 2 items for inode and inode ref
6697          * 2 items for dir items
6698          * 1 item for parent inode
6699          */
6700         trans = btrfs_start_transaction(root, 5);
6701         if (IS_ERR(trans)) {
6702                 err = PTR_ERR(trans);
6703                 trans = NULL;
6704                 goto fail;
6705         }
6706
6707         /* There are several dir indexes for this inode, clear the cache. */
6708         BTRFS_I(inode)->dir_index = 0ULL;
6709         inc_nlink(inode);
6710         inode_inc_iversion(inode);
6711         inode->i_ctime = current_time(inode);
6712         ihold(inode);
6713         set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
6714
6715         err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
6716                         1, index);
6717
6718         if (err) {
6719                 drop_inode = 1;
6720         } else {
6721                 struct dentry *parent = dentry->d_parent;
6722                 err = btrfs_update_inode(trans, root, inode);
6723                 if (err)
6724                         goto fail;
6725                 if (inode->i_nlink == 1) {
6726                         /*
6727                          * If new hard link count is 1, it's a file created
6728                          * with open(2) O_TMPFILE flag.
6729                          */
6730                         err = btrfs_orphan_del(trans, BTRFS_I(inode));
6731                         if (err)
6732                                 goto fail;
6733                 }
6734                 d_instantiate(dentry, inode);
6735                 btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
6736         }
6737
6738 fail:
6739         if (trans)
6740                 btrfs_end_transaction(trans);
6741         if (drop_inode) {
6742                 inode_dec_link_count(inode);
6743                 iput(inode);
6744         }
6745         btrfs_btree_balance_dirty(fs_info);
6746         return err;
6747 }
6748
6749 static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
6750 {
6751         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
6752         struct inode *inode = NULL;
6753         struct btrfs_trans_handle *trans;
6754         struct btrfs_root *root = BTRFS_I(dir)->root;
6755         int err = 0;
6756         int drop_on_err = 0;
6757         u64 objectid = 0;
6758         u64 index = 0;
6759
6760         /*
6761          * 2 items for inode and ref
6762          * 2 items for dir items
6763          * 1 for xattr if selinux is on
6764          */
6765         trans = btrfs_start_transaction(root, 5);
6766         if (IS_ERR(trans))
6767                 return PTR_ERR(trans);
6768
6769         err = btrfs_find_free_ino(root, &objectid);
6770         if (err)
6771                 goto out_fail;
6772
6773         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
6774                         dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
6775                         S_IFDIR | mode, &index);
6776         if (IS_ERR(inode)) {
6777                 err = PTR_ERR(inode);
6778                 goto out_fail;
6779         }
6780
6781         drop_on_err = 1;
6782         /* these must be set before we unlock the inode */
6783         inode->i_op = &btrfs_dir_inode_operations;
6784         inode->i_fop = &btrfs_dir_file_operations;
6785
6786         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
6787         if (err)
6788                 goto out_fail_inode;
6789
6790         btrfs_i_size_write(BTRFS_I(inode), 0);
6791         err = btrfs_update_inode(trans, root, inode);
6792         if (err)
6793                 goto out_fail_inode;
6794
6795         err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
6796                         dentry->d_name.name,
6797                         dentry->d_name.len, 0, index);
6798         if (err)
6799                 goto out_fail_inode;
6800
6801         d_instantiate(dentry, inode);
6802         /*
6803          * mkdir is special.  We're unlocking after we call d_instantiate
6804          * to avoid a race with nfsd calling d_instantiate.
6805          */
6806         unlock_new_inode(inode);
6807         drop_on_err = 0;
6808
6809 out_fail:
6810         btrfs_end_transaction(trans);
6811         if (drop_on_err) {
6812                 inode_dec_link_count(inode);
6813                 iput(inode);
6814         }
6815         btrfs_btree_balance_dirty(fs_info);
6816         return err;
6817
6818 out_fail_inode:
6819         unlock_new_inode(inode);
6820         goto out_fail;
6821 }
6822
6823 static noinline int uncompress_inline(struct btrfs_path *path,
6824                                       struct page *page,
6825                                       size_t pg_offset, u64 extent_offset,
6826                                       struct btrfs_file_extent_item *item)
6827 {
6828         int ret;
6829         struct extent_buffer *leaf = path->nodes[0];
6830         char *tmp;
6831         size_t max_size;
6832         unsigned long inline_size;
6833         unsigned long ptr;
6834         int compress_type;
6835
6836         WARN_ON(pg_offset != 0);
6837         compress_type = btrfs_file_extent_compression(leaf, item);
6838         max_size = btrfs_file_extent_ram_bytes(leaf, item);
6839         inline_size = btrfs_file_extent_inline_item_len(leaf,
6840                                         btrfs_item_nr(path->slots[0]));
6841         tmp = kmalloc(inline_size, GFP_NOFS);
6842         if (!tmp)
6843                 return -ENOMEM;
6844         ptr = btrfs_file_extent_inline_start(item);
6845
6846         read_extent_buffer(leaf, tmp, ptr, inline_size);
6847
6848         max_size = min_t(unsigned long, PAGE_SIZE, max_size);
6849         ret = btrfs_decompress(compress_type, tmp, page,
6850                                extent_offset, inline_size, max_size);
6851
6852         /*
6853          * decompression code contains a memset to fill in any space between the end
6854          * of the uncompressed data and the end of max_size in case the decompressed
6855          * data ends up shorter than ram_bytes.  That doesn't cover the hole between
6856          * the end of an inline extent and the beginning of the next block, so we
6857          * cover that region here.
6858          */
6859
6860         if (max_size + pg_offset < PAGE_SIZE) {
6861                 char *map = kmap(page);
6862                 memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
6863                 kunmap(page);
6864         }
6865         kfree(tmp);
6866         return ret;
6867 }
6868
6869 /*
6870  * a bit scary, this does extent mapping from logical file offset to the disk.
6871  * the ugly parts come from merging extents from the disk with the in-ram
6872  * representation.  This gets more complex because of the data=ordered code,
6873  * where the in-ram extents might be locked pending data=ordered completion.
6874  *
6875  * This also copies inline extents directly into the page.
6876  */
6877 struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
6878                 struct page *page,
6879             size_t pg_offset, u64 start, u64 len,
6880                 int create)
6881 {
6882         struct btrfs_fs_info *fs_info = btrfs_sb(inode->vfs_inode.i_sb);
6883         int ret;
6884         int err = 0;
6885         u64 extent_start = 0;
6886         u64 extent_end = 0;
6887         u64 objectid = btrfs_ino(inode);
6888         u32 found_type;
6889         struct btrfs_path *path = NULL;
6890         struct btrfs_root *root = inode->root;
6891         struct btrfs_file_extent_item *item;
6892         struct extent_buffer *leaf;
6893         struct btrfs_key found_key;
6894         struct extent_map *em = NULL;
6895         struct extent_map_tree *em_tree = &inode->extent_tree;
6896         struct extent_io_tree *io_tree = &inode->io_tree;
6897         const bool new_inline = !page || create;
6898
6899         read_lock(&em_tree->lock);
6900         em = lookup_extent_mapping(em_tree, start, len);
6901         if (em)
6902                 em->bdev = fs_info->fs_devices->latest_bdev;
6903         read_unlock(&em_tree->lock);
6904
6905         if (em) {
6906                 if (em->start > start || em->start + em->len <= start)
6907                         free_extent_map(em);
6908                 else if (em->block_start == EXTENT_MAP_INLINE && page)
6909                         free_extent_map(em);
6910                 else
6911                         goto out;
6912         }
6913         em = alloc_extent_map();
6914         if (!em) {
6915                 err = -ENOMEM;
6916                 goto out;
6917         }
6918         em->bdev = fs_info->fs_devices->latest_bdev;
6919         em->start = EXTENT_MAP_HOLE;
6920         em->orig_start = EXTENT_MAP_HOLE;
6921         em->len = (u64)-1;
6922         em->block_len = (u64)-1;
6923
6924         if (!path) {
6925                 path = btrfs_alloc_path();
6926                 if (!path) {
6927                         err = -ENOMEM;
6928                         goto out;
6929                 }
6930                 /*
6931                  * Chances are we'll be called again, so go ahead and do
6932                  * readahead
6933                  */
6934                 path->reada = READA_FORWARD;
6935         }
6936
6937         ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
6938         if (ret < 0) {
6939                 err = ret;
6940                 goto out;
6941         }
6942
6943         if (ret != 0) {
6944                 if (path->slots[0] == 0)
6945                         goto not_found;
6946                 path->slots[0]--;
6947         }
6948
6949         leaf = path->nodes[0];
6950         item = btrfs_item_ptr(leaf, path->slots[0],
6951                               struct btrfs_file_extent_item);
6952         /* are we inside the extent that was found? */
6953         btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
6954         found_type = found_key.type;
6955         if (found_key.objectid != objectid ||
6956             found_type != BTRFS_EXTENT_DATA_KEY) {
6957                 /*
6958                  * If we backup past the first extent we want to move forward
6959                  * and see if there is an extent in front of us, otherwise we'll
6960                  * say there is a hole for our whole search range which can
6961                  * cause problems.
6962                  */
6963                 extent_end = start;
6964                 goto next;
6965         }
6966
6967         found_type = btrfs_file_extent_type(leaf, item);
6968         extent_start = found_key.offset;
6969         if (found_type == BTRFS_FILE_EXTENT_REG ||
6970             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
6971                 extent_end = extent_start +
6972                        btrfs_file_extent_num_bytes(leaf, item);
6973
6974                 trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
6975                                                        extent_start);
6976         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
6977                 size_t size;
6978                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
6979                 extent_end = ALIGN(extent_start + size,
6980                                    fs_info->sectorsize);
6981
6982                 trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
6983                                                       path->slots[0],
6984                                                       extent_start);
6985         }
6986 next:
6987         if (start >= extent_end) {
6988                 path->slots[0]++;
6989                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
6990                         ret = btrfs_next_leaf(root, path);
6991                         if (ret < 0) {
6992                                 err = ret;
6993                                 goto out;
6994                         }
6995                         if (ret > 0)
6996                                 goto not_found;
6997                         leaf = path->nodes[0];
6998                 }
6999                 btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
7000                 if (found_key.objectid != objectid ||
7001                     found_key.type != BTRFS_EXTENT_DATA_KEY)
7002                         goto not_found;
7003                 if (start + len <= found_key.offset)
7004                         goto not_found;
7005                 if (start > found_key.offset)
7006                         goto next;
7007                 em->start = start;
7008                 em->orig_start = start;
7009                 em->len = found_key.offset - start;
7010                 goto not_found_em;
7011         }
7012
7013         btrfs_extent_item_to_extent_map(inode, path, item,
7014                         new_inline, em);
7015
7016         if (found_type == BTRFS_FILE_EXTENT_REG ||
7017             found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7018                 goto insert;
7019         } else if (found_type == BTRFS_FILE_EXTENT_INLINE) {
7020                 unsigned long ptr;
7021                 char *map;
7022                 size_t size;
7023                 size_t extent_offset;
7024                 size_t copy_size;
7025
7026                 if (new_inline)
7027                         goto out;
7028
7029                 size = btrfs_file_extent_inline_len(leaf, path->slots[0], item);
7030                 extent_offset = page_offset(page) + pg_offset - extent_start;
7031                 copy_size = min_t(u64, PAGE_SIZE - pg_offset,
7032                                   size - extent_offset);
7033                 em->start = extent_start + extent_offset;
7034                 em->len = ALIGN(copy_size, fs_info->sectorsize);
7035                 em->orig_block_len = em->len;
7036                 em->orig_start = em->start;
7037                 ptr = btrfs_file_extent_inline_start(item) + extent_offset;
7038                 if (!PageUptodate(page)) {
7039                         if (btrfs_file_extent_compression(leaf, item) !=
7040                             BTRFS_COMPRESS_NONE) {
7041                                 ret = uncompress_inline(path, page, pg_offset,
7042                                                         extent_offset, item);
7043                                 if (ret) {
7044                                         err = ret;
7045                                         goto out;
7046                                 }
7047                         } else {
7048                                 map = kmap(page);
7049                                 read_extent_buffer(leaf, map + pg_offset, ptr,
7050                                                    copy_size);
7051                                 if (pg_offset + copy_size < PAGE_SIZE) {
7052                                         memset(map + pg_offset + copy_size, 0,
7053                                                PAGE_SIZE - pg_offset -
7054                                                copy_size);
7055                                 }
7056                                 kunmap(page);
7057                         }
7058                         flush_dcache_page(page);
7059                 }
7060                 set_extent_uptodate(io_tree, em->start,
7061                                     extent_map_end(em) - 1, NULL, GFP_NOFS);
7062                 goto insert;
7063         }
7064 not_found:
7065         em->start = start;
7066         em->orig_start = start;
7067         em->len = len;
7068 not_found_em:
7069         em->block_start = EXTENT_MAP_HOLE;
7070 insert:
7071         btrfs_release_path(path);
7072         if (em->start > start || extent_map_end(em) <= start) {
7073                 btrfs_err(fs_info,
7074                           "bad extent! em: [%llu %llu] passed [%llu %llu]",
7075                           em->start, em->len, start, len);
7076                 err = -EIO;
7077                 goto out;
7078         }
7079
7080         err = 0;
7081         write_lock(&em_tree->lock);
7082         err = btrfs_add_extent_mapping(em_tree, &em, start, len);
7083         write_unlock(&em_tree->lock);
7084 out:
7085
7086         trace_btrfs_get_extent(root, inode, em);
7087
7088         btrfs_free_path(path);
7089         if (err) {
7090                 free_extent_map(em);
7091                 return ERR_PTR(err);
7092         }
7093         BUG_ON(!em); /* Error is always set */
7094         return em;
7095 }
7096
7097 struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
7098                 struct page *page,
7099                 size_t pg_offset, u64 start, u64 len,
7100                 int create)
7101 {
7102         struct extent_map *em;
7103         struct extent_map *hole_em = NULL;
7104         u64 range_start = start;
7105         u64 end;
7106         u64 found;
7107         u64 found_end;
7108         int err = 0;
7109
7110         em = btrfs_get_extent(inode, page, pg_offset, start, len, create);
7111         if (IS_ERR(em))
7112                 return em;
7113         /*
7114          * If our em maps to:
7115          * - a hole or
7116          * - a pre-alloc extent,
7117          * there might actually be delalloc bytes behind it.
7118          */
7119         if (em->block_start != EXTENT_MAP_HOLE &&
7120             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7121                 return em;
7122         else
7123                 hole_em = em;
7124
7125         /* check to see if we've wrapped (len == -1 or similar) */
7126         end = start + len;
7127         if (end < start)
7128                 end = (u64)-1;
7129         else
7130                 end -= 1;
7131
7132         em = NULL;
7133
7134         /* ok, we didn't find anything, lets look for delalloc */
7135         found = count_range_bits(&inode->io_tree, &range_start,
7136                                  end, len, EXTENT_DELALLOC, 1);
7137         found_end = range_start + found;
7138         if (found_end < range_start)
7139                 found_end = (u64)-1;
7140
7141         /*
7142          * we didn't find anything useful, return
7143          * the original results from get_extent()
7144          */
7145         if (range_start > end || found_end <= start) {
7146                 em = hole_em;
7147                 hole_em = NULL;
7148                 goto out;
7149         }
7150
7151         /* adjust the range_start to make sure it doesn't
7152          * go backwards from the start they passed in
7153          */
7154         range_start = max(start, range_start);
7155         found = found_end - range_start;
7156
7157         if (found > 0) {
7158                 u64 hole_start = start;
7159                 u64 hole_len = len;
7160
7161                 em = alloc_extent_map();
7162                 if (!em) {
7163                         err = -ENOMEM;
7164                         goto out;
7165                 }
7166                 /*
7167                  * when btrfs_get_extent can't find anything it
7168                  * returns one huge hole
7169                  *
7170                  * make sure what it found really fits our range, and
7171                  * adjust to make sure it is based on the start from
7172                  * the caller
7173                  */
7174                 if (hole_em) {
7175                         u64 calc_end = extent_map_end(hole_em);
7176
7177                         if (calc_end <= start || (hole_em->start > end)) {
7178                                 free_extent_map(hole_em);
7179                                 hole_em = NULL;
7180                         } else {
7181                                 hole_start = max(hole_em->start, start);
7182                                 hole_len = calc_end - hole_start;
7183                         }
7184                 }
7185                 em->bdev = NULL;
7186                 if (hole_em && range_start > hole_start) {
7187                         /* our hole starts before our delalloc, so we
7188                          * have to return just the parts of the hole
7189                          * that go until  the delalloc starts
7190                          */
7191                         em->len = min(hole_len,
7192                                       range_start - hole_start);
7193                         em->start = hole_start;
7194                         em->orig_start = hole_start;
7195                         /*
7196                          * don't adjust block start at all,
7197                          * it is fixed at EXTENT_MAP_HOLE
7198                          */
7199                         em->block_start = hole_em->block_start;
7200                         em->block_len = hole_len;
7201                         if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
7202                                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
7203                 } else {
7204                         em->start = range_start;
7205                         em->len = found;
7206                         em->orig_start = range_start;
7207                         em->block_start = EXTENT_MAP_DELALLOC;
7208                         em->block_len = found;
7209                 }
7210         } else {
7211                 return hole_em;
7212         }
7213 out:
7214
7215         free_extent_map(hole_em);
7216         if (err) {
7217                 free_extent_map(em);
7218                 return ERR_PTR(err);
7219         }
7220         return em;
7221 }
7222
7223 static struct extent_map *btrfs_create_dio_extent(struct inode *inode,
7224                                                   const u64 start,
7225                                                   const u64 len,
7226                                                   const u64 orig_start,
7227                                                   const u64 block_start,
7228                                                   const u64 block_len,
7229                                                   const u64 orig_block_len,
7230                                                   const u64 ram_bytes,
7231                                                   const int type)
7232 {
7233         struct extent_map *em = NULL;
7234         int ret;
7235
7236         if (type != BTRFS_ORDERED_NOCOW) {
7237                 em = create_io_em(inode, start, len, orig_start,
7238                                   block_start, block_len, orig_block_len,
7239                                   ram_bytes,
7240                                   BTRFS_COMPRESS_NONE, /* compress_type */
7241                                   type);
7242                 if (IS_ERR(em))
7243                         goto out;
7244         }
7245         ret = btrfs_add_ordered_extent_dio(inode, start, block_start,
7246                                            len, block_len, type);
7247         if (ret) {
7248                 if (em) {
7249                         free_extent_map(em);
7250                         btrfs_drop_extent_cache(BTRFS_I(inode), start,
7251                                                 start + len - 1, 0);
7252                 }
7253                 em = ERR_PTR(ret);
7254         }
7255  out:
7256
7257         return em;
7258 }
7259
7260 static struct extent_map *btrfs_new_extent_direct(struct inode *inode,
7261                                                   u64 start, u64 len)
7262 {
7263         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7264         struct btrfs_root *root = BTRFS_I(inode)->root;
7265         struct extent_map *em;
7266         struct btrfs_key ins;
7267         u64 alloc_hint;
7268         int ret;
7269
7270         alloc_hint = get_extent_allocation_hint(inode, start, len);
7271         ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
7272                                    0, alloc_hint, &ins, 1, 1);
7273         if (ret)
7274                 return ERR_PTR(ret);
7275
7276         em = btrfs_create_dio_extent(inode, start, ins.offset, start,
7277                                      ins.objectid, ins.offset, ins.offset,
7278                                      ins.offset, BTRFS_ORDERED_REGULAR);
7279         btrfs_dec_block_group_reservations(fs_info, ins.objectid);
7280         if (IS_ERR(em))
7281                 btrfs_free_reserved_extent(fs_info, ins.objectid,
7282                                            ins.offset, 1);
7283
7284         return em;
7285 }
7286
7287 /*
7288  * returns 1 when the nocow is safe, < 1 on error, 0 if the
7289  * block must be cow'd
7290  */
7291 noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
7292                               u64 *orig_start, u64 *orig_block_len,
7293                               u64 *ram_bytes)
7294 {
7295         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7296         struct btrfs_path *path;
7297         int ret;
7298         struct extent_buffer *leaf;
7299         struct btrfs_root *root = BTRFS_I(inode)->root;
7300         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7301         struct btrfs_file_extent_item *fi;
7302         struct btrfs_key key;
7303         u64 disk_bytenr;
7304         u64 backref_offset;
7305         u64 extent_end;
7306         u64 num_bytes;
7307         int slot;
7308         int found_type;
7309         bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
7310
7311         path = btrfs_alloc_path();
7312         if (!path)
7313                 return -ENOMEM;
7314
7315         ret = btrfs_lookup_file_extent(NULL, root, path,
7316                         btrfs_ino(BTRFS_I(inode)), offset, 0);
7317         if (ret < 0)
7318                 goto out;
7319
7320         slot = path->slots[0];
7321         if (ret == 1) {
7322                 if (slot == 0) {
7323                         /* can't find the item, must cow */
7324                         ret = 0;
7325                         goto out;
7326                 }
7327                 slot--;
7328         }
7329         ret = 0;
7330         leaf = path->nodes[0];
7331         btrfs_item_key_to_cpu(leaf, &key, slot);
7332         if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
7333             key.type != BTRFS_EXTENT_DATA_KEY) {
7334                 /* not our file or wrong item type, must cow */
7335                 goto out;
7336         }
7337
7338         if (key.offset > offset) {
7339                 /* Wrong offset, must cow */
7340                 goto out;
7341         }
7342
7343         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
7344         found_type = btrfs_file_extent_type(leaf, fi);
7345         if (found_type != BTRFS_FILE_EXTENT_REG &&
7346             found_type != BTRFS_FILE_EXTENT_PREALLOC) {
7347                 /* not a regular extent, must cow */
7348                 goto out;
7349         }
7350
7351         if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
7352                 goto out;
7353
7354         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
7355         if (extent_end <= offset)
7356                 goto out;
7357
7358         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
7359         if (disk_bytenr == 0)
7360                 goto out;
7361
7362         if (btrfs_file_extent_compression(leaf, fi) ||
7363             btrfs_file_extent_encryption(leaf, fi) ||
7364             btrfs_file_extent_other_encoding(leaf, fi))
7365                 goto out;
7366
7367         backref_offset = btrfs_file_extent_offset(leaf, fi);
7368
7369         if (orig_start) {
7370                 *orig_start = key.offset - backref_offset;
7371                 *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
7372                 *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
7373         }
7374
7375         if (btrfs_extent_readonly(fs_info, disk_bytenr))
7376                 goto out;
7377
7378         num_bytes = min(offset + *len, extent_end) - offset;
7379         if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
7380                 u64 range_end;
7381
7382                 range_end = round_up(offset + num_bytes,
7383                                      root->fs_info->sectorsize) - 1;
7384                 ret = test_range_bit(io_tree, offset, range_end,
7385                                      EXTENT_DELALLOC, 0, NULL);
7386                 if (ret) {
7387                         ret = -EAGAIN;
7388                         goto out;
7389                 }
7390         }
7391
7392         btrfs_release_path(path);
7393
7394         /*
7395          * look for other files referencing this extent, if we
7396          * find any we must cow
7397          */
7398
7399         ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
7400                                     key.offset - backref_offset, disk_bytenr);
7401         if (ret) {
7402                 ret = 0;
7403                 goto out;
7404         }
7405
7406         /*
7407          * adjust disk_bytenr and num_bytes to cover just the bytes
7408          * in this extent we are about to write.  If there
7409          * are any csums in that range we have to cow in order
7410          * to keep the csums correct
7411          */
7412         disk_bytenr += backref_offset;
7413         disk_bytenr += offset - key.offset;
7414         if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
7415                 goto out;
7416         /*
7417          * all of the above have passed, it is safe to overwrite this extent
7418          * without cow
7419          */
7420         *len = num_bytes;
7421         ret = 1;
7422 out:
7423         btrfs_free_path(path);
7424         return ret;
7425 }
7426
7427 static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
7428                               struct extent_state **cached_state, int writing)
7429 {
7430         struct btrfs_ordered_extent *ordered;
7431         int ret = 0;
7432
7433         while (1) {
7434                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7435                                  cached_state);
7436                 /*
7437                  * We're concerned with the entire range that we're going to be
7438                  * doing DIO to, so we need to make sure there's no ordered
7439                  * extents in this range.
7440                  */
7441                 ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
7442                                                      lockend - lockstart + 1);
7443
7444                 /*
7445                  * We need to make sure there are no buffered pages in this
7446                  * range either, we could have raced between the invalidate in
7447                  * generic_file_direct_write and locking the extent.  The
7448                  * invalidate needs to happen so that reads after a write do not
7449                  * get stale data.
7450                  */
7451                 if (!ordered &&
7452                     (!writing || !filemap_range_has_page(inode->i_mapping,
7453                                                          lockstart, lockend)))
7454                         break;
7455
7456                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7457                                      cached_state);
7458
7459                 if (ordered) {
7460                         /*
7461                          * If we are doing a DIO read and the ordered extent we
7462                          * found is for a buffered write, we can not wait for it
7463                          * to complete and retry, because if we do so we can
7464                          * deadlock with concurrent buffered writes on page
7465                          * locks. This happens only if our DIO read covers more
7466                          * than one extent map, if at this point has already
7467                          * created an ordered extent for a previous extent map
7468                          * and locked its range in the inode's io tree, and a
7469                          * concurrent write against that previous extent map's
7470                          * range and this range started (we unlock the ranges
7471                          * in the io tree only when the bios complete and
7472                          * buffered writes always lock pages before attempting
7473                          * to lock range in the io tree).
7474                          */
7475                         if (writing ||
7476                             test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
7477                                 btrfs_start_ordered_extent(inode, ordered, 1);
7478                         else
7479                                 ret = -ENOTBLK;
7480                         btrfs_put_ordered_extent(ordered);
7481                 } else {
7482                         /*
7483                          * We could trigger writeback for this range (and wait
7484                          * for it to complete) and then invalidate the pages for
7485                          * this range (through invalidate_inode_pages2_range()),
7486                          * but that can lead us to a deadlock with a concurrent
7487                          * call to readpages() (a buffered read or a defrag call
7488                          * triggered a readahead) on a page lock due to an
7489                          * ordered dio extent we created before but did not have
7490                          * yet a corresponding bio submitted (whence it can not
7491                          * complete), which makes readpages() wait for that
7492                          * ordered extent to complete while holding a lock on
7493                          * that page.
7494                          */
7495                         ret = -ENOTBLK;
7496                 }
7497
7498                 if (ret)
7499                         break;
7500
7501                 cond_resched();
7502         }
7503
7504         return ret;
7505 }
7506
7507 /* The callers of this must take lock_extent() */
7508 static struct extent_map *create_io_em(struct inode *inode, u64 start, u64 len,
7509                                        u64 orig_start, u64 block_start,
7510                                        u64 block_len, u64 orig_block_len,
7511                                        u64 ram_bytes, int compress_type,
7512                                        int type)
7513 {
7514         struct extent_map_tree *em_tree;
7515         struct extent_map *em;
7516         struct btrfs_root *root = BTRFS_I(inode)->root;
7517         int ret;
7518
7519         ASSERT(type == BTRFS_ORDERED_PREALLOC ||
7520                type == BTRFS_ORDERED_COMPRESSED ||
7521                type == BTRFS_ORDERED_NOCOW ||
7522                type == BTRFS_ORDERED_REGULAR);
7523
7524         em_tree = &BTRFS_I(inode)->extent_tree;
7525         em = alloc_extent_map();
7526         if (!em)
7527                 return ERR_PTR(-ENOMEM);
7528
7529         em->start = start;
7530         em->orig_start = orig_start;
7531         em->len = len;
7532         em->block_len = block_len;
7533         em->block_start = block_start;
7534         em->bdev = root->fs_info->fs_devices->latest_bdev;
7535         em->orig_block_len = orig_block_len;
7536         em->ram_bytes = ram_bytes;
7537         em->generation = -1;
7538         set_bit(EXTENT_FLAG_PINNED, &em->flags);
7539         if (type == BTRFS_ORDERED_PREALLOC) {
7540                 set_bit(EXTENT_FLAG_FILLING, &em->flags);
7541         } else if (type == BTRFS_ORDERED_COMPRESSED) {
7542                 set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
7543                 em->compress_type = compress_type;
7544         }
7545
7546         do {
7547                 btrfs_drop_extent_cache(BTRFS_I(inode), em->start,
7548                                 em->start + em->len - 1, 0);
7549                 write_lock(&em_tree->lock);
7550                 ret = add_extent_mapping(em_tree, em, 1);
7551                 write_unlock(&em_tree->lock);
7552                 /*
7553                  * The caller has taken lock_extent(), who could race with us
7554                  * to add em?
7555                  */
7556         } while (ret == -EEXIST);
7557
7558         if (ret) {
7559                 free_extent_map(em);
7560                 return ERR_PTR(ret);
7561         }
7562
7563         /* em got 2 refs now, callers needs to do free_extent_map once. */
7564         return em;
7565 }
7566
7567 static int btrfs_get_blocks_direct(struct inode *inode, sector_t iblock,
7568                                    struct buffer_head *bh_result, int create)
7569 {
7570         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7571         struct extent_map *em;
7572         struct extent_state *cached_state = NULL;
7573         struct btrfs_dio_data *dio_data = NULL;
7574         u64 start = iblock << inode->i_blkbits;
7575         u64 lockstart, lockend;
7576         u64 len = bh_result->b_size;
7577         int unlock_bits = EXTENT_LOCKED;
7578         int ret = 0;
7579
7580         if (create)
7581                 unlock_bits |= EXTENT_DIRTY;
7582         else
7583                 len = min_t(u64, len, fs_info->sectorsize);
7584
7585         lockstart = start;
7586         lockend = start + len - 1;
7587
7588         if (current->journal_info) {
7589                 /*
7590                  * Need to pull our outstanding extents and set journal_info to NULL so
7591                  * that anything that needs to check if there's a transaction doesn't get
7592                  * confused.
7593                  */
7594                 dio_data = current->journal_info;
7595                 current->journal_info = NULL;
7596         }
7597
7598         /*
7599          * If this errors out it's because we couldn't invalidate pagecache for
7600          * this range and we need to fallback to buffered.
7601          */
7602         if (lock_extent_direct(inode, lockstart, lockend, &cached_state,
7603                                create)) {
7604                 ret = -ENOTBLK;
7605                 goto err;
7606         }
7607
7608         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len, 0);
7609         if (IS_ERR(em)) {
7610                 ret = PTR_ERR(em);
7611                 goto unlock_err;
7612         }
7613
7614         /*
7615          * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
7616          * io.  INLINE is special, and we could probably kludge it in here, but
7617          * it's still buffered so for safety lets just fall back to the generic
7618          * buffered path.
7619          *
7620          * For COMPRESSED we _have_ to read the entire extent in so we can
7621          * decompress it, so there will be buffering required no matter what we
7622          * do, so go ahead and fallback to buffered.
7623          *
7624          * We return -ENOTBLK because that's what makes DIO go ahead and go back
7625          * to buffered IO.  Don't blame me, this is the price we pay for using
7626          * the generic code.
7627          */
7628         if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
7629             em->block_start == EXTENT_MAP_INLINE) {
7630                 free_extent_map(em);
7631                 ret = -ENOTBLK;
7632                 goto unlock_err;
7633         }
7634
7635         /* Just a good old fashioned hole, return */
7636         if (!create && (em->block_start == EXTENT_MAP_HOLE ||
7637                         test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
7638                 free_extent_map(em);
7639                 goto unlock_err;
7640         }
7641
7642         /*
7643          * We don't allocate a new extent in the following cases
7644          *
7645          * 1) The inode is marked as NODATACOW.  In this case we'll just use the
7646          * existing extent.
7647          * 2) The extent is marked as PREALLOC.  We're good to go here and can
7648          * just use the extent.
7649          *
7650          */
7651         if (!create) {
7652                 len = min(len, em->len - (start - em->start));
7653                 lockstart = start + len;
7654                 goto unlock;
7655         }
7656
7657         if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
7658             ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
7659              em->block_start != EXTENT_MAP_HOLE)) {
7660                 int type;
7661                 u64 block_start, orig_start, orig_block_len, ram_bytes;
7662
7663                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7664                         type = BTRFS_ORDERED_PREALLOC;
7665                 else
7666                         type = BTRFS_ORDERED_NOCOW;
7667                 len = min(len, em->len - (start - em->start));
7668                 block_start = em->block_start + (start - em->start);
7669
7670                 if (can_nocow_extent(inode, start, &len, &orig_start,
7671                                      &orig_block_len, &ram_bytes) == 1 &&
7672                     btrfs_inc_nocow_writers(fs_info, block_start)) {
7673                         struct extent_map *em2;
7674
7675                         em2 = btrfs_create_dio_extent(inode, start, len,
7676                                                       orig_start, block_start,
7677                                                       len, orig_block_len,
7678                                                       ram_bytes, type);
7679                         btrfs_dec_nocow_writers(fs_info, block_start);
7680                         if (type == BTRFS_ORDERED_PREALLOC) {
7681                                 free_extent_map(em);
7682                                 em = em2;
7683                         }
7684                         if (em2 && IS_ERR(em2)) {
7685                                 ret = PTR_ERR(em2);
7686                                 goto unlock_err;
7687                         }
7688                         /*
7689                          * For inode marked NODATACOW or extent marked PREALLOC,
7690                          * use the existing or preallocated extent, so does not
7691                          * need to adjust btrfs_space_info's bytes_may_use.
7692                          */
7693                         btrfs_free_reserved_data_space_noquota(inode,
7694                                         start, len);
7695                         goto unlock;
7696                 }
7697         }
7698
7699         /*
7700          * this will cow the extent, reset the len in case we changed
7701          * it above
7702          */
7703         len = bh_result->b_size;
7704         free_extent_map(em);
7705         em = btrfs_new_extent_direct(inode, start, len);
7706         if (IS_ERR(em)) {
7707                 ret = PTR_ERR(em);
7708                 goto unlock_err;
7709         }
7710         len = min(len, em->len - (start - em->start));
7711 unlock:
7712         bh_result->b_blocknr = (em->block_start + (start - em->start)) >>
7713                 inode->i_blkbits;
7714         bh_result->b_size = len;
7715         bh_result->b_bdev = em->bdev;
7716         set_buffer_mapped(bh_result);
7717         if (create) {
7718                 if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
7719                         set_buffer_new(bh_result);
7720
7721                 /*
7722                  * Need to update the i_size under the extent lock so buffered
7723                  * readers will get the updated i_size when we unlock.
7724                  */
7725                 if (!dio_data->overwrite && start + len > i_size_read(inode))
7726                         i_size_write(inode, start + len);
7727
7728                 WARN_ON(dio_data->reserve < len);
7729                 dio_data->reserve -= len;
7730                 dio_data->unsubmitted_oe_range_end = start + len;
7731                 current->journal_info = dio_data;
7732         }
7733
7734         /*
7735          * In the case of write we need to clear and unlock the entire range,
7736          * in the case of read we need to unlock only the end area that we
7737          * aren't using if there is any left over space.
7738          */
7739         if (lockstart < lockend) {
7740                 clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart,
7741                                  lockend, unlock_bits, 1, 0,
7742                                  &cached_state);
7743         } else {
7744                 free_extent_state(cached_state);
7745         }
7746
7747         free_extent_map(em);
7748
7749         return 0;
7750
7751 unlock_err:
7752         clear_extent_bit(&BTRFS_I(inode)->io_tree, lockstart, lockend,
7753                          unlock_bits, 1, 0, &cached_state);
7754 err:
7755         if (dio_data)
7756                 current->journal_info = dio_data;
7757         return ret;
7758 }
7759
7760 static inline blk_status_t submit_dio_repair_bio(struct inode *inode,
7761                                                  struct bio *bio,
7762                                                  int mirror_num)
7763 {
7764         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7765         blk_status_t ret;
7766
7767         BUG_ON(bio_op(bio) == REQ_OP_WRITE);
7768
7769         ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DIO_REPAIR);
7770         if (ret)
7771                 return ret;
7772
7773         ret = btrfs_map_bio(fs_info, bio, mirror_num, 0);
7774
7775         return ret;
7776 }
7777
7778 static int btrfs_check_dio_repairable(struct inode *inode,
7779                                       struct bio *failed_bio,
7780                                       struct io_failure_record *failrec,
7781                                       int failed_mirror)
7782 {
7783         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
7784         int num_copies;
7785
7786         num_copies = btrfs_num_copies(fs_info, failrec->logical, failrec->len);
7787         if (num_copies == 1) {
7788                 /*
7789                  * we only have a single copy of the data, so don't bother with
7790                  * all the retry and error correction code that follows. no
7791                  * matter what the error is, it is very likely to persist.
7792                  */
7793                 btrfs_debug(fs_info,
7794                         "Check DIO Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d",
7795                         num_copies, failrec->this_mirror, failed_mirror);
7796                 return 0;
7797         }
7798
7799         failrec->failed_mirror = failed_mirror;
7800         failrec->this_mirror++;
7801         if (failrec->this_mirror == failed_mirror)
7802                 failrec->this_mirror++;
7803
7804         if (failrec->this_mirror > num_copies) {
7805                 btrfs_debug(fs_info,
7806                         "Check DIO Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d",
7807                         num_copies, failrec->this_mirror, failed_mirror);
7808                 return 0;
7809         }
7810
7811         return 1;
7812 }
7813
7814 static blk_status_t dio_read_error(struct inode *inode, struct bio *failed_bio,
7815                                    struct page *page, unsigned int pgoff,
7816                                    u64 start, u64 end, int failed_mirror,
7817                                    bio_end_io_t *repair_endio, void *repair_arg)
7818 {
7819         struct io_failure_record *failrec;
7820         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
7821         struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
7822         struct bio *bio;
7823         int isector;
7824         unsigned int read_mode = 0;
7825         int segs;
7826         int ret;
7827         blk_status_t status;
7828         struct bio_vec bvec;
7829
7830         BUG_ON(bio_op(failed_bio) == REQ_OP_WRITE);
7831
7832         ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
7833         if (ret)
7834                 return errno_to_blk_status(ret);
7835
7836         ret = btrfs_check_dio_repairable(inode, failed_bio, failrec,
7837                                          failed_mirror);
7838         if (!ret) {
7839                 free_io_failure(failure_tree, io_tree, failrec);
7840                 return BLK_STS_IOERR;
7841         }
7842
7843         segs = bio_segments(failed_bio);
7844         bio_get_first_bvec(failed_bio, &bvec);
7845         if (segs > 1 ||
7846             (bvec.bv_len > btrfs_inode_sectorsize(inode)))
7847                 read_mode |= REQ_FAILFAST_DEV;
7848
7849         isector = start - btrfs_io_bio(failed_bio)->logical;
7850         isector >>= inode->i_sb->s_blocksize_bits;
7851         bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
7852                                 pgoff, isector, repair_endio, repair_arg);
7853         bio_set_op_attrs(bio, REQ_OP_READ, read_mode);
7854
7855         btrfs_debug(BTRFS_I(inode)->root->fs_info,
7856                     "repair DIO read error: submitting new dio read[%#x] to this_mirror=%d, in_validation=%d",
7857                     read_mode, failrec->this_mirror, failrec->in_validation);
7858
7859         status = submit_dio_repair_bio(inode, bio, failrec->this_mirror);
7860         if (status) {
7861                 free_io_failure(failure_tree, io_tree, failrec);
7862                 bio_put(bio);
7863         }
7864
7865         return status;
7866 }
7867
7868 struct btrfs_retry_complete {
7869         struct completion done;
7870         struct inode *inode;
7871         u64 start;
7872         int uptodate;
7873 };
7874
7875 static void btrfs_retry_endio_nocsum(struct bio *bio)
7876 {
7877         struct btrfs_retry_complete *done = bio->bi_private;
7878         struct inode *inode = done->inode;
7879         struct bio_vec *bvec;
7880         struct extent_io_tree *io_tree, *failure_tree;
7881         int i;
7882
7883         if (bio->bi_status)
7884                 goto end;
7885
7886         ASSERT(bio->bi_vcnt == 1);
7887         io_tree = &BTRFS_I(inode)->io_tree;
7888         failure_tree = &BTRFS_I(inode)->io_failure_tree;
7889         ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(inode));
7890
7891         done->uptodate = 1;
7892         ASSERT(!bio_flagged(bio, BIO_CLONED));
7893         bio_for_each_segment_all(bvec, bio, i)
7894                 clean_io_failure(BTRFS_I(inode)->root->fs_info, failure_tree,
7895                                  io_tree, done->start, bvec->bv_page,
7896                                  btrfs_ino(BTRFS_I(inode)), 0);
7897 end:
7898         complete(&done->done);
7899         bio_put(bio);
7900 }
7901
7902 static blk_status_t __btrfs_correct_data_nocsum(struct inode *inode,
7903                                                 struct btrfs_io_bio *io_bio)
7904 {
7905         struct btrfs_fs_info *fs_info;
7906         struct bio_vec bvec;
7907         struct bvec_iter iter;
7908         struct btrfs_retry_complete done;
7909         u64 start;
7910         unsigned int pgoff;
7911         u32 sectorsize;
7912         int nr_sectors;
7913         blk_status_t ret;
7914         blk_status_t err = BLK_STS_OK;
7915
7916         fs_info = BTRFS_I(inode)->root->fs_info;
7917         sectorsize = fs_info->sectorsize;
7918
7919         start = io_bio->logical;
7920         done.inode = inode;
7921         io_bio->bio.bi_iter = io_bio->iter;
7922
7923         bio_for_each_segment(bvec, &io_bio->bio, iter) {
7924                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
7925                 pgoff = bvec.bv_offset;
7926
7927 next_block_or_try_again:
7928                 done.uptodate = 0;
7929                 done.start = start;
7930                 init_completion(&done.done);
7931
7932                 ret = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
7933                                 pgoff, start, start + sectorsize - 1,
7934                                 io_bio->mirror_num,
7935                                 btrfs_retry_endio_nocsum, &done);
7936                 if (ret) {
7937                         err = ret;
7938                         goto next;
7939                 }
7940
7941                 wait_for_completion_io(&done.done);
7942
7943                 if (!done.uptodate) {
7944                         /* We might have another mirror, so try again */
7945                         goto next_block_or_try_again;
7946                 }
7947
7948 next:
7949                 start += sectorsize;
7950
7951                 nr_sectors--;
7952                 if (nr_sectors) {
7953                         pgoff += sectorsize;
7954                         ASSERT(pgoff < PAGE_SIZE);
7955                         goto next_block_or_try_again;
7956                 }
7957         }
7958
7959         return err;
7960 }
7961
7962 static void btrfs_retry_endio(struct bio *bio)
7963 {
7964         struct btrfs_retry_complete *done = bio->bi_private;
7965         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
7966         struct extent_io_tree *io_tree, *failure_tree;
7967         struct inode *inode = done->inode;
7968         struct bio_vec *bvec;
7969         int uptodate;
7970         int ret;
7971         int i;
7972
7973         if (bio->bi_status)
7974                 goto end;
7975
7976         uptodate = 1;
7977
7978         ASSERT(bio->bi_vcnt == 1);
7979         ASSERT(bio_first_bvec_all(bio)->bv_len == btrfs_inode_sectorsize(done->inode));
7980
7981         io_tree = &BTRFS_I(inode)->io_tree;
7982         failure_tree = &BTRFS_I(inode)->io_failure_tree;
7983
7984         ASSERT(!bio_flagged(bio, BIO_CLONED));
7985         bio_for_each_segment_all(bvec, bio, i) {
7986                 ret = __readpage_endio_check(inode, io_bio, i, bvec->bv_page,
7987                                              bvec->bv_offset, done->start,
7988                                              bvec->bv_len);
7989                 if (!ret)
7990                         clean_io_failure(BTRFS_I(inode)->root->fs_info,
7991                                          failure_tree, io_tree, done->start,
7992                                          bvec->bv_page,
7993                                          btrfs_ino(BTRFS_I(inode)),
7994                                          bvec->bv_offset);
7995                 else
7996                         uptodate = 0;
7997         }
7998
7999         done->uptodate = uptodate;
8000 end:
8001         complete(&done->done);
8002         bio_put(bio);
8003 }
8004
8005 static blk_status_t __btrfs_subio_endio_read(struct inode *inode,
8006                 struct btrfs_io_bio *io_bio, blk_status_t err)
8007 {
8008         struct btrfs_fs_info *fs_info;
8009         struct bio_vec bvec;
8010         struct bvec_iter iter;
8011         struct btrfs_retry_complete done;
8012         u64 start;
8013         u64 offset = 0;
8014         u32 sectorsize;
8015         int nr_sectors;
8016         unsigned int pgoff;
8017         int csum_pos;
8018         bool uptodate = (err == 0);
8019         int ret;
8020         blk_status_t status;
8021
8022         fs_info = BTRFS_I(inode)->root->fs_info;
8023         sectorsize = fs_info->sectorsize;
8024
8025         err = BLK_STS_OK;
8026         start = io_bio->logical;
8027         done.inode = inode;
8028         io_bio->bio.bi_iter = io_bio->iter;
8029
8030         bio_for_each_segment(bvec, &io_bio->bio, iter) {
8031                 nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
8032
8033                 pgoff = bvec.bv_offset;
8034 next_block:
8035                 if (uptodate) {
8036                         csum_pos = BTRFS_BYTES_TO_BLKS(fs_info, offset);
8037                         ret = __readpage_endio_check(inode, io_bio, csum_pos,
8038                                         bvec.bv_page, pgoff, start, sectorsize);
8039                         if (likely(!ret))
8040                                 goto next;
8041                 }
8042 try_again:
8043                 done.uptodate = 0;
8044                 done.start = start;
8045                 init_completion(&done.done);
8046
8047                 status = dio_read_error(inode, &io_bio->bio, bvec.bv_page,
8048                                         pgoff, start, start + sectorsize - 1,
8049                                         io_bio->mirror_num, btrfs_retry_endio,
8050                                         &done);
8051                 if (status) {
8052                         err = status;
8053                         goto next;
8054                 }
8055
8056                 wait_for_completion_io(&done.done);
8057
8058                 if (!done.uptodate) {
8059                         /* We might have another mirror, so try again */
8060                         goto try_again;
8061                 }
8062 next:
8063                 offset += sectorsize;
8064                 start += sectorsize;
8065
8066                 ASSERT(nr_sectors);
8067
8068                 nr_sectors--;
8069                 if (nr_sectors) {
8070                         pgoff += sectorsize;
8071                         ASSERT(pgoff < PAGE_SIZE);
8072                         goto next_block;
8073                 }
8074         }
8075
8076         return err;
8077 }
8078
8079 static blk_status_t btrfs_subio_endio_read(struct inode *inode,
8080                 struct btrfs_io_bio *io_bio, blk_status_t err)
8081 {
8082         bool skip_csum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
8083
8084         if (skip_csum) {
8085                 if (unlikely(err))
8086                         return __btrfs_correct_data_nocsum(inode, io_bio);
8087                 else
8088                         return BLK_STS_OK;
8089         } else {
8090                 return __btrfs_subio_endio_read(inode, io_bio, err);
8091         }
8092 }
8093
8094 static void btrfs_endio_direct_read(struct bio *bio)
8095 {
8096         struct btrfs_dio_private *dip = bio->bi_private;
8097         struct inode *inode = dip->inode;
8098         struct bio *dio_bio;
8099         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8100         blk_status_t err = bio->bi_status;
8101
8102         if (dip->flags & BTRFS_DIO_ORIG_BIO_SUBMITTED)
8103                 err = btrfs_subio_endio_read(inode, io_bio, err);
8104
8105         unlock_extent(&BTRFS_I(inode)->io_tree, dip->logical_offset,
8106                       dip->logical_offset + dip->bytes - 1);
8107         dio_bio = dip->dio_bio;
8108
8109         kfree(dip);
8110
8111         dio_bio->bi_status = err;
8112         dio_end_io(dio_bio);
8113
8114         if (io_bio->end_io)
8115                 io_bio->end_io(io_bio, blk_status_to_errno(err));
8116         bio_put(bio);
8117 }
8118
8119 static void __endio_write_update_ordered(struct inode *inode,
8120                                          const u64 offset, const u64 bytes,
8121                                          const bool uptodate)
8122 {
8123         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8124         struct btrfs_ordered_extent *ordered = NULL;
8125         struct btrfs_workqueue *wq;
8126         btrfs_work_func_t func;
8127         u64 ordered_offset = offset;
8128         u64 ordered_bytes = bytes;
8129         u64 last_offset;
8130         int ret;
8131
8132         if (btrfs_is_free_space_inode(BTRFS_I(inode))) {
8133                 wq = fs_info->endio_freespace_worker;
8134                 func = btrfs_freespace_write_helper;
8135         } else {
8136                 wq = fs_info->endio_write_workers;
8137                 func = btrfs_endio_write_helper;
8138         }
8139
8140 again:
8141         last_offset = ordered_offset;
8142         ret = btrfs_dec_test_first_ordered_pending(inode, &ordered,
8143                                                    &ordered_offset,
8144                                                    ordered_bytes,
8145                                                    uptodate);
8146         if (!ret)
8147                 goto out_test;
8148
8149         btrfs_init_work(&ordered->work, func, finish_ordered_fn, NULL, NULL);
8150         btrfs_queue_work(wq, &ordered->work);
8151 out_test:
8152         /*
8153          * If btrfs_dec_test_ordered_pending does not find any ordered extent
8154          * in the range, we can exit.
8155          */
8156         if (ordered_offset == last_offset)
8157                 return;
8158         /*
8159          * our bio might span multiple ordered extents.  If we haven't
8160          * completed the accounting for the whole dio, go back and try again
8161          */
8162         if (ordered_offset < offset + bytes) {
8163                 ordered_bytes = offset + bytes - ordered_offset;
8164                 ordered = NULL;
8165                 goto again;
8166         }
8167 }
8168
8169 static void btrfs_endio_direct_write(struct bio *bio)
8170 {
8171         struct btrfs_dio_private *dip = bio->bi_private;
8172         struct bio *dio_bio = dip->dio_bio;
8173
8174         __endio_write_update_ordered(dip->inode, dip->logical_offset,
8175                                      dip->bytes, !bio->bi_status);
8176
8177         kfree(dip);
8178
8179         dio_bio->bi_status = bio->bi_status;
8180         dio_end_io(dio_bio);
8181         bio_put(bio);
8182 }
8183
8184 static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data,
8185                                     struct bio *bio, u64 offset)
8186 {
8187         struct inode *inode = private_data;
8188         blk_status_t ret;
8189         ret = btrfs_csum_one_bio(inode, bio, offset, 1);
8190         BUG_ON(ret); /* -ENOMEM */
8191         return 0;
8192 }
8193
8194 static void btrfs_end_dio_bio(struct bio *bio)
8195 {
8196         struct btrfs_dio_private *dip = bio->bi_private;
8197         blk_status_t err = bio->bi_status;
8198
8199         if (err)
8200                 btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
8201                            "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
8202                            btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
8203                            bio->bi_opf,
8204                            (unsigned long long)bio->bi_iter.bi_sector,
8205                            bio->bi_iter.bi_size, err);
8206
8207         if (dip->subio_endio)
8208                 err = dip->subio_endio(dip->inode, btrfs_io_bio(bio), err);
8209
8210         if (err) {
8211                 /*
8212                  * We want to perceive the errors flag being set before
8213                  * decrementing the reference count. We don't need a barrier
8214                  * since atomic operations with a return value are fully
8215                  * ordered as per atomic_t.txt
8216                  */
8217                 dip->errors = 1;
8218         }
8219
8220         /* if there are more bios still pending for this dio, just exit */
8221         if (!atomic_dec_and_test(&dip->pending_bios))
8222                 goto out;
8223
8224         if (dip->errors) {
8225                 bio_io_error(dip->orig_bio);
8226         } else {
8227                 dip->dio_bio->bi_status = BLK_STS_OK;
8228                 bio_endio(dip->orig_bio);
8229         }
8230 out:
8231         bio_put(bio);
8232 }
8233
8234 static inline blk_status_t btrfs_lookup_and_bind_dio_csum(struct inode *inode,
8235                                                  struct btrfs_dio_private *dip,
8236                                                  struct bio *bio,
8237                                                  u64 file_offset)
8238 {
8239         struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
8240         struct btrfs_io_bio *orig_io_bio = btrfs_io_bio(dip->orig_bio);
8241         blk_status_t ret;
8242
8243         /*
8244          * We load all the csum data we need when we submit
8245          * the first bio to reduce the csum tree search and
8246          * contention.
8247          */
8248         if (dip->logical_offset == file_offset) {
8249                 ret = btrfs_lookup_bio_sums_dio(inode, dip->orig_bio,
8250                                                 file_offset);
8251                 if (ret)
8252                         return ret;
8253         }
8254
8255         if (bio == dip->orig_bio)
8256                 return 0;
8257
8258         file_offset -= dip->logical_offset;
8259         file_offset >>= inode->i_sb->s_blocksize_bits;
8260         io_bio->csum = (u8 *)(((u32 *)orig_io_bio->csum) + file_offset);
8261
8262         return 0;
8263 }
8264
8265 static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio,
8266                 struct inode *inode, u64 file_offset, int async_submit)
8267 {
8268         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8269         struct btrfs_dio_private *dip = bio->bi_private;
8270         bool write = bio_op(bio) == REQ_OP_WRITE;
8271         blk_status_t ret;
8272
8273         /* Check btrfs_submit_bio_hook() for rules about async submit. */
8274         if (async_submit)
8275                 async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
8276
8277         if (!write) {
8278                 ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
8279                 if (ret)
8280                         goto err;
8281         }
8282
8283         if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
8284                 goto map;
8285
8286         if (write && async_submit) {
8287                 ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
8288                                           file_offset, inode,
8289                                           btrfs_submit_bio_start_direct_io,
8290                                           btrfs_submit_bio_done);
8291                 goto err;
8292         } else if (write) {
8293                 /*
8294                  * If we aren't doing async submit, calculate the csum of the
8295                  * bio now.
8296                  */
8297                 ret = btrfs_csum_one_bio(inode, bio, file_offset, 1);
8298                 if (ret)
8299                         goto err;
8300         } else {
8301                 ret = btrfs_lookup_and_bind_dio_csum(inode, dip, bio,
8302                                                      file_offset);
8303                 if (ret)
8304                         goto err;
8305         }
8306 map:
8307         ret = btrfs_map_bio(fs_info, bio, 0, 0);
8308 err:
8309         return ret;
8310 }
8311
8312 static int btrfs_submit_direct_hook(struct btrfs_dio_private *dip)
8313 {
8314         struct inode *inode = dip->inode;
8315         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8316         struct bio *bio;
8317         struct bio *orig_bio = dip->orig_bio;
8318         u64 start_sector = orig_bio->bi_iter.bi_sector;
8319         u64 file_offset = dip->logical_offset;
8320         u64 map_length;
8321         int async_submit = 0;
8322         u64 submit_len;
8323         int clone_offset = 0;
8324         int clone_len;
8325         int ret;
8326         blk_status_t status;
8327
8328         map_length = orig_bio->bi_iter.bi_size;
8329         submit_len = map_length;
8330         ret = btrfs_map_block(fs_info, btrfs_op(orig_bio), start_sector << 9,
8331                               &map_length, NULL, 0);
8332         if (ret)
8333                 return -EIO;
8334
8335         if (map_length >= submit_len) {
8336                 bio = orig_bio;
8337                 dip->flags |= BTRFS_DIO_ORIG_BIO_SUBMITTED;
8338                 goto submit;
8339         }
8340
8341         /* async crcs make it difficult to collect full stripe writes. */
8342         if (btrfs_data_alloc_profile(fs_info) & BTRFS_BLOCK_GROUP_RAID56_MASK)
8343                 async_submit = 0;
8344         else
8345                 async_submit = 1;
8346
8347         /* bio split */
8348         ASSERT(map_length <= INT_MAX);
8349         atomic_inc(&dip->pending_bios);
8350         do {
8351                 clone_len = min_t(int, submit_len, map_length);
8352
8353                 /*
8354                  * This will never fail as it's passing GPF_NOFS and
8355                  * the allocation is backed by btrfs_bioset.
8356                  */
8357                 bio = btrfs_bio_clone_partial(orig_bio, clone_offset,
8358                                               clone_len);
8359                 bio->bi_private = dip;
8360                 bio->bi_end_io = btrfs_end_dio_bio;
8361                 btrfs_io_bio(bio)->logical = file_offset;
8362
8363                 ASSERT(submit_len >= clone_len);
8364                 submit_len -= clone_len;
8365                 if (submit_len == 0)
8366                         break;
8367
8368                 /*
8369                  * Increase the count before we submit the bio so we know
8370                  * the end IO handler won't happen before we increase the
8371                  * count. Otherwise, the dip might get freed before we're
8372                  * done setting it up.
8373                  */
8374                 atomic_inc(&dip->pending_bios);
8375
8376                 status = btrfs_submit_dio_bio(bio, inode, file_offset,
8377                                                 async_submit);
8378                 if (status) {
8379                         bio_put(bio);
8380                         atomic_dec(&dip->pending_bios);
8381                         goto out_err;
8382                 }
8383
8384                 clone_offset += clone_len;
8385                 start_sector += clone_len >> 9;
8386                 file_offset += clone_len;
8387
8388                 map_length = submit_len;
8389                 ret = btrfs_map_block(fs_info, btrfs_op(orig_bio),
8390                                       start_sector << 9, &map_length, NULL, 0);
8391                 if (ret)
8392                         goto out_err;
8393         } while (submit_len > 0);
8394
8395 submit:
8396         status = btrfs_submit_dio_bio(bio, inode, file_offset, async_submit);
8397         if (!status)
8398                 return 0;
8399
8400         bio_put(bio);
8401 out_err:
8402         dip->errors = 1;
8403         /*
8404          * Before atomic variable goto zero, we must  make sure dip->errors is
8405          * perceived to be set. This ordering is ensured by the fact that an
8406          * atomic operations with a return value are fully ordered as per
8407          * atomic_t.txt
8408          */
8409         if (atomic_dec_and_test(&dip->pending_bios))
8410                 bio_io_error(dip->orig_bio);
8411
8412         /* bio_end_io() will handle error, so we needn't return it */
8413         return 0;
8414 }
8415
8416 static void btrfs_submit_direct(struct bio *dio_bio, struct inode *inode,
8417                                 loff_t file_offset)
8418 {
8419         struct btrfs_dio_private *dip = NULL;
8420         struct bio *bio = NULL;
8421         struct btrfs_io_bio *io_bio;
8422         bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
8423         int ret = 0;
8424
8425         bio = btrfs_bio_clone(dio_bio);
8426
8427         dip = kzalloc(sizeof(*dip), GFP_NOFS);
8428         if (!dip) {
8429                 ret = -ENOMEM;
8430                 goto free_ordered;
8431         }
8432
8433         dip->private = dio_bio->bi_private;
8434         dip->inode = inode;
8435         dip->logical_offset = file_offset;
8436         dip->bytes = dio_bio->bi_iter.bi_size;
8437         dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
8438         bio->bi_private = dip;
8439         dip->orig_bio = bio;
8440         dip->dio_bio = dio_bio;
8441         atomic_set(&dip->pending_bios, 0);
8442         io_bio = btrfs_io_bio(bio);
8443         io_bio->logical = file_offset;
8444
8445         if (write) {
8446                 bio->bi_end_io = btrfs_endio_direct_write;
8447         } else {
8448                 bio->bi_end_io = btrfs_endio_direct_read;
8449                 dip->subio_endio = btrfs_subio_endio_read;
8450         }
8451
8452         /*
8453          * Reset the range for unsubmitted ordered extents (to a 0 length range)
8454          * even if we fail to submit a bio, because in such case we do the
8455          * corresponding error handling below and it must not be done a second
8456          * time by btrfs_direct_IO().
8457          */
8458         if (write) {
8459                 struct btrfs_dio_data *dio_data = current->journal_info;
8460
8461                 dio_data->unsubmitted_oe_range_end = dip->logical_offset +
8462                         dip->bytes;
8463                 dio_data->unsubmitted_oe_range_start =
8464                         dio_data->unsubmitted_oe_range_end;
8465         }
8466
8467         ret = btrfs_submit_direct_hook(dip);
8468         if (!ret)
8469                 return;
8470
8471         if (io_bio->end_io)
8472                 io_bio->end_io(io_bio, ret);
8473
8474 free_ordered:
8475         /*
8476          * If we arrived here it means either we failed to submit the dip
8477          * or we either failed to clone the dio_bio or failed to allocate the
8478          * dip. If we cloned the dio_bio and allocated the dip, we can just
8479          * call bio_endio against our io_bio so that we get proper resource
8480          * cleanup if we fail to submit the dip, otherwise, we must do the
8481          * same as btrfs_endio_direct_[write|read] because we can't call these
8482          * callbacks - they require an allocated dip and a clone of dio_bio.
8483          */
8484         if (bio && dip) {
8485                 bio_io_error(bio);
8486                 /*
8487                  * The end io callbacks free our dip, do the final put on bio
8488                  * and all the cleanup and final put for dio_bio (through
8489                  * dio_end_io()).
8490                  */
8491                 dip = NULL;
8492                 bio = NULL;
8493         } else {
8494                 if (write)
8495                         __endio_write_update_ordered(inode,
8496                                                 file_offset,
8497                                                 dio_bio->bi_iter.bi_size,
8498                                                 false);
8499                 else
8500                         unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
8501                               file_offset + dio_bio->bi_iter.bi_size - 1);
8502
8503                 dio_bio->bi_status = BLK_STS_IOERR;
8504                 /*
8505                  * Releases and cleans up our dio_bio, no need to bio_put()
8506                  * nor bio_endio()/bio_io_error() against dio_bio.
8507                  */
8508                 dio_end_io(dio_bio);
8509         }
8510         if (bio)
8511                 bio_put(bio);
8512         kfree(dip);
8513 }
8514
8515 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
8516                                const struct iov_iter *iter, loff_t offset)
8517 {
8518         int seg;
8519         int i;
8520         unsigned int blocksize_mask = fs_info->sectorsize - 1;
8521         ssize_t retval = -EINVAL;
8522
8523         if (offset & blocksize_mask)
8524                 goto out;
8525
8526         if (iov_iter_alignment(iter) & blocksize_mask)
8527                 goto out;
8528
8529         /* If this is a write we don't need to check anymore */
8530         if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
8531                 return 0;
8532         /*
8533          * Check to make sure we don't have duplicate iov_base's in this
8534          * iovec, if so return EINVAL, otherwise we'll get csum errors
8535          * when reading back.
8536          */
8537         for (seg = 0; seg < iter->nr_segs; seg++) {
8538                 for (i = seg + 1; i < iter->nr_segs; i++) {
8539                         if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
8540                                 goto out;
8541                 }
8542         }
8543         retval = 0;
8544 out:
8545         return retval;
8546 }
8547
8548 static ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
8549 {
8550         struct file *file = iocb->ki_filp;
8551         struct inode *inode = file->f_mapping->host;
8552         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8553         struct btrfs_dio_data dio_data = { 0 };
8554         struct extent_changeset *data_reserved = NULL;
8555         loff_t offset = iocb->ki_pos;
8556         size_t count = 0;
8557         int flags = 0;
8558         bool wakeup = true;
8559         bool relock = false;
8560         ssize_t ret;
8561
8562         if (check_direct_IO(fs_info, iter, offset))
8563                 return 0;
8564
8565         inode_dio_begin(inode);
8566
8567         /*
8568          * The generic stuff only does filemap_write_and_wait_range, which
8569          * isn't enough if we've written compressed pages to this area, so
8570          * we need to flush the dirty pages again to make absolutely sure
8571          * that any outstanding dirty pages are on disk.
8572          */
8573         count = iov_iter_count(iter);
8574         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
8575                      &BTRFS_I(inode)->runtime_flags))
8576                 filemap_fdatawrite_range(inode->i_mapping, offset,
8577                                          offset + count - 1);
8578
8579         if (iov_iter_rw(iter) == WRITE) {
8580                 /*
8581                  * If the write DIO is beyond the EOF, we need update
8582                  * the isize, but it is protected by i_mutex. So we can
8583                  * not unlock the i_mutex at this case.
8584                  */
8585                 if (offset + count <= inode->i_size) {
8586                         dio_data.overwrite = 1;
8587                         inode_unlock(inode);
8588                         relock = true;
8589                 } else if (iocb->ki_flags & IOCB_NOWAIT) {
8590                         ret = -EAGAIN;
8591                         goto out;
8592                 }
8593                 ret = btrfs_delalloc_reserve_space(inode, &data_reserved,
8594                                                    offset, count);
8595                 if (ret)
8596                         goto out;
8597
8598                 /*
8599                  * We need to know how many extents we reserved so that we can
8600                  * do the accounting properly if we go over the number we
8601                  * originally calculated.  Abuse current->journal_info for this.
8602                  */
8603                 dio_data.reserve = round_up(count,
8604                                             fs_info->sectorsize);
8605                 dio_data.unsubmitted_oe_range_start = (u64)offset;
8606                 dio_data.unsubmitted_oe_range_end = (u64)offset;
8607                 current->journal_info = &dio_data;
8608                 down_read(&BTRFS_I(inode)->dio_sem);
8609         } else if (test_bit(BTRFS_INODE_READDIO_NEED_LOCK,
8610                                      &BTRFS_I(inode)->runtime_flags)) {
8611                 inode_dio_end(inode);
8612                 flags = DIO_LOCKING | DIO_SKIP_HOLES;
8613                 wakeup = false;
8614         }
8615
8616         ret = __blockdev_direct_IO(iocb, inode,
8617                                    fs_info->fs_devices->latest_bdev,
8618                                    iter, btrfs_get_blocks_direct, NULL,
8619                                    btrfs_submit_direct, flags);
8620         if (iov_iter_rw(iter) == WRITE) {
8621                 up_read(&BTRFS_I(inode)->dio_sem);
8622                 current->journal_info = NULL;
8623                 if (ret < 0 && ret != -EIOCBQUEUED) {
8624                         if (dio_data.reserve)
8625                                 btrfs_delalloc_release_space(inode, data_reserved,
8626                                         offset, dio_data.reserve, true);
8627                         /*
8628                          * On error we might have left some ordered extents
8629                          * without submitting corresponding bios for them, so
8630                          * cleanup them up to avoid other tasks getting them
8631                          * and waiting for them to complete forever.
8632                          */
8633                         if (dio_data.unsubmitted_oe_range_start <
8634                             dio_data.unsubmitted_oe_range_end)
8635                                 __endio_write_update_ordered(inode,
8636                                         dio_data.unsubmitted_oe_range_start,
8637                                         dio_data.unsubmitted_oe_range_end -
8638                                         dio_data.unsubmitted_oe_range_start,
8639                                         false);
8640                 } else if (ret >= 0 && (size_t)ret < count)
8641                         btrfs_delalloc_release_space(inode, data_reserved,
8642                                         offset, count - (size_t)ret, true);
8643                 btrfs_delalloc_release_extents(BTRFS_I(inode), count, false);
8644         }
8645 out:
8646         if (wakeup)
8647                 inode_dio_end(inode);
8648         if (relock)
8649                 inode_lock(inode);
8650
8651         extent_changeset_free(data_reserved);
8652         return ret;
8653 }
8654
8655 #define BTRFS_FIEMAP_FLAGS      (FIEMAP_FLAG_SYNC)
8656
8657 static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
8658                 __u64 start, __u64 len)
8659 {
8660         int     ret;
8661
8662         ret = fiemap_check_flags(fieinfo, BTRFS_FIEMAP_FLAGS);
8663         if (ret)
8664                 return ret;
8665
8666         return extent_fiemap(inode, fieinfo, start, len);
8667 }
8668
8669 int btrfs_readpage(struct file *file, struct page *page)
8670 {
8671         struct extent_io_tree *tree;
8672         tree = &BTRFS_I(page->mapping->host)->io_tree;
8673         return extent_read_full_page(tree, page, btrfs_get_extent, 0);
8674 }
8675
8676 static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
8677 {
8678         struct inode *inode = page->mapping->host;
8679         int ret;
8680
8681         if (current->flags & PF_MEMALLOC) {
8682                 redirty_page_for_writepage(wbc, page);
8683                 unlock_page(page);
8684                 return 0;
8685         }
8686
8687         /*
8688          * If we are under memory pressure we will call this directly from the
8689          * VM, we need to make sure we have the inode referenced for the ordered
8690          * extent.  If not just return like we didn't do anything.
8691          */
8692         if (!igrab(inode)) {
8693                 redirty_page_for_writepage(wbc, page);
8694                 return AOP_WRITEPAGE_ACTIVATE;
8695         }
8696         ret = extent_write_full_page(page, wbc);
8697         btrfs_add_delayed_iput(inode);
8698         return ret;
8699 }
8700
8701 static int btrfs_writepages(struct address_space *mapping,
8702                             struct writeback_control *wbc)
8703 {
8704         struct extent_io_tree *tree;
8705
8706         tree = &BTRFS_I(mapping->host)->io_tree;
8707         return extent_writepages(tree, mapping, wbc);
8708 }
8709
8710 static int
8711 btrfs_readpages(struct file *file, struct address_space *mapping,
8712                 struct list_head *pages, unsigned nr_pages)
8713 {
8714         struct extent_io_tree *tree;
8715         tree = &BTRFS_I(mapping->host)->io_tree;
8716         return extent_readpages(tree, mapping, pages, nr_pages);
8717 }
8718 static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8719 {
8720         struct extent_io_tree *tree;
8721         struct extent_map_tree *map;
8722         int ret;
8723
8724         tree = &BTRFS_I(page->mapping->host)->io_tree;
8725         map = &BTRFS_I(page->mapping->host)->extent_tree;
8726         ret = try_release_extent_mapping(map, tree, page, gfp_flags);
8727         if (ret == 1) {
8728                 ClearPagePrivate(page);
8729                 set_page_private(page, 0);
8730                 put_page(page);
8731         }
8732         return ret;
8733 }
8734
8735 static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
8736 {
8737         if (PageWriteback(page) || PageDirty(page))
8738                 return 0;
8739         return __btrfs_releasepage(page, gfp_flags);
8740 }
8741
8742 static void btrfs_invalidatepage(struct page *page, unsigned int offset,
8743                                  unsigned int length)
8744 {
8745         struct inode *inode = page->mapping->host;
8746         struct extent_io_tree *tree;
8747         struct btrfs_ordered_extent *ordered;
8748         struct extent_state *cached_state = NULL;
8749         u64 page_start = page_offset(page);
8750         u64 page_end = page_start + PAGE_SIZE - 1;
8751         u64 start;
8752         u64 end;
8753         int inode_evicting = inode->i_state & I_FREEING;
8754
8755         /*
8756          * we have the page locked, so new writeback can't start,
8757          * and the dirty bit won't be cleared while we are here.
8758          *
8759          * Wait for IO on this page so that we can safely clear
8760          * the PagePrivate2 bit and do ordered accounting
8761          */
8762         wait_on_page_writeback(page);
8763
8764         tree = &BTRFS_I(inode)->io_tree;
8765         if (offset) {
8766                 btrfs_releasepage(page, GFP_NOFS);
8767                 return;
8768         }
8769
8770         if (!inode_evicting)
8771                 lock_extent_bits(tree, page_start, page_end, &cached_state);
8772 again:
8773         start = page_start;
8774         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), start,
8775                                         page_end - start + 1);
8776         if (ordered) {
8777                 end = min(page_end, ordered->file_offset + ordered->len - 1);
8778                 /*
8779                  * IO on this page will never be started, so we need
8780                  * to account for any ordered extents now
8781                  */
8782                 if (!inode_evicting)
8783                         clear_extent_bit(tree, start, end,
8784                                          EXTENT_DIRTY | EXTENT_DELALLOC |
8785                                          EXTENT_DELALLOC_NEW |
8786                                          EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
8787                                          EXTENT_DEFRAG, 1, 0, &cached_state);
8788                 /*
8789                  * whoever cleared the private bit is responsible
8790                  * for the finish_ordered_io
8791                  */
8792                 if (TestClearPagePrivate2(page)) {
8793                         struct btrfs_ordered_inode_tree *tree;
8794                         u64 new_len;
8795
8796                         tree = &BTRFS_I(inode)->ordered_tree;
8797
8798                         spin_lock_irq(&tree->lock);
8799                         set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
8800                         new_len = start - ordered->file_offset;
8801                         if (new_len < ordered->truncated_len)
8802                                 ordered->truncated_len = new_len;
8803                         spin_unlock_irq(&tree->lock);
8804
8805                         if (btrfs_dec_test_ordered_pending(inode, &ordered,
8806                                                            start,
8807                                                            end - start + 1, 1))
8808                                 btrfs_finish_ordered_io(ordered);
8809                 }
8810                 btrfs_put_ordered_extent(ordered);
8811                 if (!inode_evicting) {
8812                         cached_state = NULL;
8813                         lock_extent_bits(tree, start, end,
8814                                          &cached_state);
8815                 }
8816
8817                 start = end + 1;
8818                 if (start < page_end)
8819                         goto again;
8820         }
8821
8822         /*
8823          * Qgroup reserved space handler
8824          * Page here will be either
8825          * 1) Already written to disk
8826          *    In this case, its reserved space is released from data rsv map
8827          *    and will be freed by delayed_ref handler finally.
8828          *    So even we call qgroup_free_data(), it won't decrease reserved
8829          *    space.
8830          * 2) Not written to disk
8831          *    This means the reserved space should be freed here. However,
8832          *    if a truncate invalidates the page (by clearing PageDirty)
8833          *    and the page is accounted for while allocating extent
8834          *    in btrfs_check_data_free_space() we let delayed_ref to
8835          *    free the entire extent.
8836          */
8837         if (PageDirty(page))
8838                 btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
8839         if (!inode_evicting) {
8840                 clear_extent_bit(tree, page_start, page_end,
8841                                  EXTENT_LOCKED | EXTENT_DIRTY |
8842                                  EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
8843                                  EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
8844                                  &cached_state);
8845
8846                 __btrfs_releasepage(page, GFP_NOFS);
8847         }
8848
8849         ClearPageChecked(page);
8850         if (PagePrivate(page)) {
8851                 ClearPagePrivate(page);
8852                 set_page_private(page, 0);
8853                 put_page(page);
8854         }
8855 }
8856
8857 /*
8858  * btrfs_page_mkwrite() is not allowed to change the file size as it gets
8859  * called from a page fault handler when a page is first dirtied. Hence we must
8860  * be careful to check for EOF conditions here. We set the page up correctly
8861  * for a written page which means we get ENOSPC checking when writing into
8862  * holes and correct delalloc and unwritten extent mapping on filesystems that
8863  * support these features.
8864  *
8865  * We are not allowed to take the i_mutex here so we have to play games to
8866  * protect against truncate races as the page could now be beyond EOF.  Because
8867  * vmtruncate() writes the inode size before removing pages, once we have the
8868  * page lock we can determine safely if the page is beyond EOF. If it is not
8869  * beyond EOF, then the page is guaranteed safe against truncation until we
8870  * unlock the page.
8871  */
8872 int btrfs_page_mkwrite(struct vm_fault *vmf)
8873 {
8874         struct page *page = vmf->page;
8875         struct inode *inode = file_inode(vmf->vma->vm_file);
8876         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
8877         struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
8878         struct btrfs_ordered_extent *ordered;
8879         struct extent_state *cached_state = NULL;
8880         struct extent_changeset *data_reserved = NULL;
8881         char *kaddr;
8882         unsigned long zero_start;
8883         loff_t size;
8884         int ret;
8885         int reserved = 0;
8886         u64 reserved_space;
8887         u64 page_start;
8888         u64 page_end;
8889         u64 end;
8890
8891         reserved_space = PAGE_SIZE;
8892
8893         sb_start_pagefault(inode->i_sb);
8894         page_start = page_offset(page);
8895         page_end = page_start + PAGE_SIZE - 1;
8896         end = page_end;
8897
8898         /*
8899          * Reserving delalloc space after obtaining the page lock can lead to
8900          * deadlock. For example, if a dirty page is locked by this function
8901          * and the call to btrfs_delalloc_reserve_space() ends up triggering
8902          * dirty page write out, then the btrfs_writepage() function could
8903          * end up waiting indefinitely to get a lock on the page currently
8904          * being processed by btrfs_page_mkwrite() function.
8905          */
8906         ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
8907                                            reserved_space);
8908         if (!ret) {
8909                 ret = file_update_time(vmf->vma->vm_file);
8910                 reserved = 1;
8911         }
8912         if (ret) {
8913                 if (ret == -ENOMEM)
8914                         ret = VM_FAULT_OOM;
8915                 else /* -ENOSPC, -EIO, etc */
8916                         ret = VM_FAULT_SIGBUS;
8917                 if (reserved)
8918                         goto out;
8919                 goto out_noreserve;
8920         }
8921
8922         ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
8923 again:
8924         lock_page(page);
8925         size = i_size_read(inode);
8926
8927         if ((page->mapping != inode->i_mapping) ||
8928             (page_start >= size)) {
8929                 /* page got truncated out from underneath us */
8930                 goto out_unlock;
8931         }
8932         wait_on_page_writeback(page);
8933
8934         lock_extent_bits(io_tree, page_start, page_end, &cached_state);
8935         set_page_extent_mapped(page);
8936
8937         /*
8938          * we can't set the delalloc bits if there are pending ordered
8939          * extents.  Drop our locks and wait for them to finish
8940          */
8941         ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
8942                         PAGE_SIZE);
8943         if (ordered) {
8944                 unlock_extent_cached(io_tree, page_start, page_end,
8945                                      &cached_state);
8946                 unlock_page(page);
8947                 btrfs_start_ordered_extent(inode, ordered, 1);
8948                 btrfs_put_ordered_extent(ordered);
8949                 goto again;
8950         }
8951
8952         if (page->index == ((size - 1) >> PAGE_SHIFT)) {
8953                 reserved_space = round_up(size - page_start,
8954                                           fs_info->sectorsize);
8955                 if (reserved_space < PAGE_SIZE) {
8956                         end = page_start + reserved_space - 1;
8957                         btrfs_delalloc_release_space(inode, data_reserved,
8958                                         page_start, PAGE_SIZE - reserved_space,
8959                                         true);
8960                 }
8961         }
8962
8963         /*
8964          * page_mkwrite gets called when the page is firstly dirtied after it's
8965          * faulted in, but write(2) could also dirty a page and set delalloc
8966          * bits, thus in this case for space account reason, we still need to
8967          * clear any delalloc bits within this page range since we have to
8968          * reserve data&meta space before lock_page() (see above comments).
8969          */
8970         clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
8971                           EXTENT_DIRTY | EXTENT_DELALLOC |
8972                           EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
8973                           0, 0, &cached_state);
8974
8975         ret = btrfs_set_extent_delalloc(inode, page_start, end, 0,
8976                                         &cached_state, 0);
8977         if (ret) {
8978                 unlock_extent_cached(io_tree, page_start, page_end,
8979                                      &cached_state);
8980                 ret = VM_FAULT_SIGBUS;
8981                 goto out_unlock;
8982         }
8983         ret = 0;
8984
8985         /* page is wholly or partially inside EOF */
8986         if (page_start + PAGE_SIZE > size)
8987                 zero_start = size & ~PAGE_MASK;
8988         else
8989                 zero_start = PAGE_SIZE;
8990
8991         if (zero_start != PAGE_SIZE) {
8992                 kaddr = kmap(page);
8993                 memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
8994                 flush_dcache_page(page);
8995                 kunmap(page);
8996         }
8997         ClearPageChecked(page);
8998         set_page_dirty(page);
8999         SetPageUptodate(page);
9000
9001         BTRFS_I(inode)->last_trans = fs_info->generation;
9002         BTRFS_I(inode)->last_sub_trans = BTRFS_I(inode)->root->log_transid;
9003         BTRFS_I(inode)->last_log_commit = BTRFS_I(inode)->root->last_log_commit;
9004
9005         unlock_extent_cached(io_tree, page_start, page_end, &cached_state);
9006
9007 out_unlock:
9008         if (!ret) {
9009                 btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, true);
9010                 sb_end_pagefault(inode->i_sb);
9011                 extent_changeset_free(data_reserved);
9012                 return VM_FAULT_LOCKED;
9013         }
9014         unlock_page(page);
9015 out:
9016         btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE, (ret != 0));
9017         btrfs_delalloc_release_space(inode, data_reserved, page_start,
9018                                      reserved_space, (ret != 0));
9019 out_noreserve:
9020         sb_end_pagefault(inode->i_sb);
9021         extent_changeset_free(data_reserved);
9022         return ret;
9023 }
9024
9025 static int btrfs_truncate(struct inode *inode, bool skip_writeback)
9026 {
9027         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9028         struct btrfs_root *root = BTRFS_I(inode)->root;
9029         struct btrfs_block_rsv *rsv;
9030         int ret = 0;
9031         int err = 0;
9032         struct btrfs_trans_handle *trans;
9033         u64 mask = fs_info->sectorsize - 1;
9034         u64 min_size = btrfs_calc_trunc_metadata_size(fs_info, 1);
9035
9036         if (!skip_writeback) {
9037                 ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
9038                                                (u64)-1);
9039                 if (ret)
9040                         return ret;
9041         }
9042
9043         /*
9044          * Yes ladies and gentlemen, this is indeed ugly.  The fact is we have
9045          * 3 things going on here
9046          *
9047          * 1) We need to reserve space for our orphan item and the space to
9048          * delete our orphan item.  Lord knows we don't want to have a dangling
9049          * orphan item because we didn't reserve space to remove it.
9050          *
9051          * 2) We need to reserve space to update our inode.
9052          *
9053          * 3) We need to have something to cache all the space that is going to
9054          * be free'd up by the truncate operation, but also have some slack
9055          * space reserved in case it uses space during the truncate (thank you
9056          * very much snapshotting).
9057          *
9058          * And we need these to all be separate.  The fact is we can use a lot of
9059          * space doing the truncate, and we have no earthly idea how much space
9060          * we will use, so we need the truncate reservation to be separate so it
9061          * doesn't end up using space reserved for updating the inode or
9062          * removing the orphan item.  We also need to be able to stop the
9063          * transaction and start a new one, which means we need to be able to
9064          * update the inode several times, and we have no idea of knowing how
9065          * many times that will be, so we can't just reserve 1 item for the
9066          * entirety of the operation, so that has to be done separately as well.
9067          * Then there is the orphan item, which does indeed need to be held on
9068          * to for the whole operation, and we need nobody to touch this reserved
9069          * space except the orphan code.
9070          *
9071          * So that leaves us with
9072          *
9073          * 1) root->orphan_block_rsv - for the orphan deletion.
9074          * 2) rsv - for the truncate reservation, which we will steal from the
9075          * transaction reservation.
9076          * 3) fs_info->trans_block_rsv - this will have 1 items worth left for
9077          * updating the inode.
9078          */
9079         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
9080         if (!rsv)
9081                 return -ENOMEM;
9082         rsv->size = min_size;
9083         rsv->failfast = 1;
9084
9085         /*
9086          * 1 for the truncate slack space
9087          * 1 for updating the inode.
9088          */
9089         trans = btrfs_start_transaction(root, 2);
9090         if (IS_ERR(trans)) {
9091                 err = PTR_ERR(trans);
9092                 goto out;
9093         }
9094
9095         /* Migrate the slack space for the truncate to our reserve */
9096         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
9097                                       min_size, 0);
9098         BUG_ON(ret);
9099
9100         /*
9101          * So if we truncate and then write and fsync we normally would just
9102          * write the extents that changed, which is a problem if we need to
9103          * first truncate that entire inode.  So set this flag so we write out
9104          * all of the extents in the inode to the sync log so we're completely
9105          * safe.
9106          */
9107         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
9108         trans->block_rsv = rsv;
9109
9110         while (1) {
9111                 ret = btrfs_truncate_inode_items(trans, root, inode,
9112                                                  inode->i_size,
9113                                                  BTRFS_EXTENT_DATA_KEY);
9114                 trans->block_rsv = &fs_info->trans_block_rsv;
9115                 if (ret != -ENOSPC && ret != -EAGAIN) {
9116                         err = ret;
9117                         break;
9118                 }
9119
9120                 ret = btrfs_update_inode(trans, root, inode);
9121                 if (ret) {
9122                         err = ret;
9123                         break;
9124                 }
9125
9126                 btrfs_end_transaction(trans);
9127                 btrfs_btree_balance_dirty(fs_info);
9128
9129                 trans = btrfs_start_transaction(root, 2);
9130                 if (IS_ERR(trans)) {
9131                         ret = err = PTR_ERR(trans);
9132                         trans = NULL;
9133                         break;
9134                 }
9135
9136                 btrfs_block_rsv_release(fs_info, rsv, -1);
9137                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
9138                                               rsv, min_size, 0);
9139                 BUG_ON(ret);    /* shouldn't happen */
9140                 trans->block_rsv = rsv;
9141         }
9142
9143         /*
9144          * We can't call btrfs_truncate_block inside a trans handle as we could
9145          * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
9146          * we've truncated everything except the last little bit, and can do
9147          * btrfs_truncate_block and then update the disk_i_size.
9148          */
9149         if (ret == NEED_TRUNCATE_BLOCK) {
9150                 btrfs_end_transaction(trans);
9151                 btrfs_btree_balance_dirty(fs_info);
9152
9153                 ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
9154                 if (ret)
9155                         goto out;
9156                 trans = btrfs_start_transaction(root, 1);
9157                 if (IS_ERR(trans)) {
9158                         ret = PTR_ERR(trans);
9159                         goto out;
9160                 }
9161                 btrfs_ordered_update_i_size(inode, inode->i_size, NULL);
9162         }
9163
9164         if (ret == 0 && inode->i_nlink > 0) {
9165                 trans->block_rsv = root->orphan_block_rsv;
9166                 ret = btrfs_orphan_del(trans, BTRFS_I(inode));
9167                 if (ret)
9168                         err = ret;
9169         }
9170
9171         if (trans) {
9172                 trans->block_rsv = &fs_info->trans_block_rsv;
9173                 ret = btrfs_update_inode(trans, root, inode);
9174                 if (ret && !err)
9175                         err = ret;
9176
9177                 ret = btrfs_end_transaction(trans);
9178                 btrfs_btree_balance_dirty(fs_info);
9179         }
9180 out:
9181         btrfs_free_block_rsv(fs_info, rsv);
9182
9183         if (ret && !err)
9184                 err = ret;
9185
9186         return err;
9187 }
9188
9189 /*
9190  * create a new subvolume directory/inode (helper for the ioctl).
9191  */
9192 int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
9193                              struct btrfs_root *new_root,
9194                              struct btrfs_root *parent_root,
9195                              u64 new_dirid)
9196 {
9197         struct inode *inode;
9198         int err;
9199         u64 index = 0;
9200
9201         inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
9202                                 new_dirid, new_dirid,
9203                                 S_IFDIR | (~current_umask() & S_IRWXUGO),
9204                                 &index);
9205         if (IS_ERR(inode))
9206                 return PTR_ERR(inode);
9207         inode->i_op = &btrfs_dir_inode_operations;
9208         inode->i_fop = &btrfs_dir_file_operations;
9209
9210         set_nlink(inode, 1);
9211         btrfs_i_size_write(BTRFS_I(inode), 0);
9212         unlock_new_inode(inode);
9213
9214         err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
9215         if (err)
9216                 btrfs_err(new_root->fs_info,
9217                           "error inheriting subvolume %llu properties: %d",
9218                           new_root->root_key.objectid, err);
9219
9220         err = btrfs_update_inode(trans, new_root, inode);
9221
9222         iput(inode);
9223         return err;
9224 }
9225
9226 struct inode *btrfs_alloc_inode(struct super_block *sb)
9227 {
9228         struct btrfs_fs_info *fs_info = btrfs_sb(sb);
9229         struct btrfs_inode *ei;
9230         struct inode *inode;
9231
9232         ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL);
9233         if (!ei)
9234                 return NULL;
9235
9236         ei->root = NULL;
9237         ei->generation = 0;
9238         ei->last_trans = 0;
9239         ei->last_sub_trans = 0;
9240         ei->logged_trans = 0;
9241         ei->delalloc_bytes = 0;
9242         ei->new_delalloc_bytes = 0;
9243         ei->defrag_bytes = 0;
9244         ei->disk_i_size = 0;
9245         ei->flags = 0;
9246         ei->csum_bytes = 0;
9247         ei->index_cnt = (u64)-1;
9248         ei->dir_index = 0;
9249         ei->last_unlink_trans = 0;
9250         ei->last_log_commit = 0;
9251
9252         spin_lock_init(&ei->lock);
9253         ei->outstanding_extents = 0;
9254         if (sb->s_magic != BTRFS_TEST_MAGIC)
9255                 btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
9256                                               BTRFS_BLOCK_RSV_DELALLOC);
9257         ei->runtime_flags = 0;
9258         ei->prop_compress = BTRFS_COMPRESS_NONE;
9259         ei->defrag_compress = BTRFS_COMPRESS_NONE;
9260
9261         ei->delayed_node = NULL;
9262
9263         ei->i_otime.tv_sec = 0;
9264         ei->i_otime.tv_nsec = 0;
9265
9266         inode = &ei->vfs_inode;
9267         extent_map_tree_init(&ei->extent_tree);
9268         extent_io_tree_init(&ei->io_tree, inode);
9269         extent_io_tree_init(&ei->io_failure_tree, inode);
9270         ei->io_tree.track_uptodate = 1;
9271         ei->io_failure_tree.track_uptodate = 1;
9272         atomic_set(&ei->sync_writers, 0);
9273         mutex_init(&ei->log_mutex);
9274         mutex_init(&ei->delalloc_mutex);
9275         btrfs_ordered_inode_tree_init(&ei->ordered_tree);
9276         INIT_LIST_HEAD(&ei->delalloc_inodes);
9277         INIT_LIST_HEAD(&ei->delayed_iput);
9278         RB_CLEAR_NODE(&ei->rb_node);
9279         init_rwsem(&ei->dio_sem);
9280
9281         return inode;
9282 }
9283
9284 #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
9285 void btrfs_test_destroy_inode(struct inode *inode)
9286 {
9287         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9288         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9289 }
9290 #endif
9291
9292 static void btrfs_i_callback(struct rcu_head *head)
9293 {
9294         struct inode *inode = container_of(head, struct inode, i_rcu);
9295         kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
9296 }
9297
9298 void btrfs_destroy_inode(struct inode *inode)
9299 {
9300         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
9301         struct btrfs_ordered_extent *ordered;
9302         struct btrfs_root *root = BTRFS_I(inode)->root;
9303
9304         WARN_ON(!hlist_empty(&inode->i_dentry));
9305         WARN_ON(inode->i_data.nrpages);
9306         WARN_ON(BTRFS_I(inode)->block_rsv.reserved);
9307         WARN_ON(BTRFS_I(inode)->block_rsv.size);
9308         WARN_ON(BTRFS_I(inode)->outstanding_extents);
9309         WARN_ON(BTRFS_I(inode)->delalloc_bytes);
9310         WARN_ON(BTRFS_I(inode)->new_delalloc_bytes);
9311         WARN_ON(BTRFS_I(inode)->csum_bytes);
9312         WARN_ON(BTRFS_I(inode)->defrag_bytes);
9313
9314         /*
9315          * This can happen where we create an inode, but somebody else also
9316          * created the same inode and we need to destroy the one we already
9317          * created.
9318          */
9319         if (!root)
9320                 goto free;
9321
9322         if (test_bit(BTRFS_INODE_HAS_ORPHAN_ITEM,
9323                      &BTRFS_I(inode)->runtime_flags)) {
9324                 btrfs_info(fs_info, "inode %llu still on the orphan list",
9325                            btrfs_ino(BTRFS_I(inode)));
9326                 atomic_dec(&root->orphan_inodes);
9327         }
9328
9329         while (1) {
9330                 ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
9331                 if (!ordered)
9332                         break;
9333                 else {
9334                         btrfs_err(fs_info,
9335                                   "found ordered extent %llu %llu on inode cleanup",
9336                                   ordered->file_offset, ordered->len);
9337                         btrfs_remove_ordered_extent(inode, ordered);
9338                         btrfs_put_ordered_extent(ordered);
9339                         btrfs_put_ordered_extent(ordered);
9340                 }
9341         }
9342         btrfs_qgroup_check_reserved_leak(inode);
9343         inode_tree_del(inode);
9344         btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
9345 free:
9346         call_rcu(&inode->i_rcu, btrfs_i_callback);
9347 }
9348
9349 int btrfs_drop_inode(struct inode *inode)
9350 {
9351         struct btrfs_root *root = BTRFS_I(inode)->root;
9352
9353         if (root == NULL)
9354                 return 1;
9355
9356         /* the snap/subvol tree is on deleting */
9357         if (btrfs_root_refs(&root->root_item) == 0)
9358                 return 1;
9359         else
9360                 return generic_drop_inode(inode);
9361 }
9362
9363 static void init_once(void *foo)
9364 {
9365         struct btrfs_inode *ei = (struct btrfs_inode *) foo;
9366
9367         inode_init_once(&ei->vfs_inode);
9368 }
9369
9370 void __cold btrfs_destroy_cachep(void)
9371 {
9372         /*
9373          * Make sure all delayed rcu free inodes are flushed before we
9374          * destroy cache.
9375          */
9376         rcu_barrier();
9377         kmem_cache_destroy(btrfs_inode_cachep);
9378         kmem_cache_destroy(btrfs_trans_handle_cachep);
9379         kmem_cache_destroy(btrfs_path_cachep);
9380         kmem_cache_destroy(btrfs_free_space_cachep);
9381 }
9382
9383 int __init btrfs_init_cachep(void)
9384 {
9385         btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
9386                         sizeof(struct btrfs_inode), 0,
9387                         SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
9388                         init_once);
9389         if (!btrfs_inode_cachep)
9390                 goto fail;
9391
9392         btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
9393                         sizeof(struct btrfs_trans_handle), 0,
9394                         SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
9395         if (!btrfs_trans_handle_cachep)
9396                 goto fail;
9397
9398         btrfs_path_cachep = kmem_cache_create("btrfs_path",
9399                         sizeof(struct btrfs_path), 0,
9400                         SLAB_MEM_SPREAD, NULL);
9401         if (!btrfs_path_cachep)
9402                 goto fail;
9403
9404         btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
9405                         sizeof(struct btrfs_free_space), 0,
9406                         SLAB_MEM_SPREAD, NULL);
9407         if (!btrfs_free_space_cachep)
9408                 goto fail;
9409
9410         return 0;
9411 fail:
9412         btrfs_destroy_cachep();
9413         return -ENOMEM;
9414 }
9415
9416 static int btrfs_getattr(const struct path *path, struct kstat *stat,
9417                          u32 request_mask, unsigned int flags)
9418 {
9419         u64 delalloc_bytes;
9420         struct inode *inode = d_inode(path->dentry);
9421         u32 blocksize = inode->i_sb->s_blocksize;
9422         u32 bi_flags = BTRFS_I(inode)->flags;
9423
9424         stat->result_mask |= STATX_BTIME;
9425         stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
9426         stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
9427         if (bi_flags & BTRFS_INODE_APPEND)
9428                 stat->attributes |= STATX_ATTR_APPEND;
9429         if (bi_flags & BTRFS_INODE_COMPRESS)
9430                 stat->attributes |= STATX_ATTR_COMPRESSED;
9431         if (bi_flags & BTRFS_INODE_IMMUTABLE)
9432                 stat->attributes |= STATX_ATTR_IMMUTABLE;
9433         if (bi_flags & BTRFS_INODE_NODUMP)
9434                 stat->attributes |= STATX_ATTR_NODUMP;
9435
9436         stat->attributes_mask |= (STATX_ATTR_APPEND |
9437                                   STATX_ATTR_COMPRESSED |
9438                                   STATX_ATTR_IMMUTABLE |
9439                                   STATX_ATTR_NODUMP);
9440
9441         generic_fillattr(inode, stat);
9442         stat->dev = BTRFS_I(inode)->root->anon_dev;
9443
9444         spin_lock(&BTRFS_I(inode)->lock);
9445         delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
9446         spin_unlock(&BTRFS_I(inode)->lock);
9447         stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
9448                         ALIGN(delalloc_bytes, blocksize)) >> 9;
9449         return 0;
9450 }
9451
9452 static int btrfs_rename_exchange(struct inode *old_dir,
9453                               struct dentry *old_dentry,
9454                               struct inode *new_dir,
9455                               struct dentry *new_dentry)
9456 {
9457         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9458         struct btrfs_trans_handle *trans;
9459         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9460         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9461         struct inode *new_inode = new_dentry->d_inode;
9462         struct inode *old_inode = old_dentry->d_inode;
9463         struct timespec ctime = current_time(old_inode);
9464         struct dentry *parent;
9465         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9466         u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
9467         u64 old_idx = 0;
9468         u64 new_idx = 0;
9469         u64 root_objectid;
9470         int ret;
9471         bool root_log_pinned = false;
9472         bool dest_log_pinned = false;
9473
9474         /* we only allow rename subvolume link between subvolumes */
9475         if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9476                 return -EXDEV;
9477
9478         /* close the race window with snapshot create/destroy ioctl */
9479         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9480                 down_read(&fs_info->subvol_sem);
9481         if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9482                 down_read(&fs_info->subvol_sem);
9483
9484         /*
9485          * We want to reserve the absolute worst case amount of items.  So if
9486          * both inodes are subvols and we need to unlink them then that would
9487          * require 4 item modifications, but if they are both normal inodes it
9488          * would require 5 item modifications, so we'll assume their normal
9489          * inodes.  So 5 * 2 is 10, plus 2 for the new links, so 12 total items
9490          * should cover the worst case number of items we'll modify.
9491          */
9492         trans = btrfs_start_transaction(root, 12);
9493         if (IS_ERR(trans)) {
9494                 ret = PTR_ERR(trans);
9495                 goto out_notrans;
9496         }
9497
9498         /*
9499          * We need to find a free sequence number both in the source and
9500          * in the destination directory for the exchange.
9501          */
9502         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
9503         if (ret)
9504                 goto out_fail;
9505         ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
9506         if (ret)
9507                 goto out_fail;
9508
9509         BTRFS_I(old_inode)->dir_index = 0ULL;
9510         BTRFS_I(new_inode)->dir_index = 0ULL;
9511
9512         /* Reference for the source. */
9513         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9514                 /* force full log commit if subvolume involved. */
9515                 btrfs_set_log_full_commit(fs_info, trans);
9516         } else {
9517                 btrfs_pin_log_trans(root);
9518                 root_log_pinned = true;
9519                 ret = btrfs_insert_inode_ref(trans, dest,
9520                                              new_dentry->d_name.name,
9521                                              new_dentry->d_name.len,
9522                                              old_ino,
9523                                              btrfs_ino(BTRFS_I(new_dir)),
9524                                              old_idx);
9525                 if (ret)
9526                         goto out_fail;
9527         }
9528
9529         /* And now for the dest. */
9530         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9531                 /* force full log commit if subvolume involved. */
9532                 btrfs_set_log_full_commit(fs_info, trans);
9533         } else {
9534                 btrfs_pin_log_trans(dest);
9535                 dest_log_pinned = true;
9536                 ret = btrfs_insert_inode_ref(trans, root,
9537                                              old_dentry->d_name.name,
9538                                              old_dentry->d_name.len,
9539                                              new_ino,
9540                                              btrfs_ino(BTRFS_I(old_dir)),
9541                                              new_idx);
9542                 if (ret)
9543                         goto out_fail;
9544         }
9545
9546         /* Update inode version and ctime/mtime. */
9547         inode_inc_iversion(old_dir);
9548         inode_inc_iversion(new_dir);
9549         inode_inc_iversion(old_inode);
9550         inode_inc_iversion(new_inode);
9551         old_dir->i_ctime = old_dir->i_mtime = ctime;
9552         new_dir->i_ctime = new_dir->i_mtime = ctime;
9553         old_inode->i_ctime = ctime;
9554         new_inode->i_ctime = ctime;
9555
9556         if (old_dentry->d_parent != new_dentry->d_parent) {
9557                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
9558                                 BTRFS_I(old_inode), 1);
9559                 btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
9560                                 BTRFS_I(new_inode), 1);
9561         }
9562
9563         /* src is a subvolume */
9564         if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
9565                 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9566                 ret = btrfs_unlink_subvol(trans, root, old_dir,
9567                                           root_objectid,
9568                                           old_dentry->d_name.name,
9569                                           old_dentry->d_name.len);
9570         } else { /* src is an inode */
9571                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
9572                                            BTRFS_I(old_dentry->d_inode),
9573                                            old_dentry->d_name.name,
9574                                            old_dentry->d_name.len);
9575                 if (!ret)
9576                         ret = btrfs_update_inode(trans, root, old_inode);
9577         }
9578         if (ret) {
9579                 btrfs_abort_transaction(trans, ret);
9580                 goto out_fail;
9581         }
9582
9583         /* dest is a subvolume */
9584         if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
9585                 root_objectid = BTRFS_I(new_inode)->root->root_key.objectid;
9586                 ret = btrfs_unlink_subvol(trans, dest, new_dir,
9587                                           root_objectid,
9588                                           new_dentry->d_name.name,
9589                                           new_dentry->d_name.len);
9590         } else { /* dest is an inode */
9591                 ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
9592                                            BTRFS_I(new_dentry->d_inode),
9593                                            new_dentry->d_name.name,
9594                                            new_dentry->d_name.len);
9595                 if (!ret)
9596                         ret = btrfs_update_inode(trans, dest, new_inode);
9597         }
9598         if (ret) {
9599                 btrfs_abort_transaction(trans, ret);
9600                 goto out_fail;
9601         }
9602
9603         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
9604                              new_dentry->d_name.name,
9605                              new_dentry->d_name.len, 0, old_idx);
9606         if (ret) {
9607                 btrfs_abort_transaction(trans, ret);
9608                 goto out_fail;
9609         }
9610
9611         ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
9612                              old_dentry->d_name.name,
9613                              old_dentry->d_name.len, 0, new_idx);
9614         if (ret) {
9615                 btrfs_abort_transaction(trans, ret);
9616                 goto out_fail;
9617         }
9618
9619         if (old_inode->i_nlink == 1)
9620                 BTRFS_I(old_inode)->dir_index = old_idx;
9621         if (new_inode->i_nlink == 1)
9622                 BTRFS_I(new_inode)->dir_index = new_idx;
9623
9624         if (root_log_pinned) {
9625                 parent = new_dentry->d_parent;
9626                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
9627                                 parent);
9628                 btrfs_end_log_trans(root);
9629                 root_log_pinned = false;
9630         }
9631         if (dest_log_pinned) {
9632                 parent = old_dentry->d_parent;
9633                 btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
9634                                 parent);
9635                 btrfs_end_log_trans(dest);
9636                 dest_log_pinned = false;
9637         }
9638 out_fail:
9639         /*
9640          * If we have pinned a log and an error happened, we unpin tasks
9641          * trying to sync the log and force them to fallback to a transaction
9642          * commit if the log currently contains any of the inodes involved in
9643          * this rename operation (to ensure we do not persist a log with an
9644          * inconsistent state for any of these inodes or leading to any
9645          * inconsistencies when replayed). If the transaction was aborted, the
9646          * abortion reason is propagated to userspace when attempting to commit
9647          * the transaction. If the log does not contain any of these inodes, we
9648          * allow the tasks to sync it.
9649          */
9650         if (ret && (root_log_pinned || dest_log_pinned)) {
9651                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
9652                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
9653                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
9654                     (new_inode &&
9655                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
9656                         btrfs_set_log_full_commit(fs_info, trans);
9657
9658                 if (root_log_pinned) {
9659                         btrfs_end_log_trans(root);
9660                         root_log_pinned = false;
9661                 }
9662                 if (dest_log_pinned) {
9663                         btrfs_end_log_trans(dest);
9664                         dest_log_pinned = false;
9665                 }
9666         }
9667         ret = btrfs_end_transaction(trans);
9668 out_notrans:
9669         if (new_ino == BTRFS_FIRST_FREE_OBJECTID)
9670                 up_read(&fs_info->subvol_sem);
9671         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9672                 up_read(&fs_info->subvol_sem);
9673
9674         return ret;
9675 }
9676
9677 static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
9678                                      struct btrfs_root *root,
9679                                      struct inode *dir,
9680                                      struct dentry *dentry)
9681 {
9682         int ret;
9683         struct inode *inode;
9684         u64 objectid;
9685         u64 index;
9686
9687         ret = btrfs_find_free_ino(root, &objectid);
9688         if (ret)
9689                 return ret;
9690
9691         inode = btrfs_new_inode(trans, root, dir,
9692                                 dentry->d_name.name,
9693                                 dentry->d_name.len,
9694                                 btrfs_ino(BTRFS_I(dir)),
9695                                 objectid,
9696                                 S_IFCHR | WHITEOUT_MODE,
9697                                 &index);
9698
9699         if (IS_ERR(inode)) {
9700                 ret = PTR_ERR(inode);
9701                 return ret;
9702         }
9703
9704         inode->i_op = &btrfs_special_inode_operations;
9705         init_special_inode(inode, inode->i_mode,
9706                 WHITEOUT_DEV);
9707
9708         ret = btrfs_init_inode_security(trans, inode, dir,
9709                                 &dentry->d_name);
9710         if (ret)
9711                 goto out;
9712
9713         ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
9714                                 BTRFS_I(inode), 0, index);
9715         if (ret)
9716                 goto out;
9717
9718         ret = btrfs_update_inode(trans, root, inode);
9719 out:
9720         unlock_new_inode(inode);
9721         if (ret)
9722                 inode_dec_link_count(inode);
9723         iput(inode);
9724
9725         return ret;
9726 }
9727
9728 static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
9729                            struct inode *new_dir, struct dentry *new_dentry,
9730                            unsigned int flags)
9731 {
9732         struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
9733         struct btrfs_trans_handle *trans;
9734         unsigned int trans_num_items;
9735         struct btrfs_root *root = BTRFS_I(old_dir)->root;
9736         struct btrfs_root *dest = BTRFS_I(new_dir)->root;
9737         struct inode *new_inode = d_inode(new_dentry);
9738         struct inode *old_inode = d_inode(old_dentry);
9739         u64 index = 0;
9740         u64 root_objectid;
9741         int ret;
9742         u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
9743         bool log_pinned = false;
9744
9745         if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
9746                 return -EPERM;
9747
9748         /* we only allow rename subvolume link between subvolumes */
9749         if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
9750                 return -EXDEV;
9751
9752         if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
9753             (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
9754                 return -ENOTEMPTY;
9755
9756         if (S_ISDIR(old_inode->i_mode) && new_inode &&
9757             new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
9758                 return -ENOTEMPTY;
9759
9760
9761         /* check for collisions, even if the  name isn't there */
9762         ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
9763                              new_dentry->d_name.name,
9764                              new_dentry->d_name.len);
9765
9766         if (ret) {
9767                 if (ret == -EEXIST) {
9768                         /* we shouldn't get
9769                          * eexist without a new_inode */
9770                         if (WARN_ON(!new_inode)) {
9771                                 return ret;
9772                         }
9773                 } else {
9774                         /* maybe -EOVERFLOW */
9775                         return ret;
9776                 }
9777         }
9778         ret = 0;
9779
9780         /*
9781          * we're using rename to replace one file with another.  Start IO on it
9782          * now so  we don't add too much work to the end of the transaction
9783          */
9784         if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
9785                 filemap_flush(old_inode->i_mapping);
9786
9787         /* close the racy window with snapshot create/destroy ioctl */
9788         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9789                 down_read(&fs_info->subvol_sem);
9790         /*
9791          * We want to reserve the absolute worst case amount of items.  So if
9792          * both inodes are subvols and we need to unlink them then that would
9793          * require 4 item modifications, but if they are both normal inodes it
9794          * would require 5 item modifications, so we'll assume they are normal
9795          * inodes.  So 5 * 2 is 10, plus 1 for the new link, so 11 total items
9796          * should cover the worst case number of items we'll modify.
9797          * If our rename has the whiteout flag, we need more 5 units for the
9798          * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
9799          * when selinux is enabled).
9800          */
9801         trans_num_items = 11;
9802         if (flags & RENAME_WHITEOUT)
9803                 trans_num_items += 5;
9804         trans = btrfs_start_transaction(root, trans_num_items);
9805         if (IS_ERR(trans)) {
9806                 ret = PTR_ERR(trans);
9807                 goto out_notrans;
9808         }
9809
9810         if (dest != root)
9811                 btrfs_record_root_in_trans(trans, dest);
9812
9813         ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
9814         if (ret)
9815                 goto out_fail;
9816
9817         BTRFS_I(old_inode)->dir_index = 0ULL;
9818         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9819                 /* force full log commit if subvolume involved. */
9820                 btrfs_set_log_full_commit(fs_info, trans);
9821         } else {
9822                 btrfs_pin_log_trans(root);
9823                 log_pinned = true;
9824                 ret = btrfs_insert_inode_ref(trans, dest,
9825                                              new_dentry->d_name.name,
9826                                              new_dentry->d_name.len,
9827                                              old_ino,
9828                                              btrfs_ino(BTRFS_I(new_dir)), index);
9829                 if (ret)
9830                         goto out_fail;
9831         }
9832
9833         inode_inc_iversion(old_dir);
9834         inode_inc_iversion(new_dir);
9835         inode_inc_iversion(old_inode);
9836         old_dir->i_ctime = old_dir->i_mtime =
9837         new_dir->i_ctime = new_dir->i_mtime =
9838         old_inode->i_ctime = current_time(old_dir);
9839
9840         if (old_dentry->d_parent != new_dentry->d_parent)
9841                 btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
9842                                 BTRFS_I(old_inode), 1);
9843
9844         if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
9845                 root_objectid = BTRFS_I(old_inode)->root->root_key.objectid;
9846                 ret = btrfs_unlink_subvol(trans, root, old_dir, root_objectid,
9847                                         old_dentry->d_name.name,
9848                                         old_dentry->d_name.len);
9849         } else {
9850                 ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
9851                                         BTRFS_I(d_inode(old_dentry)),
9852                                         old_dentry->d_name.name,
9853                                         old_dentry->d_name.len);
9854                 if (!ret)
9855                         ret = btrfs_update_inode(trans, root, old_inode);
9856         }
9857         if (ret) {
9858                 btrfs_abort_transaction(trans, ret);
9859                 goto out_fail;
9860         }
9861
9862         if (new_inode) {
9863                 inode_inc_iversion(new_inode);
9864                 new_inode->i_ctime = current_time(new_inode);
9865                 if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
9866                              BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
9867                         root_objectid = BTRFS_I(new_inode)->location.objectid;
9868                         ret = btrfs_unlink_subvol(trans, dest, new_dir,
9869                                                 root_objectid,
9870                                                 new_dentry->d_name.name,
9871                                                 new_dentry->d_name.len);
9872                         BUG_ON(new_inode->i_nlink == 0);
9873                 } else {
9874                         ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
9875                                                  BTRFS_I(d_inode(new_dentry)),
9876                                                  new_dentry->d_name.name,
9877                                                  new_dentry->d_name.len);
9878                 }
9879                 if (!ret && new_inode->i_nlink == 0)
9880                         ret = btrfs_orphan_add(trans,
9881                                         BTRFS_I(d_inode(new_dentry)));
9882                 if (ret) {
9883                         btrfs_abort_transaction(trans, ret);
9884                         goto out_fail;
9885                 }
9886         }
9887
9888         ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
9889                              new_dentry->d_name.name,
9890                              new_dentry->d_name.len, 0, index);
9891         if (ret) {
9892                 btrfs_abort_transaction(trans, ret);
9893                 goto out_fail;
9894         }
9895
9896         if (old_inode->i_nlink == 1)
9897                 BTRFS_I(old_inode)->dir_index = index;
9898
9899         if (log_pinned) {
9900                 struct dentry *parent = new_dentry->d_parent;
9901
9902                 btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
9903                                 parent);
9904                 btrfs_end_log_trans(root);
9905                 log_pinned = false;
9906         }
9907
9908         if (flags & RENAME_WHITEOUT) {
9909                 ret = btrfs_whiteout_for_rename(trans, root, old_dir,
9910                                                 old_dentry);
9911
9912                 if (ret) {
9913                         btrfs_abort_transaction(trans, ret);
9914                         goto out_fail;
9915                 }
9916         }
9917 out_fail:
9918         /*
9919          * If we have pinned the log and an error happened, we unpin tasks
9920          * trying to sync the log and force them to fallback to a transaction
9921          * commit if the log currently contains any of the inodes involved in
9922          * this rename operation (to ensure we do not persist a log with an
9923          * inconsistent state for any of these inodes or leading to any
9924          * inconsistencies when replayed). If the transaction was aborted, the
9925          * abortion reason is propagated to userspace when attempting to commit
9926          * the transaction. If the log does not contain any of these inodes, we
9927          * allow the tasks to sync it.
9928          */
9929         if (ret && log_pinned) {
9930                 if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
9931                     btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
9932                     btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
9933                     (new_inode &&
9934                      btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
9935                         btrfs_set_log_full_commit(fs_info, trans);
9936
9937                 btrfs_end_log_trans(root);
9938                 log_pinned = false;
9939         }
9940         btrfs_end_transaction(trans);
9941 out_notrans:
9942         if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
9943                 up_read(&fs_info->subvol_sem);
9944
9945         return ret;
9946 }
9947
9948 static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
9949                          struct inode *new_dir, struct dentry *new_dentry,
9950                          unsigned int flags)
9951 {
9952         if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
9953                 return -EINVAL;
9954
9955         if (flags & RENAME_EXCHANGE)
9956                 return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
9957                                           new_dentry);
9958
9959         return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
9960 }
9961
9962 static void btrfs_run_delalloc_work(struct btrfs_work *work)
9963 {
9964         struct btrfs_delalloc_work *delalloc_work;
9965         struct inode *inode;
9966
9967         delalloc_work = container_of(work, struct btrfs_delalloc_work,
9968                                      work);
9969         inode = delalloc_work->inode;
9970         filemap_flush(inode->i_mapping);
9971         if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
9972                                 &BTRFS_I(inode)->runtime_flags))
9973                 filemap_flush(inode->i_mapping);
9974
9975         if (delalloc_work->delay_iput)
9976                 btrfs_add_delayed_iput(inode);
9977         else
9978                 iput(inode);
9979         complete(&delalloc_work->completion);
9980 }
9981
9982 struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode,
9983                                                     int delay_iput)
9984 {
9985         struct btrfs_delalloc_work *work;
9986
9987         work = kmalloc(sizeof(*work), GFP_NOFS);
9988         if (!work)
9989                 return NULL;
9990
9991         init_completion(&work->completion);
9992         INIT_LIST_HEAD(&work->list);
9993         work->inode = inode;
9994         work->delay_iput = delay_iput;
9995         WARN_ON_ONCE(!inode);
9996         btrfs_init_work(&work->work, btrfs_flush_delalloc_helper,
9997                         btrfs_run_delalloc_work, NULL, NULL);
9998
9999         return work;
10000 }
10001
10002 void btrfs_wait_and_free_delalloc_work(struct btrfs_delalloc_work *work)
10003 {
10004         wait_for_completion(&work->completion);
10005         kfree(work);
10006 }
10007
10008 /*
10009  * some fairly slow code that needs optimization. This walks the list
10010  * of all the inodes with pending delalloc and forces them to disk.
10011  */
10012 static int __start_delalloc_inodes(struct btrfs_root *root, int delay_iput,
10013                                    int nr)
10014 {
10015         struct btrfs_inode *binode;
10016         struct inode *inode;
10017         struct btrfs_delalloc_work *work, *next;
10018         struct list_head works;
10019         struct list_head splice;
10020         int ret = 0;
10021
10022         INIT_LIST_HEAD(&works);
10023         INIT_LIST_HEAD(&splice);
10024
10025         mutex_lock(&root->delalloc_mutex);
10026         spin_lock(&root->delalloc_lock);
10027         list_splice_init(&root->delalloc_inodes, &splice);
10028         while (!list_empty(&splice)) {
10029                 binode = list_entry(splice.next, struct btrfs_inode,
10030                                     delalloc_inodes);
10031
10032                 list_move_tail(&binode->delalloc_inodes,
10033                                &root->delalloc_inodes);
10034                 inode = igrab(&binode->vfs_inode);
10035                 if (!inode) {
10036                         cond_resched_lock(&root->delalloc_lock);
10037                         continue;
10038                 }
10039                 spin_unlock(&root->delalloc_lock);
10040
10041                 work = btrfs_alloc_delalloc_work(inode, delay_iput);
10042                 if (!work) {
10043                         if (delay_iput)
10044                                 btrfs_add_delayed_iput(inode);
10045                         else
10046                                 iput(inode);
10047                         ret = -ENOMEM;
10048                         goto out;
10049                 }
10050                 list_add_tail(&work->list, &works);
10051                 btrfs_queue_work(root->fs_info->flush_workers,
10052                                  &work->work);
10053                 ret++;
10054                 if (nr != -1 && ret >= nr)
10055                         goto out;
10056                 cond_resched();
10057                 spin_lock(&root->delalloc_lock);
10058         }
10059         spin_unlock(&root->delalloc_lock);
10060
10061 out:
10062         list_for_each_entry_safe(work, next, &works, list) {
10063                 list_del_init(&work->list);
10064                 btrfs_wait_and_free_delalloc_work(work);
10065         }
10066
10067         if (!list_empty_careful(&splice)) {
10068                 spin_lock(&root->delalloc_lock);
10069                 list_splice_tail(&splice, &root->delalloc_inodes);
10070                 spin_unlock(&root->delalloc_lock);
10071         }
10072         mutex_unlock(&root->delalloc_mutex);
10073         return ret;
10074 }
10075
10076 int btrfs_start_delalloc_inodes(struct btrfs_root *root, int delay_iput)
10077 {
10078         struct btrfs_fs_info *fs_info = root->fs_info;
10079         int ret;
10080
10081         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10082                 return -EROFS;
10083
10084         ret = __start_delalloc_inodes(root, delay_iput, -1);
10085         if (ret > 0)
10086                 ret = 0;
10087         return ret;
10088 }
10089
10090 int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, int delay_iput,
10091                                int nr)
10092 {
10093         struct btrfs_root *root;
10094         struct list_head splice;
10095         int ret;
10096
10097         if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
10098                 return -EROFS;
10099
10100         INIT_LIST_HEAD(&splice);
10101
10102         mutex_lock(&fs_info->delalloc_root_mutex);
10103         spin_lock(&fs_info->delalloc_root_lock);
10104         list_splice_init(&fs_info->delalloc_roots, &splice);
10105         while (!list_empty(&splice) && nr) {
10106                 root = list_first_entry(&splice, struct btrfs_root,
10107                                         delalloc_root);
10108                 root = btrfs_grab_fs_root(root);
10109                 BUG_ON(!root);
10110                 list_move_tail(&root->delalloc_root,
10111                                &fs_info->delalloc_roots);
10112                 spin_unlock(&fs_info->delalloc_root_lock);
10113
10114                 ret = __start_delalloc_inodes(root, delay_iput, nr);
10115                 btrfs_put_fs_root(root);
10116                 if (ret < 0)
10117                         goto out;
10118
10119                 if (nr != -1) {
10120                         nr -= ret;
10121                         WARN_ON(nr < 0);
10122                 }
10123                 spin_lock(&fs_info->delalloc_root_lock);
10124         }
10125         spin_unlock(&fs_info->delalloc_root_lock);
10126
10127         ret = 0;
10128 out:
10129         if (!list_empty_careful(&splice)) {
10130                 spin_lock(&fs_info->delalloc_root_lock);
10131                 list_splice_tail(&splice, &fs_info->delalloc_roots);
10132                 spin_unlock(&fs_info->delalloc_root_lock);
10133         }
10134         mutex_unlock(&fs_info->delalloc_root_mutex);
10135         return ret;
10136 }
10137
10138 static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
10139                          const char *symname)
10140 {
10141         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10142         struct btrfs_trans_handle *trans;
10143         struct btrfs_root *root = BTRFS_I(dir)->root;
10144         struct btrfs_path *path;
10145         struct btrfs_key key;
10146         struct inode *inode = NULL;
10147         int err;
10148         int drop_inode = 0;
10149         u64 objectid;
10150         u64 index = 0;
10151         int name_len;
10152         int datasize;
10153         unsigned long ptr;
10154         struct btrfs_file_extent_item *ei;
10155         struct extent_buffer *leaf;
10156
10157         name_len = strlen(symname);
10158         if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
10159                 return -ENAMETOOLONG;
10160
10161         /*
10162          * 2 items for inode item and ref
10163          * 2 items for dir items
10164          * 1 item for updating parent inode item
10165          * 1 item for the inline extent item
10166          * 1 item for xattr if selinux is on
10167          */
10168         trans = btrfs_start_transaction(root, 7);
10169         if (IS_ERR(trans))
10170                 return PTR_ERR(trans);
10171
10172         err = btrfs_find_free_ino(root, &objectid);
10173         if (err)
10174                 goto out_unlock;
10175
10176         inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
10177                                 dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
10178                                 objectid, S_IFLNK|S_IRWXUGO, &index);
10179         if (IS_ERR(inode)) {
10180                 err = PTR_ERR(inode);
10181                 goto out_unlock;
10182         }
10183
10184         /*
10185         * If the active LSM wants to access the inode during
10186         * d_instantiate it needs these. Smack checks to see
10187         * if the filesystem supports xattrs by looking at the
10188         * ops vector.
10189         */
10190         inode->i_fop = &btrfs_file_operations;
10191         inode->i_op = &btrfs_file_inode_operations;
10192         inode->i_mapping->a_ops = &btrfs_aops;
10193         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10194
10195         err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
10196         if (err)
10197                 goto out_unlock_inode;
10198
10199         path = btrfs_alloc_path();
10200         if (!path) {
10201                 err = -ENOMEM;
10202                 goto out_unlock_inode;
10203         }
10204         key.objectid = btrfs_ino(BTRFS_I(inode));
10205         key.offset = 0;
10206         key.type = BTRFS_EXTENT_DATA_KEY;
10207         datasize = btrfs_file_extent_calc_inline_size(name_len);
10208         err = btrfs_insert_empty_item(trans, root, path, &key,
10209                                       datasize);
10210         if (err) {
10211                 btrfs_free_path(path);
10212                 goto out_unlock_inode;
10213         }
10214         leaf = path->nodes[0];
10215         ei = btrfs_item_ptr(leaf, path->slots[0],
10216                             struct btrfs_file_extent_item);
10217         btrfs_set_file_extent_generation(leaf, ei, trans->transid);
10218         btrfs_set_file_extent_type(leaf, ei,
10219                                    BTRFS_FILE_EXTENT_INLINE);
10220         btrfs_set_file_extent_encryption(leaf, ei, 0);
10221         btrfs_set_file_extent_compression(leaf, ei, 0);
10222         btrfs_set_file_extent_other_encoding(leaf, ei, 0);
10223         btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
10224
10225         ptr = btrfs_file_extent_inline_start(ei);
10226         write_extent_buffer(leaf, symname, ptr, name_len);
10227         btrfs_mark_buffer_dirty(leaf);
10228         btrfs_free_path(path);
10229
10230         inode->i_op = &btrfs_symlink_inode_operations;
10231         inode_nohighmem(inode);
10232         inode->i_mapping->a_ops = &btrfs_symlink_aops;
10233         inode_set_bytes(inode, name_len);
10234         btrfs_i_size_write(BTRFS_I(inode), name_len);
10235         err = btrfs_update_inode(trans, root, inode);
10236         /*
10237          * Last step, add directory indexes for our symlink inode. This is the
10238          * last step to avoid extra cleanup of these indexes if an error happens
10239          * elsewhere above.
10240          */
10241         if (!err)
10242                 err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
10243                                 BTRFS_I(inode), 0, index);
10244         if (err) {
10245                 drop_inode = 1;
10246                 goto out_unlock_inode;
10247         }
10248
10249         unlock_new_inode(inode);
10250         d_instantiate(dentry, inode);
10251
10252 out_unlock:
10253         btrfs_end_transaction(trans);
10254         if (drop_inode) {
10255                 inode_dec_link_count(inode);
10256                 iput(inode);
10257         }
10258         btrfs_btree_balance_dirty(fs_info);
10259         return err;
10260
10261 out_unlock_inode:
10262         drop_inode = 1;
10263         unlock_new_inode(inode);
10264         goto out_unlock;
10265 }
10266
10267 static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
10268                                        u64 start, u64 num_bytes, u64 min_size,
10269                                        loff_t actual_len, u64 *alloc_hint,
10270                                        struct btrfs_trans_handle *trans)
10271 {
10272         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
10273         struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
10274         struct extent_map *em;
10275         struct btrfs_root *root = BTRFS_I(inode)->root;
10276         struct btrfs_key ins;
10277         u64 cur_offset = start;
10278         u64 i_size;
10279         u64 cur_bytes;
10280         u64 last_alloc = (u64)-1;
10281         int ret = 0;
10282         bool own_trans = true;
10283         u64 end = start + num_bytes - 1;
10284
10285         if (trans)
10286                 own_trans = false;
10287         while (num_bytes > 0) {
10288                 if (own_trans) {
10289                         trans = btrfs_start_transaction(root, 3);
10290                         if (IS_ERR(trans)) {
10291                                 ret = PTR_ERR(trans);
10292                                 break;
10293                         }
10294                 }
10295
10296                 cur_bytes = min_t(u64, num_bytes, SZ_256M);
10297                 cur_bytes = max(cur_bytes, min_size);
10298                 /*
10299                  * If we are severely fragmented we could end up with really
10300                  * small allocations, so if the allocator is returning small
10301                  * chunks lets make its job easier by only searching for those
10302                  * sized chunks.
10303                  */
10304                 cur_bytes = min(cur_bytes, last_alloc);
10305                 ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
10306                                 min_size, 0, *alloc_hint, &ins, 1, 0);
10307                 if (ret) {
10308                         if (own_trans)
10309                                 btrfs_end_transaction(trans);
10310                         break;
10311                 }
10312                 btrfs_dec_block_group_reservations(fs_info, ins.objectid);
10313
10314                 last_alloc = ins.offset;
10315                 ret = insert_reserved_file_extent(trans, inode,
10316                                                   cur_offset, ins.objectid,
10317                                                   ins.offset, ins.offset,
10318                                                   ins.offset, 0, 0, 0,
10319                                                   BTRFS_FILE_EXTENT_PREALLOC);
10320                 if (ret) {
10321                         btrfs_free_reserved_extent(fs_info, ins.objectid,
10322                                                    ins.offset, 0);
10323                         btrfs_abort_transaction(trans, ret);
10324                         if (own_trans)
10325                                 btrfs_end_transaction(trans);
10326                         break;
10327                 }
10328
10329                 btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10330                                         cur_offset + ins.offset -1, 0);
10331
10332                 em = alloc_extent_map();
10333                 if (!em) {
10334                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
10335                                 &BTRFS_I(inode)->runtime_flags);
10336                         goto next;
10337                 }
10338
10339                 em->start = cur_offset;
10340                 em->orig_start = cur_offset;
10341                 em->len = ins.offset;
10342                 em->block_start = ins.objectid;
10343                 em->block_len = ins.offset;
10344                 em->orig_block_len = ins.offset;
10345                 em->ram_bytes = ins.offset;
10346                 em->bdev = fs_info->fs_devices->latest_bdev;
10347                 set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
10348                 em->generation = trans->transid;
10349
10350                 while (1) {
10351                         write_lock(&em_tree->lock);
10352                         ret = add_extent_mapping(em_tree, em, 1);
10353                         write_unlock(&em_tree->lock);
10354                         if (ret != -EEXIST)
10355                                 break;
10356                         btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
10357                                                 cur_offset + ins.offset - 1,
10358                                                 0);
10359                 }
10360                 free_extent_map(em);
10361 next:
10362                 num_bytes -= ins.offset;
10363                 cur_offset += ins.offset;
10364                 *alloc_hint = ins.objectid + ins.offset;
10365
10366                 inode_inc_iversion(inode);
10367                 inode->i_ctime = current_time(inode);
10368                 BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
10369                 if (!(mode & FALLOC_FL_KEEP_SIZE) &&
10370                     (actual_len > inode->i_size) &&
10371                     (cur_offset > inode->i_size)) {
10372                         if (cur_offset > actual_len)
10373                                 i_size = actual_len;
10374                         else
10375                                 i_size = cur_offset;
10376                         i_size_write(inode, i_size);
10377                         btrfs_ordered_update_i_size(inode, i_size, NULL);
10378                 }
10379
10380                 ret = btrfs_update_inode(trans, root, inode);
10381
10382                 if (ret) {
10383                         btrfs_abort_transaction(trans, ret);
10384                         if (own_trans)
10385                                 btrfs_end_transaction(trans);
10386                         break;
10387                 }
10388
10389                 if (own_trans)
10390                         btrfs_end_transaction(trans);
10391         }
10392         if (cur_offset < end)
10393                 btrfs_free_reserved_data_space(inode, NULL, cur_offset,
10394                         end - cur_offset + 1);
10395         return ret;
10396 }
10397
10398 int btrfs_prealloc_file_range(struct inode *inode, int mode,
10399                               u64 start, u64 num_bytes, u64 min_size,
10400                               loff_t actual_len, u64 *alloc_hint)
10401 {
10402         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10403                                            min_size, actual_len, alloc_hint,
10404                                            NULL);
10405 }
10406
10407 int btrfs_prealloc_file_range_trans(struct inode *inode,
10408                                     struct btrfs_trans_handle *trans, int mode,
10409                                     u64 start, u64 num_bytes, u64 min_size,
10410                                     loff_t actual_len, u64 *alloc_hint)
10411 {
10412         return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
10413                                            min_size, actual_len, alloc_hint, trans);
10414 }
10415
10416 static int btrfs_set_page_dirty(struct page *page)
10417 {
10418         return __set_page_dirty_nobuffers(page);
10419 }
10420
10421 static int btrfs_permission(struct inode *inode, int mask)
10422 {
10423         struct btrfs_root *root = BTRFS_I(inode)->root;
10424         umode_t mode = inode->i_mode;
10425
10426         if (mask & MAY_WRITE &&
10427             (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
10428                 if (btrfs_root_readonly(root))
10429                         return -EROFS;
10430                 if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
10431                         return -EACCES;
10432         }
10433         return generic_permission(inode, mask);
10434 }
10435
10436 static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
10437 {
10438         struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
10439         struct btrfs_trans_handle *trans;
10440         struct btrfs_root *root = BTRFS_I(dir)->root;
10441         struct inode *inode = NULL;
10442         u64 objectid;
10443         u64 index;
10444         int ret = 0;
10445
10446         /*
10447          * 5 units required for adding orphan entry
10448          */
10449         trans = btrfs_start_transaction(root, 5);
10450         if (IS_ERR(trans))
10451                 return PTR_ERR(trans);
10452
10453         ret = btrfs_find_free_ino(root, &objectid);
10454         if (ret)
10455                 goto out;
10456
10457         inode = btrfs_new_inode(trans, root, dir, NULL, 0,
10458                         btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
10459         if (IS_ERR(inode)) {
10460                 ret = PTR_ERR(inode);
10461                 inode = NULL;
10462                 goto out;
10463         }
10464
10465         inode->i_fop = &btrfs_file_operations;
10466         inode->i_op = &btrfs_file_inode_operations;
10467
10468         inode->i_mapping->a_ops = &btrfs_aops;
10469         BTRFS_I(inode)->io_tree.ops = &btrfs_extent_io_ops;
10470
10471         ret = btrfs_init_inode_security(trans, inode, dir, NULL);
10472         if (ret)
10473                 goto out_inode;
10474
10475         ret = btrfs_update_inode(trans, root, inode);
10476         if (ret)
10477                 goto out_inode;
10478         ret = btrfs_orphan_add(trans, BTRFS_I(inode));
10479         if (ret)
10480                 goto out_inode;
10481
10482         /*
10483          * We set number of links to 0 in btrfs_new_inode(), and here we set
10484          * it to 1 because d_tmpfile() will issue a warning if the count is 0,
10485          * through:
10486          *
10487          *    d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
10488          */
10489         set_nlink(inode, 1);
10490         unlock_new_inode(inode);
10491         d_tmpfile(dentry, inode);
10492         mark_inode_dirty(inode);
10493
10494 out:
10495         btrfs_end_transaction(trans);
10496         if (ret)
10497                 iput(inode);
10498         btrfs_btree_balance_dirty(fs_info);
10499         return ret;
10500
10501 out_inode:
10502         unlock_new_inode(inode);
10503         goto out;
10504
10505 }
10506
10507 __attribute__((const))
10508 static int btrfs_readpage_io_failed_hook(struct page *page, int failed_mirror)
10509 {
10510         return -EAGAIN;
10511 }
10512
10513 static struct btrfs_fs_info *iotree_fs_info(void *private_data)
10514 {
10515         struct inode *inode = private_data;
10516         return btrfs_sb(inode->i_sb);
10517 }
10518
10519 static void btrfs_check_extent_io_range(void *private_data, const char *caller,
10520                                         u64 start, u64 end)
10521 {
10522         struct inode *inode = private_data;
10523         u64 isize;
10524
10525         isize = i_size_read(inode);
10526         if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
10527                 btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
10528                     "%s: ino %llu isize %llu odd range [%llu,%llu]",
10529                         caller, btrfs_ino(BTRFS_I(inode)), isize, start, end);
10530         }
10531 }
10532
10533 void btrfs_set_range_writeback(void *private_data, u64 start, u64 end)
10534 {
10535         struct inode *inode = private_data;
10536         unsigned long index = start >> PAGE_SHIFT;
10537         unsigned long end_index = end >> PAGE_SHIFT;
10538         struct page *page;
10539
10540         while (index <= end_index) {
10541                 page = find_get_page(inode->i_mapping, index);
10542                 ASSERT(page); /* Pages should be in the extent_io_tree */
10543                 set_page_writeback(page);
10544                 put_page(page);
10545                 index++;
10546         }
10547 }
10548
10549 static const struct inode_operations btrfs_dir_inode_operations = {
10550         .getattr        = btrfs_getattr,
10551         .lookup         = btrfs_lookup,
10552         .create         = btrfs_create,
10553         .unlink         = btrfs_unlink,
10554         .link           = btrfs_link,
10555         .mkdir          = btrfs_mkdir,
10556         .rmdir          = btrfs_rmdir,
10557         .rename         = btrfs_rename2,
10558         .symlink        = btrfs_symlink,
10559         .setattr        = btrfs_setattr,
10560         .mknod          = btrfs_mknod,
10561         .listxattr      = btrfs_listxattr,
10562         .permission     = btrfs_permission,
10563         .get_acl        = btrfs_get_acl,
10564         .set_acl        = btrfs_set_acl,
10565         .update_time    = btrfs_update_time,
10566         .tmpfile        = btrfs_tmpfile,
10567 };
10568 static const struct inode_operations btrfs_dir_ro_inode_operations = {
10569         .lookup         = btrfs_lookup,
10570         .permission     = btrfs_permission,
10571         .update_time    = btrfs_update_time,
10572 };
10573
10574 static const struct file_operations btrfs_dir_file_operations = {
10575         .llseek         = generic_file_llseek,
10576         .read           = generic_read_dir,
10577         .iterate_shared = btrfs_real_readdir,
10578         .open           = btrfs_opendir,
10579         .unlocked_ioctl = btrfs_ioctl,
10580 #ifdef CONFIG_COMPAT
10581         .compat_ioctl   = btrfs_compat_ioctl,
10582 #endif
10583         .release        = btrfs_release_file,
10584         .fsync          = btrfs_sync_file,
10585 };
10586
10587 static const struct extent_io_ops btrfs_extent_io_ops = {
10588         /* mandatory callbacks */
10589         .submit_bio_hook = btrfs_submit_bio_hook,
10590         .readpage_end_io_hook = btrfs_readpage_end_io_hook,
10591         .merge_bio_hook = btrfs_merge_bio_hook,
10592         .readpage_io_failed_hook = btrfs_readpage_io_failed_hook,
10593         .tree_fs_info = iotree_fs_info,
10594         .set_range_writeback = btrfs_set_range_writeback,
10595
10596         /* optional callbacks */
10597         .fill_delalloc = run_delalloc_range,
10598         .writepage_end_io_hook = btrfs_writepage_end_io_hook,
10599         .writepage_start_hook = btrfs_writepage_start_hook,
10600         .set_bit_hook = btrfs_set_bit_hook,
10601         .clear_bit_hook = btrfs_clear_bit_hook,
10602         .merge_extent_hook = btrfs_merge_extent_hook,
10603         .split_extent_hook = btrfs_split_extent_hook,
10604         .check_extent_io_range = btrfs_check_extent_io_range,
10605 };
10606
10607 /*
10608  * btrfs doesn't support the bmap operation because swapfiles
10609  * use bmap to make a mapping of extents in the file.  They assume
10610  * these extents won't change over the life of the file and they
10611  * use the bmap result to do IO directly to the drive.
10612  *
10613  * the btrfs bmap call would return logical addresses that aren't
10614  * suitable for IO and they also will change frequently as COW
10615  * operations happen.  So, swapfile + btrfs == corruption.
10616  *
10617  * For now we're avoiding this by dropping bmap.
10618  */
10619 static const struct address_space_operations btrfs_aops = {
10620         .readpage       = btrfs_readpage,
10621         .writepage      = btrfs_writepage,
10622         .writepages     = btrfs_writepages,
10623         .readpages      = btrfs_readpages,
10624         .direct_IO      = btrfs_direct_IO,
10625         .invalidatepage = btrfs_invalidatepage,
10626         .releasepage    = btrfs_releasepage,
10627         .set_page_dirty = btrfs_set_page_dirty,
10628         .error_remove_page = generic_error_remove_page,
10629 };
10630
10631 static const struct address_space_operations btrfs_symlink_aops = {
10632         .readpage       = btrfs_readpage,
10633         .writepage      = btrfs_writepage,
10634         .invalidatepage = btrfs_invalidatepage,
10635         .releasepage    = btrfs_releasepage,
10636 };
10637
10638 static const struct inode_operations btrfs_file_inode_operations = {
10639         .getattr        = btrfs_getattr,
10640         .setattr        = btrfs_setattr,
10641         .listxattr      = btrfs_listxattr,
10642         .permission     = btrfs_permission,
10643         .fiemap         = btrfs_fiemap,
10644         .get_acl        = btrfs_get_acl,
10645         .set_acl        = btrfs_set_acl,
10646         .update_time    = btrfs_update_time,
10647 };
10648 static const struct inode_operations btrfs_special_inode_operations = {
10649         .getattr        = btrfs_getattr,
10650         .setattr        = btrfs_setattr,
10651         .permission     = btrfs_permission,
10652         .listxattr      = btrfs_listxattr,
10653         .get_acl        = btrfs_get_acl,
10654         .set_acl        = btrfs_set_acl,
10655         .update_time    = btrfs_update_time,
10656 };
10657 static const struct inode_operations btrfs_symlink_inode_operations = {
10658         .get_link       = page_get_link,
10659         .getattr        = btrfs_getattr,
10660         .setattr        = btrfs_setattr,
10661         .permission     = btrfs_permission,
10662         .listxattr      = btrfs_listxattr,
10663         .update_time    = btrfs_update_time,
10664 };
10665
10666 const struct dentry_operations btrfs_dentry_operations = {
10667         .d_delete       = btrfs_dentry_delete,
10668         .d_release      = btrfs_dentry_release,
10669 };