fs/btrfs/file.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * Copyright (C) 2007 Oracle.  All rights reserved.
   4  */
   5
   6 #include <linux/fs.h>
   7 #include <linux/pagemap.h>
   8 #include <linux/time.h>
   9 #include <linux/init.h>
  10 #include <linux/string.h>
  11 #include <linux/backing-dev.h>
  12 #include <linux/falloc.h>
  13 #include <linux/writeback.h>
  14 #include <linux/compat.h>
  15 #include <linux/slab.h>
  16 #include <linux/btrfs.h>
  17 #include <linux/uio.h>
  18 #include <linux/iversion.h>
  19 #include <linux/fsverity.h>
  20 #include "ctree.h"
  21 #include "disk-io.h"
  22 #include "transaction.h"
  23 #include "btrfs_inode.h"
  24 #include "print-tree.h"
  25 #include "tree-log.h"
  26 #include "locking.h"
  27 #include "volumes.h"
  28 #include "qgroup.h"
  29 #include "compression.h"
  30 #include "delalloc-space.h"
  31 #include "reflink.h"
  32 #include "subpage.h"
  33
  34 static struct kmem_cache *btrfs_inode_defrag_cachep;
  35 /*
  36  * when auto defrag is enabled we
  37  * queue up these defrag structs to remember which
  38  * inodes need defragging passes
  39  */
  40 struct inode_defrag {
  41         struct rb_node rb_node;
  42         /* objectid */
  43         u64 ino;
  44         /*
  45          * transid where the defrag was added, we search for
  46          * extents newer than this
  47          */
  48         u64 transid;
  49
  50         /* root objectid */
  51         u64 root;
  52
  53         /*
  54          * The extent size threshold for autodefrag.
  55          *
  56          * This value is different for compressed/non-compressed extents,
  57          * thus needs to be passed from higher layer.
  58          * (aka, inode_should_defrag())
  59          */
  60         u32 extent_thresh;
  61 };
  62
  63 static int __compare_inode_defrag(struct inode_defrag *defrag1,
  64                                   struct inode_defrag *defrag2)
  65 {
  66         if (defrag1->root > defrag2->root)
  67                 return 1;
  68         else if (defrag1->root < defrag2->root)
  69                 return -1;
  70         else if (defrag1->ino > defrag2->ino)
  71                 return 1;
  72         else if (defrag1->ino < defrag2->ino)
  73                 return -1;
  74         else
  75                 return 0;
  76 }
  77
  78 /* pop a record for an inode into the defrag tree.  The lock
  79  * must be held already
  80  *
  81  * If you're inserting a record for an older transid than an
  82  * existing record, the transid already in the tree is lowered
  83  *
  84  * If an existing record is found the defrag item you
  85  * pass in is freed
  86  */
  87 static int __btrfs_add_inode_defrag(struct btrfs_inode *inode,
  88                                     struct inode_defrag *defrag)
  89 {
  90         struct btrfs_fs_info *fs_info = inode->root->fs_info;
  91         struct inode_defrag *entry;
  92         struct rb_node **p;
  93         struct rb_node *parent = NULL;
  94         int ret;
  95
  96         p = &fs_info->defrag_inodes.rb_node;
  97         while (*p) {
  98                 parent = *p;
  99                 entry = rb_entry(parent, struct inode_defrag, rb_node);
 100
 101                 ret = __compare_inode_defrag(defrag, entry);
 102                 if (ret < 0)
 103                         p = &parent->rb_left;
 104                 else if (ret > 0)
 105                         p = &parent->rb_right;
 106                 else {
 107                         /* if we're reinserting an entry for
 108                          * an old defrag run, make sure to
 109                          * lower the transid of our existing record
 110                          */
 111                         if (defrag->transid < entry->transid)
 112                                 entry->transid = defrag->transid;
 113                         entry->extent_thresh = min(defrag->extent_thresh,
 114                                                    entry->extent_thresh);
 115                         return -EEXIST;
 116                 }
 117         }
 118         set_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags);
 119         rb_link_node(&defrag->rb_node, parent, p);
 120         rb_insert_color(&defrag->rb_node, &fs_info->defrag_inodes);
 121         return 0;
 122 }
 123
 124 static inline int __need_auto_defrag(struct btrfs_fs_info *fs_info)
 125 {
 126         if (!btrfs_test_opt(fs_info, AUTO_DEFRAG))
 127                 return 0;
 128
 129         if (btrfs_fs_closing(fs_info))
 130                 return 0;
 131
 132         return 1;
 133 }
 134
 135 /*
 136  * insert a defrag record for this inode if auto defrag is
 137  * enabled
 138  */
 139 int btrfs_add_inode_defrag(struct btrfs_trans_handle *trans,
 140                            struct btrfs_inode *inode, u32 extent_thresh)
 141 {
 142         struct btrfs_root *root = inode->root;
 143         struct btrfs_fs_info *fs_info = root->fs_info;
 144         struct inode_defrag *defrag;
 145         u64 transid;
 146         int ret;
 147
 148         if (!__need_auto_defrag(fs_info))
 149                 return 0;
 150
 151         if (test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags))
 152                 return 0;
 153
 154         if (trans)
 155                 transid = trans->transid;
 156         else
 157                 transid = inode->root->last_trans;
 158
 159         defrag = kmem_cache_zalloc(btrfs_inode_defrag_cachep, GFP_NOFS);
 160         if (!defrag)
 161                 return -ENOMEM;
 162
 163         defrag->ino = btrfs_ino(inode);
 164         defrag->transid = transid;
 165         defrag->root = root->root_key.objectid;
 166         defrag->extent_thresh = extent_thresh;
 167
 168         spin_lock(&fs_info->defrag_inodes_lock);
 169         if (!test_bit(BTRFS_INODE_IN_DEFRAG, &inode->runtime_flags)) {
 170                 /*
 171                  * If we set IN_DEFRAG flag and evict the inode from memory,
 172                  * and then re-read this inode, this new inode doesn't have
 173                  * IN_DEFRAG flag. At the case, we may find the existed defrag.
 174                  */
 175                 ret = __btrfs_add_inode_defrag(inode, defrag);
 176                 if (ret)
 177                         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 178         } else {
 179                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 180         }
 181         spin_unlock(&fs_info->defrag_inodes_lock);
 182         return 0;
 183 }
 184
 185 /*
 186  * pick the defragable inode that we want, if it doesn't exist, we will get
 187  * the next one.
 188  */
 189 static struct inode_defrag *
 190 btrfs_pick_defrag_inode(struct btrfs_fs_info *fs_info, u64 root, u64 ino)
 191 {
 192         struct inode_defrag *entry = NULL;
 193         struct inode_defrag tmp;
 194         struct rb_node *p;
 195         struct rb_node *parent = NULL;
 196         int ret;
 197
 198         tmp.ino = ino;
 199         tmp.root = root;
 200
 201         spin_lock(&fs_info->defrag_inodes_lock);
 202         p = fs_info->defrag_inodes.rb_node;
 203         while (p) {
 204                 parent = p;
 205                 entry = rb_entry(parent, struct inode_defrag, rb_node);
 206
 207                 ret = __compare_inode_defrag(&tmp, entry);
 208                 if (ret < 0)
 209                         p = parent->rb_left;
 210                 else if (ret > 0)
 211                         p = parent->rb_right;
 212                 else
 213                         goto out;
 214         }
 215
 216         if (parent && __compare_inode_defrag(&tmp, entry) > 0) {
 217                 parent = rb_next(parent);
 218                 if (parent)
 219                         entry = rb_entry(parent, struct inode_defrag, rb_node);
 220                 else
 221                         entry = NULL;
 222         }
 223 out:
 224         if (entry)
 225                 rb_erase(parent, &fs_info->defrag_inodes);
 226         spin_unlock(&fs_info->defrag_inodes_lock);
 227         return entry;
 228 }
 229
 230 void btrfs_cleanup_defrag_inodes(struct btrfs_fs_info *fs_info)
 231 {
 232         struct inode_defrag *defrag;
 233         struct rb_node *node;
 234
 235         spin_lock(&fs_info->defrag_inodes_lock);
 236         node = rb_first(&fs_info->defrag_inodes);
 237         while (node) {
 238                 rb_erase(node, &fs_info->defrag_inodes);
 239                 defrag = rb_entry(node, struct inode_defrag, rb_node);
 240                 kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 241
 242                 cond_resched_lock(&fs_info->defrag_inodes_lock);
 243
 244                 node = rb_first(&fs_info->defrag_inodes);
 245         }
 246         spin_unlock(&fs_info->defrag_inodes_lock);
 247 }
 248
 249 #define BTRFS_DEFRAG_BATCH      1024
 250
 251 static int __btrfs_run_defrag_inode(struct btrfs_fs_info *fs_info,
 252                                     struct inode_defrag *defrag)
 253 {
 254         struct btrfs_root *inode_root;
 255         struct inode *inode;
 256         struct btrfs_ioctl_defrag_range_args range;
 257         int ret = 0;
 258         u64 cur = 0;
 259
 260 again:
 261         if (test_bit(BTRFS_FS_STATE_REMOUNTING, &fs_info->fs_state))
 262                 goto cleanup;
 263         if (!__need_auto_defrag(fs_info))
 264                 goto cleanup;
 265
 266         /* get the inode */
 267         inode_root = btrfs_get_fs_root(fs_info, defrag->root, true);
 268         if (IS_ERR(inode_root)) {
 269                 ret = PTR_ERR(inode_root);
 270                 goto cleanup;
 271         }
 272
 273         inode = btrfs_iget(fs_info->sb, defrag->ino, inode_root);
 274         btrfs_put_root(inode_root);
 275         if (IS_ERR(inode)) {
 276                 ret = PTR_ERR(inode);
 277                 goto cleanup;
 278         }
 279
 280         if (cur >= i_size_read(inode)) {
 281                 iput(inode);
 282                 goto cleanup;
 283         }
 284
 285         /* do a chunk of defrag */
 286         clear_bit(BTRFS_INODE_IN_DEFRAG, &BTRFS_I(inode)->runtime_flags);
 287         memset(&range, 0, sizeof(range));
 288         range.len = (u64)-1;
 289         range.start = cur;
 290         range.extent_thresh = defrag->extent_thresh;
 291
 292         sb_start_write(fs_info->sb);
 293         ret = btrfs_defrag_file(inode, NULL, &range, defrag->transid,
 294                                        BTRFS_DEFRAG_BATCH);
 295         sb_end_write(fs_info->sb);
 296         iput(inode);
 297
 298         if (ret < 0)
 299                 goto cleanup;
 300
 301         cur = max(cur + fs_info->sectorsize, range.start);
 302         goto again;
 303
 304 cleanup:
 305         kmem_cache_free(btrfs_inode_defrag_cachep, defrag);
 306         return ret;
 307 }
 308
 309 /*
 310  * run through the list of inodes in the FS that need
 311  * defragging
 312  */
 313 int btrfs_run_defrag_inodes(struct btrfs_fs_info *fs_info)
 314 {
 315         struct inode_defrag *defrag;
 316         u64 first_ino = 0;
 317         u64 root_objectid = 0;
 318
 319         atomic_inc(&fs_info->defrag_running);
 320         while (1) {
 321                 /* Pause the auto defragger. */
 322                 if (test_bit(BTRFS_FS_STATE_REMOUNTING,
 323                              &fs_info->fs_state))
 324                         break;
 325
 326                 if (!__need_auto_defrag(fs_info))
 327                         break;
 328
 329                 /* find an inode to defrag */
 330                 defrag = btrfs_pick_defrag_inode(fs_info, root_objectid,
 331                                                  first_ino);
 332                 if (!defrag) {
 333                         if (root_objectid || first_ino) {
 334                                 root_objectid = 0;
 335                                 first_ino = 0;
 336                                 continue;
 337                         } else {
 338                                 break;
 339                         }
 340                 }
 341
 342                 first_ino = defrag->ino + 1;
 343                 root_objectid = defrag->root;
 344
 345                 __btrfs_run_defrag_inode(fs_info, defrag);
 346         }
 347         atomic_dec(&fs_info->defrag_running);
 348
 349         /*
 350          * during unmount, we use the transaction_wait queue to
 351          * wait for the defragger to stop
 352          */
 353         wake_up(&fs_info->transaction_wait);
 354         return 0;
 355 }
 356
 357 /* simple helper to fault in pages and copy.  This should go away
 358  * and be replaced with calls into generic code.
 359  */
 360 static noinline int btrfs_copy_from_user(loff_t pos, size_t write_bytes,
 361                                          struct page **prepared_pages,
 362                                          struct iov_iter *i)
 363 {
 364         size_t copied = 0;
 365         size_t total_copied = 0;
 366         int pg = 0;
 367         int offset = offset_in_page(pos);
 368
 369         while (write_bytes > 0) {
 370                 size_t count = min_t(size_t,
 371                                      PAGE_SIZE - offset, write_bytes);
 372                 struct page *page = prepared_pages[pg];
 373                 /*
 374                  * Copy data from userspace to the current page
 375                  */
 376                 copied = copy_page_from_iter_atomic(page, offset, count, i);
 377
 378                 /* Flush processor's dcache for this page */
 379                 flush_dcache_page(page);
 380
 381                 /*
 382                  * if we get a partial write, we can end up with
 383                  * partially up to date pages.  These add
 384                  * a lot of complexity, so make sure they don't
 385                  * happen by forcing this copy to be retried.
 386                  *
 387                  * The rest of the btrfs_file_write code will fall
 388                  * back to page at a time copies after we return 0.
 389                  */
 390                 if (unlikely(copied < count)) {
 391                         if (!PageUptodate(page)) {
 392                                 iov_iter_revert(i, copied);
 393                                 copied = 0;
 394                         }
 395                         if (!copied)
 396                                 break;
 397                 }
 398
 399                 write_bytes -= copied;
 400                 total_copied += copied;
 401                 offset += copied;
 402                 if (offset == PAGE_SIZE) {
 403                         pg++;
 404                         offset = 0;
 405                 }
 406         }
 407         return total_copied;
 408 }
 409
 410 /*
 411  * unlocks pages after btrfs_file_write is done with them
 412  */
 413 static void btrfs_drop_pages(struct btrfs_fs_info *fs_info,
 414                              struct page **pages, size_t num_pages,
 415                              u64 pos, u64 copied)
 416 {
 417         size_t i;
 418         u64 block_start = round_down(pos, fs_info->sectorsize);
 419         u64 block_len = round_up(pos + copied, fs_info->sectorsize) - block_start;
 420
 421         ASSERT(block_len <= U32_MAX);
 422         for (i = 0; i < num_pages; i++) {
 423                 /* page checked is some magic around finding pages that
 424                  * have been modified without going through btrfs_set_page_dirty
 425                  * clear it here. There should be no need to mark the pages
 426                  * accessed as prepare_pages should have marked them accessed
 427                  * in prepare_pages via find_or_create_page()
 428                  */
 429                 btrfs_page_clamp_clear_checked(fs_info, pages[i], block_start,
 430                                                block_len);
 431                 unlock_page(pages[i]);
 432                 put_page(pages[i]);
 433         }
 434 }
 435
 436 /*
 437  * After btrfs_copy_from_user(), update the following things for delalloc:
 438  * - Mark newly dirtied pages as DELALLOC in the io tree.
 439  *   Used to advise which range is to be written back.
 440  * - Mark modified pages as Uptodate/Dirty and not needing COW fixup
 441  * - Update inode size for past EOF write
 442  */
 443 int btrfs_dirty_pages(struct btrfs_inode *inode, struct page **pages,
 444                       size_t num_pages, loff_t pos, size_t write_bytes,
 445                       struct extent_state **cached, bool noreserve)
 446 {
 447         struct btrfs_fs_info *fs_info = inode->root->fs_info;
 448         int err = 0;
 449         int i;
 450         u64 num_bytes;
 451         u64 start_pos;
 452         u64 end_of_last_block;
 453         u64 end_pos = pos + write_bytes;
 454         loff_t isize = i_size_read(&inode->vfs_inode);
 455         unsigned int extra_bits = 0;
 456
 457         if (write_bytes == 0)
 458                 return 0;
 459
 460         if (noreserve)
 461                 extra_bits |= EXTENT_NORESERVE;
 462
 463         start_pos = round_down(pos, fs_info->sectorsize);
 464         num_bytes = round_up(write_bytes + pos - start_pos,
 465                              fs_info->sectorsize);
 466         ASSERT(num_bytes <= U32_MAX);
 467
 468         end_of_last_block = start_pos + num_bytes - 1;
 469
 470         /*
 471          * The pages may have already been dirty, clear out old accounting so
 472          * we can set things up properly
 473          */
 474         clear_extent_bit(&inode->io_tree, start_pos, end_of_last_block,
 475                          EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
 476                          0, 0, cached);
 477
 478         err = btrfs_set_extent_delalloc(inode, start_pos, end_of_last_block,
 479                                         extra_bits, cached);
 480         if (err)
 481                 return err;
 482
 483         for (i = 0; i < num_pages; i++) {
 484                 struct page *p = pages[i];
 485
 486                 btrfs_page_clamp_set_uptodate(fs_info, p, start_pos, num_bytes);
 487                 btrfs_page_clamp_clear_checked(fs_info, p, start_pos, num_bytes);
 488                 btrfs_page_clamp_set_dirty(fs_info, p, start_pos, num_bytes);
 489         }
 490
 491         /*
 492          * we've only changed i_size in ram, and we haven't updated
 493          * the disk i_size.  There is no need to log the inode
 494          * at this time.
 495          */
 496         if (end_pos > isize)
 497                 i_size_write(&inode->vfs_inode, end_pos);
 498         return 0;
 499 }
 500
 501 /*
 502  * this drops all the extents in the cache that intersect the range
 503  * [start, end].  Existing extents are split as required.
 504  */
 505 void btrfs_drop_extent_cache(struct btrfs_inode *inode, u64 start, u64 end,
 506                              int skip_pinned)
 507 {
 508         struct extent_map *em;
 509         struct extent_map *split = NULL;
 510         struct extent_map *split2 = NULL;
 511         struct extent_map_tree *em_tree = &inode->extent_tree;
 512         u64 len = end - start + 1;
 513         u64 gen;
 514         int ret;
 515         int testend = 1;
 516         unsigned long flags;
 517         int compressed = 0;
 518         bool modified;
 519
 520         WARN_ON(end < start);
 521         if (end == (u64)-1) {
 522                 len = (u64)-1;
 523                 testend = 0;
 524         }
 525         while (1) {
 526                 int no_splits = 0;
 527
 528                 modified = false;
 529                 if (!split)
 530                         split = alloc_extent_map();
 531                 if (!split2)
 532                         split2 = alloc_extent_map();
 533                 if (!split || !split2)
 534                         no_splits = 1;
 535
 536                 write_lock(&em_tree->lock);
 537                 em = lookup_extent_mapping(em_tree, start, len);
 538                 if (!em) {
 539                         write_unlock(&em_tree->lock);
 540                         break;
 541                 }
 542                 flags = em->flags;
 543                 gen = em->generation;
 544                 if (skip_pinned && test_bit(EXTENT_FLAG_PINNED, &em->flags)) {
 545                         if (testend && em->start + em->len >= start + len) {
 546                                 free_extent_map(em);
 547                                 write_unlock(&em_tree->lock);
 548                                 break;
 549                         }
 550                         start = em->start + em->len;
 551                         if (testend)
 552                                 len = start + len - (em->start + em->len);
 553                         free_extent_map(em);
 554                         write_unlock(&em_tree->lock);
 555                         continue;
 556                 }
 557                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
 558                 clear_bit(EXTENT_FLAG_PINNED, &em->flags);
 559                 clear_bit(EXTENT_FLAG_LOGGING, &flags);
 560                 modified = !list_empty(&em->list);
 561                 if (no_splits)
 562                         goto next;
 563
 564                 if (em->start < start) {
 565                         split->start = em->start;
 566                         split->len = start - em->start;
 567
 568                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 569                                 split->orig_start = em->orig_start;
 570                                 split->block_start = em->block_start;
 571
 572                                 if (compressed)
 573                                         split->block_len = em->block_len;
 574                                 else
 575                                         split->block_len = split->len;
 576                                 split->orig_block_len = max(split->block_len,
 577                                                 em->orig_block_len);
 578                                 split->ram_bytes = em->ram_bytes;
 579                         } else {
 580                                 split->orig_start = split->start;
 581                                 split->block_len = 0;
 582                                 split->block_start = em->block_start;
 583                                 split->orig_block_len = 0;
 584                                 split->ram_bytes = split->len;
 585                         }
 586
 587                         split->generation = gen;
 588                         split->flags = flags;
 589                         split->compress_type = em->compress_type;
 590                         replace_extent_mapping(em_tree, em, split, modified);
 591                         free_extent_map(split);
 592                         split = split2;
 593                         split2 = NULL;
 594                 }
 595                 if (testend && em->start + em->len > start + len) {
 596                         u64 diff = start + len - em->start;
 597
 598                         split->start = start + len;
 599                         split->len = em->start + em->len - (start + len);
 600                         split->flags = flags;
 601                         split->compress_type = em->compress_type;
 602                         split->generation = gen;
 603
 604                         if (em->block_start < EXTENT_MAP_LAST_BYTE) {
 605                                 split->orig_block_len = max(em->block_len,
 606                                                     em->orig_block_len);
 607
 608                                 split->ram_bytes = em->ram_bytes;
 609                                 if (compressed) {
 610                                         split->block_len = em->block_len;
 611                                         split->block_start = em->block_start;
 612                                         split->orig_start = em->orig_start;
 613                                 } else {
 614                                         split->block_len = split->len;
 615                                         split->block_start = em->block_start
 616                                                 + diff;
 617                                         split->orig_start = em->orig_start;
 618                                 }
 619                         } else {
 620                                 split->ram_bytes = split->len;
 621                                 split->orig_start = split->start;
 622                                 split->block_len = 0;
 623                                 split->block_start = em->block_start;
 624                                 split->orig_block_len = 0;
 625                         }
 626
 627                         if (extent_map_in_tree(em)) {
 628                                 replace_extent_mapping(em_tree, em, split,
 629                                                        modified);
 630                         } else {
 631                                 ret = add_extent_mapping(em_tree, split,
 632                                                          modified);
 633                                 ASSERT(ret == 0); /* Logic error */
 634                         }
 635                         free_extent_map(split);
 636                         split = NULL;
 637                 }
 638 next:
 639                 if (extent_map_in_tree(em))
 640                         remove_extent_mapping(em_tree, em);
 641                 write_unlock(&em_tree->lock);
 642
 643                 /* once for us */
 644                 free_extent_map(em);
 645                 /* once for the tree*/
 646                 free_extent_map(em);
 647         }
 648         if (split)
 649                 free_extent_map(split);
 650         if (split2)
 651                 free_extent_map(split2);
 652 }
 653
 654 /*
 655  * this is very complex, but the basic idea is to drop all extents
 656  * in the range start - end.  hint_block is filled in with a block number
 657  * that would be a good hint to the block allocator for this file.
 658  *
 659  * If an extent intersects the range but is not entirely inside the range
 660  * it is either truncated or split.  Anything entirely inside the range
 661  * is deleted from the tree.
 662  *
 663  * Note: the VFS' inode number of bytes is not updated, it's up to the caller
 664  * to deal with that. We set the field 'bytes_found' of the arguments structure
 665  * with the number of allocated bytes found in the target range, so that the
 666  * caller can update the inode's number of bytes in an atomic way when
 667  * replacing extents in a range to avoid races with stat(2).
 668  */
 669 int btrfs_drop_extents(struct btrfs_trans_handle *trans,
 670                        struct btrfs_root *root, struct btrfs_inode *inode,
 671                        struct btrfs_drop_extents_args *args)
 672 {
 673         struct btrfs_fs_info *fs_info = root->fs_info;
 674         struct extent_buffer *leaf;
 675         struct btrfs_file_extent_item *fi;
 676         struct btrfs_ref ref = { 0 };
 677         struct btrfs_key key;
 678         struct btrfs_key new_key;
 679         u64 ino = btrfs_ino(inode);
 680         u64 search_start = args->start;
 681         u64 disk_bytenr = 0;
 682         u64 num_bytes = 0;
 683         u64 extent_offset = 0;
 684         u64 extent_end = 0;
 685         u64 last_end = args->start;
 686         int del_nr = 0;
 687         int del_slot = 0;
 688         int extent_type;
 689         int recow;
 690         int ret;
 691         int modify_tree = -1;
 692         int update_refs;
 693         int found = 0;
 694         int leafs_visited = 0;
 695         struct btrfs_path *path = args->path;
 696
 697         args->bytes_found = 0;
 698         args->extent_inserted = false;
 699
 700         /* Must always have a path if ->replace_extent is true */
 701         ASSERT(!(args->replace_extent && !args->path));
 702
 703         if (!path) {
 704                 path = btrfs_alloc_path();
 705                 if (!path) {
 706                         ret = -ENOMEM;
 707                         goto out;
 708                 }
 709         }
 710
 711         if (args->drop_cache)
 712                 btrfs_drop_extent_cache(inode, args->start, args->end - 1, 0);
 713
 714         if (args->start >= inode->disk_i_size && !args->replace_extent)
 715                 modify_tree = 0;
 716
 717         update_refs = (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID);
 718         while (1) {
 719                 recow = 0;
 720                 ret = btrfs_lookup_file_extent(trans, root, path, ino,
 721                                                search_start, modify_tree);
 722                 if (ret < 0)
 723                         break;
 724                 if (ret > 0 && path->slots[0] > 0 && search_start == args->start) {
 725                         leaf = path->nodes[0];
 726                         btrfs_item_key_to_cpu(leaf, &key, path->slots[0] - 1);
 727                         if (key.objectid == ino &&
 728                             key.type == BTRFS_EXTENT_DATA_KEY)
 729                                 path->slots[0]--;
 730                 }
 731                 ret = 0;
 732                 leafs_visited++;
 733 next_slot:
 734                 leaf = path->nodes[0];
 735                 if (path->slots[0] >= btrfs_header_nritems(leaf)) {
 736                         BUG_ON(del_nr > 0);
 737                         ret = btrfs_next_leaf(root, path);
 738                         if (ret < 0)
 739                                 break;
 740                         if (ret > 0) {
 741                                 ret = 0;
 742                                 break;
 743                         }
 744                         leafs_visited++;
 745                         leaf = path->nodes[0];
 746                         recow = 1;
 747                 }
 748
 749                 btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 750
 751                 if (key.objectid > ino)
 752                         break;
 753                 if (WARN_ON_ONCE(key.objectid < ino) ||
 754                     key.type < BTRFS_EXTENT_DATA_KEY) {
 755                         ASSERT(del_nr == 0);
 756                         path->slots[0]++;
 757                         goto next_slot;
 758                 }
 759                 if (key.type > BTRFS_EXTENT_DATA_KEY || key.offset >= args->end)
 760                         break;
 761
 762                 fi = btrfs_item_ptr(leaf, path->slots[0],
 763                                     struct btrfs_file_extent_item);
 764                 extent_type = btrfs_file_extent_type(leaf, fi);
 765
 766                 if (extent_type == BTRFS_FILE_EXTENT_REG ||
 767                     extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
 768                         disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
 769                         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
 770                         extent_offset = btrfs_file_extent_offset(leaf, fi);
 771                         extent_end = key.offset +
 772                                 btrfs_file_extent_num_bytes(leaf, fi);
 773                 } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 774                         extent_end = key.offset +
 775                                 btrfs_file_extent_ram_bytes(leaf, fi);
 776                 } else {
 777                         /* can't happen */
 778                         BUG();
 779                 }
 780
 781                 /*
 782                  * Don't skip extent items representing 0 byte lengths. They
 783                  * used to be created (bug) if while punching holes we hit
 784                  * -ENOSPC condition. So if we find one here, just ensure we
 785                  * delete it, otherwise we would insert a new file extent item
 786                  * with the same key (offset) as that 0 bytes length file
 787                  * extent item in the call to setup_items_for_insert() later
 788                  * in this function.
 789                  */
 790                 if (extent_end == key.offset && extent_end >= search_start) {
 791                         last_end = extent_end;
 792                         goto delete_extent_item;
 793                 }
 794
 795                 if (extent_end <= search_start) {
 796                         path->slots[0]++;
 797                         goto next_slot;
 798                 }
 799
 800                 found = 1;
 801                 search_start = max(key.offset, args->start);
 802                 if (recow || !modify_tree) {
 803                         modify_tree = -1;
 804                         btrfs_release_path(path);
 805                         continue;
 806                 }
 807
 808                 /*
 809                  *     | - range to drop - |
 810                  *  | -------- extent -------- |
 811                  */
 812                 if (args->start > key.offset && args->end < extent_end) {
 813                         BUG_ON(del_nr > 0);
 814                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 815                                 ret = -EOPNOTSUPP;
 816                                 break;
 817                         }
 818
 819                         memcpy(&new_key, &key, sizeof(new_key));
 820                         new_key.offset = args->start;
 821                         ret = btrfs_duplicate_item(trans, root, path,
 822                                                    &new_key);
 823                         if (ret == -EAGAIN) {
 824                                 btrfs_release_path(path);
 825                                 continue;
 826                         }
 827                         if (ret < 0)
 828                                 break;
 829
 830                         leaf = path->nodes[0];
 831                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
 832                                             struct btrfs_file_extent_item);
 833                         btrfs_set_file_extent_num_bytes(leaf, fi,
 834                                                         args->start - key.offset);
 835
 836                         fi = btrfs_item_ptr(leaf, path->slots[0],
 837                                             struct btrfs_file_extent_item);
 838
 839                         extent_offset += args->start - key.offset;
 840                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 841                         btrfs_set_file_extent_num_bytes(leaf, fi,
 842                                                         extent_end - args->start);
 843                         btrfs_mark_buffer_dirty(leaf);
 844
 845                         if (update_refs && disk_bytenr > 0) {
 846                                 btrfs_init_generic_ref(&ref,
 847                                                 BTRFS_ADD_DELAYED_REF,
 848                                                 disk_bytenr, num_bytes, 0);
 849                                 btrfs_init_data_ref(&ref,
 850                                                 root->root_key.objectid,
 851                                                 new_key.objectid,
 852                                                 args->start - extent_offset,
 853                                                 0, false);
 854                                 ret = btrfs_inc_extent_ref(trans, &ref);
 855                                 BUG_ON(ret); /* -ENOMEM */
 856                         }
 857                         key.offset = args->start;
 858                 }
 859                 /*
 860                  * From here on out we will have actually dropped something, so
 861                  * last_end can be updated.
 862                  */
 863                 last_end = extent_end;
 864
 865                 /*
 866                  *  | ---- range to drop ----- |
 867                  *      | -------- extent -------- |
 868                  */
 869                 if (args->start <= key.offset && args->end < extent_end) {
 870                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 871                                 ret = -EOPNOTSUPP;
 872                                 break;
 873                         }
 874
 875                         memcpy(&new_key, &key, sizeof(new_key));
 876                         new_key.offset = args->end;
 877                         btrfs_set_item_key_safe(fs_info, path, &new_key);
 878
 879                         extent_offset += args->end - key.offset;
 880                         btrfs_set_file_extent_offset(leaf, fi, extent_offset);
 881                         btrfs_set_file_extent_num_bytes(leaf, fi,
 882                                                         extent_end - args->end);
 883                         btrfs_mark_buffer_dirty(leaf);
 884                         if (update_refs && disk_bytenr > 0)
 885                                 args->bytes_found += args->end - key.offset;
 886                         break;
 887                 }
 888
 889                 search_start = extent_end;
 890                 /*
 891                  *       | ---- range to drop ----- |
 892                  *  | -------- extent -------- |
 893                  */
 894                 if (args->start > key.offset && args->end >= extent_end) {
 895                         BUG_ON(del_nr > 0);
 896                         if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
 897                                 ret = -EOPNOTSUPP;
 898                                 break;
 899                         }
 900
 901                         btrfs_set_file_extent_num_bytes(leaf, fi,
 902                                                         args->start - key.offset);
 903                         btrfs_mark_buffer_dirty(leaf);
 904                         if (update_refs && disk_bytenr > 0)
 905                                 args->bytes_found += extent_end - args->start;
 906                         if (args->end == extent_end)
 907                                 break;
 908
 909                         path->slots[0]++;
 910                         goto next_slot;
 911                 }
 912
 913                 /*
 914                  *  | ---- range to drop ----- |
 915                  *    | ------ extent ------ |
 916                  */
 917                 if (args->start <= key.offset && args->end >= extent_end) {
 918 delete_extent_item:
 919                         if (del_nr == 0) {
 920                                 del_slot = path->slots[0];
 921                                 del_nr = 1;
 922                         } else {
 923                                 BUG_ON(del_slot + del_nr != path->slots[0]);
 924                                 del_nr++;
 925                         }
 926
 927                         if (update_refs &&
 928                             extent_type == BTRFS_FILE_EXTENT_INLINE) {
 929                                 args->bytes_found += extent_end - key.offset;
 930                                 extent_end = ALIGN(extent_end,
 931                                                    fs_info->sectorsize);
 932                         } else if (update_refs && disk_bytenr > 0) {
 933                                 btrfs_init_generic_ref(&ref,
 934                                                 BTRFS_DROP_DELAYED_REF,
 935                                                 disk_bytenr, num_bytes, 0);
 936                                 btrfs_init_data_ref(&ref,
 937                                                 root->root_key.objectid,
 938                                                 key.objectid,
 939                                                 key.offset - extent_offset, 0,
 940                                                 false);
 941                                 ret = btrfs_free_extent(trans, &ref);
 942                                 BUG_ON(ret); /* -ENOMEM */
 943                                 args->bytes_found += extent_end - key.offset;
 944                         }
 945
 946                         if (args->end == extent_end)
 947                                 break;
 948
 949                         if (path->slots[0] + 1 < btrfs_header_nritems(leaf)) {
 950                                 path->slots[0]++;
 951                                 goto next_slot;
 952                         }
 953
 954                         ret = btrfs_del_items(trans, root, path, del_slot,
 955                                               del_nr);
 956                         if (ret) {
 957                                 btrfs_abort_transaction(trans, ret);
 958                                 break;
 959                         }
 960
 961                         del_nr = 0;
 962                         del_slot = 0;
 963
 964                         btrfs_release_path(path);
 965                         continue;
 966                 }
 967
 968                 BUG();
 969         }
 970
 971         if (!ret && del_nr > 0) {
 972                 /*
 973                  * Set path->slots[0] to first slot, so that after the delete
 974                  * if items are move off from our leaf to its immediate left or
 975                  * right neighbor leafs, we end up with a correct and adjusted
 976                  * path->slots[0] for our insertion (if args->replace_extent).
 977                  */
 978                 path->slots[0] = del_slot;
 979                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
 980                 if (ret)
 981                         btrfs_abort_transaction(trans, ret);
 982         }
 983
 984         leaf = path->nodes[0];
 985         /*
 986          * If btrfs_del_items() was called, it might have deleted a leaf, in
 987          * which case it unlocked our path, so check path->locks[0] matches a
 988          * write lock.
 989          */
 990         if (!ret && args->replace_extent && leafs_visited == 1 &&
 991             path->locks[0] == BTRFS_WRITE_LOCK &&
 992             btrfs_leaf_free_space(leaf) >=
 993             sizeof(struct btrfs_item) + args->extent_item_size) {
 994
 995                 key.objectid = ino;
 996                 key.type = BTRFS_EXTENT_DATA_KEY;
 997                 key.offset = args->start;
 998                 if (!del_nr && path->slots[0] < btrfs_header_nritems(leaf)) {
 999                         struct btrfs_key slot_key;
1000
1001                         btrfs_item_key_to_cpu(leaf, &slot_key, path->slots[0]);
1002                         if (btrfs_comp_cpu_keys(&key, &slot_key) > 0)
1003                                 path->slots[0]++;
1004                 }
1005                 btrfs_setup_item_for_insert(root, path, &key, args->extent_item_size);
1006                 args->extent_inserted = true;
1007         }
1008
1009         if (!args->path)
1010                 btrfs_free_path(path);
1011         else if (!args->extent_inserted)
1012                 btrfs_release_path(path);
1013 out:
1014         args->drop_end = found ? min(args->end, last_end) : args->end;
1015
1016         return ret;
1017 }
1018
1019 static int extent_mergeable(struct extent_buffer *leaf, int slot,
1020                             u64 objectid, u64 bytenr, u64 orig_offset,
1021                             u64 *start, u64 *end)
1022 {
1023         struct btrfs_file_extent_item *fi;
1024         struct btrfs_key key;
1025         u64 extent_end;
1026
1027         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
1028                 return 0;
1029
1030         btrfs_item_key_to_cpu(leaf, &key, slot);
1031         if (key.objectid != objectid || key.type != BTRFS_EXTENT_DATA_KEY)
1032                 return 0;
1033
1034         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
1035         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG ||
1036             btrfs_file_extent_disk_bytenr(leaf, fi) != bytenr ||
1037             btrfs_file_extent_offset(leaf, fi) != key.offset - orig_offset ||
1038             btrfs_file_extent_compression(leaf, fi) ||
1039             btrfs_file_extent_encryption(leaf, fi) ||
1040             btrfs_file_extent_other_encoding(leaf, fi))
1041                 return 0;
1042
1043         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1044         if ((*start && *start != key.offset) || (*end && *end != extent_end))
1045                 return 0;
1046
1047         *start = key.offset;
1048         *end = extent_end;
1049         return 1;
1050 }
1051
1052 /*
1053  * Mark extent in the range start - end as written.
1054  *
1055  * This changes extent type from 'pre-allocated' to 'regular'. If only
1056  * part of extent is marked as written, the extent will be split into
1057  * two or three.
1058  */
1059 int btrfs_mark_extent_written(struct btrfs_trans_handle *trans,
1060                               struct btrfs_inode *inode, u64 start, u64 end)
1061 {
1062         struct btrfs_fs_info *fs_info = trans->fs_info;
1063         struct btrfs_root *root = inode->root;
1064         struct extent_buffer *leaf;
1065         struct btrfs_path *path;
1066         struct btrfs_file_extent_item *fi;
1067         struct btrfs_ref ref = { 0 };
1068         struct btrfs_key key;
1069         struct btrfs_key new_key;
1070         u64 bytenr;
1071         u64 num_bytes;
1072         u64 extent_end;
1073         u64 orig_offset;
1074         u64 other_start;
1075         u64 other_end;
1076         u64 split;
1077         int del_nr = 0;
1078         int del_slot = 0;
1079         int recow;
1080         int ret = 0;
1081         u64 ino = btrfs_ino(inode);
1082
1083         path = btrfs_alloc_path();
1084         if (!path)
1085                 return -ENOMEM;
1086 again:
1087         recow = 0;
1088         split = start;
1089         key.objectid = ino;
1090         key.type = BTRFS_EXTENT_DATA_KEY;
1091         key.offset = split;
1092
1093         ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1094         if (ret < 0)
1095                 goto out;
1096         if (ret > 0 && path->slots[0] > 0)
1097                 path->slots[0]--;
1098
1099         leaf = path->nodes[0];
1100         btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
1101         if (key.objectid != ino ||
1102             key.type != BTRFS_EXTENT_DATA_KEY) {
1103                 ret = -EINVAL;
1104                 btrfs_abort_transaction(trans, ret);
1105                 goto out;
1106         }
1107         fi = btrfs_item_ptr(leaf, path->slots[0],
1108                             struct btrfs_file_extent_item);
1109         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_PREALLOC) {
1110                 ret = -EINVAL;
1111                 btrfs_abort_transaction(trans, ret);
1112                 goto out;
1113         }
1114         extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
1115         if (key.offset > start || extent_end < end) {
1116                 ret = -EINVAL;
1117                 btrfs_abort_transaction(trans, ret);
1118                 goto out;
1119         }
1120
1121         bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
1122         num_bytes = btrfs_file_extent_disk_num_bytes(leaf, fi);
1123         orig_offset = key.offset - btrfs_file_extent_offset(leaf, fi);
1124         memcpy(&new_key, &key, sizeof(new_key));
1125
1126         if (start == key.offset && end < extent_end) {
1127                 other_start = 0;
1128                 other_end = start;
1129                 if (extent_mergeable(leaf, path->slots[0] - 1,
1130                                      ino, bytenr, orig_offset,
1131                                      &other_start, &other_end)) {
1132                         new_key.offset = end;
1133                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1134                         fi = btrfs_item_ptr(leaf, path->slots[0],
1135                                             struct btrfs_file_extent_item);
1136                         btrfs_set_file_extent_generation(leaf, fi,
1137                                                          trans->transid);
1138                         btrfs_set_file_extent_num_bytes(leaf, fi,
1139                                                         extent_end - end);
1140                         btrfs_set_file_extent_offset(leaf, fi,
1141                                                      end - orig_offset);
1142                         fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1143                                             struct btrfs_file_extent_item);
1144                         btrfs_set_file_extent_generation(leaf, fi,
1145                                                          trans->transid);
1146                         btrfs_set_file_extent_num_bytes(leaf, fi,
1147                                                         end - other_start);
1148                         btrfs_mark_buffer_dirty(leaf);
1149                         goto out;
1150                 }
1151         }
1152
1153         if (start > key.offset && end == extent_end) {
1154                 other_start = end;
1155                 other_end = 0;
1156                 if (extent_mergeable(leaf, path->slots[0] + 1,
1157                                      ino, bytenr, orig_offset,
1158                                      &other_start, &other_end)) {
1159                         fi = btrfs_item_ptr(leaf, path->slots[0],
1160                                             struct btrfs_file_extent_item);
1161                         btrfs_set_file_extent_num_bytes(leaf, fi,
1162                                                         start - key.offset);
1163                         btrfs_set_file_extent_generation(leaf, fi,
1164                                                          trans->transid);
1165                         path->slots[0]++;
1166                         new_key.offset = start;
1167                         btrfs_set_item_key_safe(fs_info, path, &new_key);
1168
1169                         fi = btrfs_item_ptr(leaf, path->slots[0],
1170                                             struct btrfs_file_extent_item);
1171                         btrfs_set_file_extent_generation(leaf, fi,
1172                                                          trans->transid);
1173                         btrfs_set_file_extent_num_bytes(leaf, fi,
1174                                                         other_end - start);
1175                         btrfs_set_file_extent_offset(leaf, fi,
1176                                                      start - orig_offset);
1177                         btrfs_mark_buffer_dirty(leaf);
1178                         goto out;
1179                 }
1180         }
1181
1182         while (start > key.offset || end < extent_end) {
1183                 if (key.offset == start)
1184                         split = end;
1185
1186                 new_key.offset = split;
1187                 ret = btrfs_duplicate_item(trans, root, path, &new_key);
1188                 if (ret == -EAGAIN) {
1189                         btrfs_release_path(path);
1190                         goto again;
1191                 }
1192                 if (ret < 0) {
1193                         btrfs_abort_transaction(trans, ret);
1194                         goto out;
1195                 }
1196
1197                 leaf = path->nodes[0];
1198                 fi = btrfs_item_ptr(leaf, path->slots[0] - 1,
1199                                     struct btrfs_file_extent_item);
1200                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1201                 btrfs_set_file_extent_num_bytes(leaf, fi,
1202                                                 split - key.offset);
1203
1204                 fi = btrfs_item_ptr(leaf, path->slots[0],
1205                                     struct btrfs_file_extent_item);
1206
1207                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1208                 btrfs_set_file_extent_offset(leaf, fi, split - orig_offset);
1209                 btrfs_set_file_extent_num_bytes(leaf, fi,
1210                                                 extent_end - split);
1211                 btrfs_mark_buffer_dirty(leaf);
1212
1213                 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF, bytenr,
1214                                        num_bytes, 0);
1215                 btrfs_init_data_ref(&ref, root->root_key.objectid, ino,
1216                                     orig_offset, 0, false);
1217                 ret = btrfs_inc_extent_ref(trans, &ref);
1218                 if (ret) {
1219                         btrfs_abort_transaction(trans, ret);
1220                         goto out;
1221                 }
1222
1223                 if (split == start) {
1224                         key.offset = start;
1225                 } else {
1226                         if (start != key.offset) {
1227                                 ret = -EINVAL;
1228                                 btrfs_abort_transaction(trans, ret);
1229                                 goto out;
1230                         }
1231                         path->slots[0]--;
1232                         extent_end = end;
1233                 }
1234                 recow = 1;
1235         }
1236
1237         other_start = end;
1238         other_end = 0;
1239         btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF, bytenr,
1240                                num_bytes, 0);
1241         btrfs_init_data_ref(&ref, root->root_key.objectid, ino, orig_offset,
1242                             0, false);
1243         if (extent_mergeable(leaf, path->slots[0] + 1,
1244                              ino, bytenr, orig_offset,
1245                              &other_start, &other_end)) {
1246                 if (recow) {
1247                         btrfs_release_path(path);
1248                         goto again;
1249                 }
1250                 extent_end = other_end;
1251                 del_slot = path->slots[0] + 1;
1252                 del_nr++;
1253                 ret = btrfs_free_extent(trans, &ref);
1254                 if (ret) {
1255                         btrfs_abort_transaction(trans, ret);
1256                         goto out;
1257                 }
1258         }
1259         other_start = 0;
1260         other_end = start;
1261         if (extent_mergeable(leaf, path->slots[0] - 1,
1262                              ino, bytenr, orig_offset,
1263                              &other_start, &other_end)) {
1264                 if (recow) {
1265                         btrfs_release_path(path);
1266                         goto again;
1267                 }
1268                 key.offset = other_start;
1269                 del_slot = path->slots[0];
1270                 del_nr++;
1271                 ret = btrfs_free_extent(trans, &ref);
1272                 if (ret) {
1273                         btrfs_abort_transaction(trans, ret);
1274                         goto out;
1275                 }
1276         }
1277         if (del_nr == 0) {
1278                 fi = btrfs_item_ptr(leaf, path->slots[0],
1279                            struct btrfs_file_extent_item);
1280                 btrfs_set_file_extent_type(leaf, fi,
1281                                            BTRFS_FILE_EXTENT_REG);
1282                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1283                 btrfs_mark_buffer_dirty(leaf);
1284         } else {
1285                 fi = btrfs_item_ptr(leaf, del_slot - 1,
1286                            struct btrfs_file_extent_item);
1287                 btrfs_set_file_extent_type(leaf, fi,
1288                                            BTRFS_FILE_EXTENT_REG);
1289                 btrfs_set_file_extent_generation(leaf, fi, trans->transid);
1290                 btrfs_set_file_extent_num_bytes(leaf, fi,
1291                                                 extent_end - key.offset);
1292                 btrfs_mark_buffer_dirty(leaf);
1293
1294                 ret = btrfs_del_items(trans, root, path, del_slot, del_nr);
1295                 if (ret < 0) {
1296                         btrfs_abort_transaction(trans, ret);
1297                         goto out;
1298                 }
1299         }
1300 out:
1301         btrfs_free_path(path);
1302         return ret;
1303 }
1304
1305 /*
1306  * on error we return an unlocked page and the error value
1307  * on success we return a locked page and 0
1308  */
1309 static int prepare_uptodate_page(struct inode *inode,
1310                                  struct page *page, u64 pos,
1311                                  bool force_uptodate)
1312 {
1313         int ret = 0;
1314
1315         if (((pos & (PAGE_SIZE - 1)) || force_uptodate) &&
1316             !PageUptodate(page)) {
1317                 ret = btrfs_readpage(NULL, page);
1318                 if (ret)
1319                         return ret;
1320                 lock_page(page);
1321                 if (!PageUptodate(page)) {
1322                         unlock_page(page);
1323                         return -EIO;
1324                 }
1325
1326                 /*
1327                  * Since btrfs_readpage() will unlock the page before it
1328                  * returns, there is a window where btrfs_releasepage() can be
1329                  * called to release the page.  Here we check both inode
1330                  * mapping and PagePrivate() to make sure the page was not
1331                  * released.
1332                  *
1333                  * The private flag check is essential for subpage as we need
1334                  * to store extra bitmap using page->private.
1335                  */
1336                 if (page->mapping != inode->i_mapping || !PagePrivate(page)) {
1337                         unlock_page(page);
1338                         return -EAGAIN;
1339                 }
1340         }
1341         return 0;
1342 }
1343
1344 /*
1345  * this just gets pages into the page cache and locks them down.
1346  */
1347 static noinline int prepare_pages(struct inode *inode, struct page **pages,
1348                                   size_t num_pages, loff_t pos,
1349                                   size_t write_bytes, bool force_uptodate)
1350 {
1351         int i;
1352         unsigned long index = pos >> PAGE_SHIFT;
1353         gfp_t mask = btrfs_alloc_write_mask(inode->i_mapping);
1354         int err = 0;
1355         int faili;
1356
1357         for (i = 0; i < num_pages; i++) {
1358 again:
1359                 pages[i] = find_or_create_page(inode->i_mapping, index + i,
1360                                                mask | __GFP_WRITE);
1361                 if (!pages[i]) {
1362                         faili = i - 1;
1363                         err = -ENOMEM;
1364                         goto fail;
1365                 }
1366
1367                 err = set_page_extent_mapped(pages[i]);
1368                 if (err < 0) {
1369                         faili = i;
1370                         goto fail;
1371                 }
1372
1373                 if (i == 0)
1374                         err = prepare_uptodate_page(inode, pages[i], pos,
1375                                                     force_uptodate);
1376                 if (!err && i == num_pages - 1)
1377                         err = prepare_uptodate_page(inode, pages[i],
1378                                                     pos + write_bytes, false);
1379                 if (err) {
1380                         put_page(pages[i]);
1381                         if (err == -EAGAIN) {
1382                                 err = 0;
1383                                 goto again;
1384                         }
1385                         faili = i - 1;
1386                         goto fail;
1387                 }
1388                 wait_on_page_writeback(pages[i]);
1389         }
1390
1391         return 0;
1392 fail:
1393         while (faili >= 0) {
1394                 unlock_page(pages[faili]);
1395                 put_page(pages[faili]);
1396                 faili--;
1397         }
1398         return err;
1399
1400 }
1401
1402 /*
1403  * This function locks the extent and properly waits for data=ordered extents
1404  * to finish before allowing the pages to be modified if need.
1405  *
1406  * The return value:
1407  * 1 - the extent is locked
1408  * 0 - the extent is not locked, and everything is OK
1409  * -EAGAIN - need re-prepare the pages
1410  * the other < 0 number - Something wrong happens
1411  */
1412 static noinline int
1413 lock_and_cleanup_extent_if_need(struct btrfs_inode *inode, struct page **pages,
1414                                 size_t num_pages, loff_t pos,
1415                                 size_t write_bytes,
1416                                 u64 *lockstart, u64 *lockend,
1417                                 struct extent_state **cached_state)
1418 {
1419         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1420         u64 start_pos;
1421         u64 last_pos;
1422         int i;
1423         int ret = 0;
1424
1425         start_pos = round_down(pos, fs_info->sectorsize);
1426         last_pos = round_up(pos + write_bytes, fs_info->sectorsize) - 1;
1427
1428         if (start_pos < inode->vfs_inode.i_size) {
1429                 struct btrfs_ordered_extent *ordered;
1430
1431                 lock_extent_bits(&inode->io_tree, start_pos, last_pos,
1432                                 cached_state);
1433                 ordered = btrfs_lookup_ordered_range(inode, start_pos,
1434                                                      last_pos - start_pos + 1);
1435                 if (ordered &&
1436                     ordered->file_offset + ordered->num_bytes > start_pos &&
1437                     ordered->file_offset <= last_pos) {
1438                         unlock_extent_cached(&inode->io_tree, start_pos,
1439                                         last_pos, cached_state);
1440                         for (i = 0; i < num_pages; i++) {
1441                                 unlock_page(pages[i]);
1442                                 put_page(pages[i]);
1443                         }
1444                         btrfs_start_ordered_extent(ordered, 1);
1445                         btrfs_put_ordered_extent(ordered);
1446                         return -EAGAIN;
1447                 }
1448                 if (ordered)
1449                         btrfs_put_ordered_extent(ordered);
1450
1451                 *lockstart = start_pos;
1452                 *lockend = last_pos;
1453                 ret = 1;
1454         }
1455
1456         /*
1457          * We should be called after prepare_pages() which should have locked
1458          * all pages in the range.
1459          */
1460         for (i = 0; i < num_pages; i++)
1461                 WARN_ON(!PageLocked(pages[i]));
1462
1463         return ret;
1464 }
1465
1466 static int check_can_nocow(struct btrfs_inode *inode, loff_t pos,
1467                            size_t *write_bytes, bool nowait)
1468 {
1469         struct btrfs_fs_info *fs_info = inode->root->fs_info;
1470         struct btrfs_root *root = inode->root;
1471         u64 lockstart, lockend;
1472         u64 num_bytes;
1473         int ret;
1474
1475         if (!(inode->flags & (BTRFS_INODE_NODATACOW | BTRFS_INODE_PREALLOC)))
1476                 return 0;
1477
1478         if (!nowait && !btrfs_drew_try_write_lock(&root->snapshot_lock))
1479                 return -EAGAIN;
1480
1481         lockstart = round_down(pos, fs_info->sectorsize);
1482         lockend = round_up(pos + *write_bytes,
1483                            fs_info->sectorsize) - 1;
1484         num_bytes = lockend - lockstart + 1;
1485
1486         if (nowait) {
1487                 struct btrfs_ordered_extent *ordered;
1488
1489                 if (!try_lock_extent(&inode->io_tree, lockstart, lockend))
1490                         return -EAGAIN;
1491
1492                 ordered = btrfs_lookup_ordered_range(inode, lockstart,
1493                                                      num_bytes);
1494                 if (ordered) {
1495                         btrfs_put_ordered_extent(ordered);
1496                         ret = -EAGAIN;
1497                         goto out_unlock;
1498                 }
1499         } else {
1500                 btrfs_lock_and_flush_ordered_range(inode, lockstart,
1501                                                    lockend, NULL);
1502         }
1503
1504         ret = can_nocow_extent(&inode->vfs_inode, lockstart, &num_bytes,
1505                         NULL, NULL, NULL, false);
1506         if (ret <= 0) {
1507                 ret = 0;
1508                 if (!nowait)
1509                         btrfs_drew_write_unlock(&root->snapshot_lock);
1510         } else {
1511                 *write_bytes = min_t(size_t, *write_bytes ,
1512                                      num_bytes - pos + lockstart);
1513         }
1514 out_unlock:
1515         unlock_extent(&inode->io_tree, lockstart, lockend);
1516
1517         return ret;
1518 }
1519
1520 static int check_nocow_nolock(struct btrfs_inode *inode, loff_t pos,
1521                               size_t *write_bytes)
1522 {
1523         return check_can_nocow(inode, pos, write_bytes, true);
1524 }
1525
1526 /*
1527  * Check if we can do nocow write into the range [@pos, @pos + @write_bytes)
1528  *
1529  * @pos:         File offset
1530  * @write_bytes: The length to write, will be updated to the nocow writeable
1531  *               range
1532  *
1533  * This function will flush ordered extents in the range to ensure proper
1534  * nocow checks.
1535  *
1536  * Return:
1537  * >0           and update @write_bytes if we can do nocow write
1538  *  0           if we can't do nocow write
1539  * -EAGAIN      if we can't get the needed lock or there are ordered extents
1540  *              for * (nowait == true) case
1541  * <0           if other error happened
1542  *
1543  * NOTE: Callers need to release the lock by btrfs_check_nocow_unlock().
1544  */
1545 int btrfs_check_nocow_lock(struct btrfs_inode *inode, loff_t pos,
1546                            size_t *write_bytes)
1547 {
1548         return check_can_nocow(inode, pos, write_bytes, false);
1549 }
1550
1551 void btrfs_check_nocow_unlock(struct btrfs_inode *inode)
1552 {
1553         btrfs_drew_write_unlock(&inode->root->snapshot_lock);
1554 }
1555
1556 static void update_time_for_write(struct inode *inode)
1557 {
1558         struct timespec64 now;
1559
1560         if (IS_NOCMTIME(inode))
1561                 return;
1562
1563         now = current_time(inode);
1564         if (!timespec64_equal(&inode->i_mtime, &now))
1565                 inode->i_mtime = now;
1566
1567         if (!timespec64_equal(&inode->i_ctime, &now))
1568                 inode->i_ctime = now;
1569
1570         if (IS_I_VERSION(inode))
1571                 inode_inc_iversion(inode);
1572 }
1573
1574 static int btrfs_write_check(struct kiocb *iocb, struct iov_iter *from,
1575                              size_t count)
1576 {
1577         struct file *file = iocb->ki_filp;
1578         struct inode *inode = file_inode(file);
1579         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1580         loff_t pos = iocb->ki_pos;
1581         int ret;
1582         loff_t oldsize;
1583         loff_t start_pos;
1584
1585         if (iocb->ki_flags & IOCB_NOWAIT) {
1586                 size_t nocow_bytes = count;
1587
1588                 /* We will allocate space in case nodatacow is not set, so bail */
1589                 if (check_nocow_nolock(BTRFS_I(inode), pos, &nocow_bytes) <= 0)
1590                         return -EAGAIN;
1591                 /*
1592                  * There are holes in the range or parts of the range that must
1593                  * be COWed (shared extents, RO block groups, etc), so just bail
1594                  * out.
1595                  */
1596                 if (nocow_bytes < count)
1597                         return -EAGAIN;
1598         }
1599
1600         current->backing_dev_info = inode_to_bdi(inode);
1601         ret = file_remove_privs(file);
1602         if (ret)
1603                 return ret;
1604
1605         /*
1606          * We reserve space for updating the inode when we reserve space for the
1607          * extent we are going to write, so we will enospc out there.  We don't
1608          * need to start yet another transaction to update the inode as we will
1609          * update the inode when we finish writing whatever data we write.
1610          */
1611         update_time_for_write(inode);
1612
1613         start_pos = round_down(pos, fs_info->sectorsize);
1614         oldsize = i_size_read(inode);
1615         if (start_pos > oldsize) {
1616                 /* Expand hole size to cover write data, preventing empty gap */
1617                 loff_t end_pos = round_up(pos + count, fs_info->sectorsize);
1618
1619                 ret = btrfs_cont_expand(BTRFS_I(inode), oldsize, end_pos);
1620                 if (ret) {
1621                         current->backing_dev_info = NULL;
1622                         return ret;
1623                 }
1624         }
1625
1626         return 0;
1627 }
1628
1629 static noinline ssize_t btrfs_buffered_write(struct kiocb *iocb,
1630                                                struct iov_iter *i)
1631 {
1632         struct file *file = iocb->ki_filp;
1633         loff_t pos;
1634         struct inode *inode = file_inode(file);
1635         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1636         struct page **pages = NULL;
1637         struct extent_changeset *data_reserved = NULL;
1638         u64 release_bytes = 0;
1639         u64 lockstart;
1640         u64 lockend;
1641         size_t num_written = 0;
1642         int nrptrs;
1643         ssize_t ret;
1644         bool only_release_metadata = false;
1645         bool force_page_uptodate = false;
1646         loff_t old_isize = i_size_read(inode);
1647         unsigned int ilock_flags = 0;
1648
1649         if (iocb->ki_flags & IOCB_NOWAIT)
1650                 ilock_flags |= BTRFS_ILOCK_TRY;
1651
1652         ret = btrfs_inode_lock(inode, ilock_flags);
1653         if (ret < 0)
1654                 return ret;
1655
1656         ret = generic_write_checks(iocb, i);
1657         if (ret <= 0)
1658                 goto out;
1659
1660         ret = btrfs_write_check(iocb, i, ret);
1661         if (ret < 0)
1662                 goto out;
1663
1664         pos = iocb->ki_pos;
1665         nrptrs = min(DIV_ROUND_UP(iov_iter_count(i), PAGE_SIZE),
1666                         PAGE_SIZE / (sizeof(struct page *)));
1667         nrptrs = min(nrptrs, current->nr_dirtied_pause - current->nr_dirtied);
1668         nrptrs = max(nrptrs, 8);
1669         pages = kmalloc_array(nrptrs, sizeof(struct page *), GFP_KERNEL);
1670         if (!pages) {
1671                 ret = -ENOMEM;
1672                 goto out;
1673         }
1674
1675         while (iov_iter_count(i) > 0) {
1676                 struct extent_state *cached_state = NULL;
1677                 size_t offset = offset_in_page(pos);
1678                 size_t sector_offset;
1679                 size_t write_bytes = min(iov_iter_count(i),
1680                                          nrptrs * (size_t)PAGE_SIZE -
1681                                          offset);
1682                 size_t num_pages;
1683                 size_t reserve_bytes;
1684                 size_t dirty_pages;
1685                 size_t copied;
1686                 size_t dirty_sectors;
1687                 size_t num_sectors;
1688                 int extents_locked;
1689
1690                 /*
1691                  * Fault pages before locking them in prepare_pages
1692                  * to avoid recursive lock
1693                  */
1694                 if (unlikely(fault_in_iov_iter_readable(i, write_bytes))) {
1695                         ret = -EFAULT;
1696                         break;
1697                 }
1698
1699                 only_release_metadata = false;
1700                 sector_offset = pos & (fs_info->sectorsize - 1);
1701
1702                 extent_changeset_release(data_reserved);
1703                 ret = btrfs_check_data_free_space(BTRFS_I(inode),
1704                                                   &data_reserved, pos,
1705                                                   write_bytes);
1706                 if (ret < 0) {
1707                         /*
1708                          * If we don't have to COW at the offset, reserve
1709                          * metadata only. write_bytes may get smaller than
1710                          * requested here.
1711                          */
1712                         if (btrfs_check_nocow_lock(BTRFS_I(inode), pos,
1713                                                    &write_bytes) > 0)
1714                                 only_release_metadata = true;
1715                         else
1716                                 break;
1717                 }
1718
1719                 num_pages = DIV_ROUND_UP(write_bytes + offset, PAGE_SIZE);
1720                 WARN_ON(num_pages > nrptrs);
1721                 reserve_bytes = round_up(write_bytes + sector_offset,
1722                                          fs_info->sectorsize);
1723                 WARN_ON(reserve_bytes == 0);
1724                 ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode),
1725                                 reserve_bytes);
1726                 if (ret) {
1727                         if (!only_release_metadata)
1728                                 btrfs_free_reserved_data_space(BTRFS_I(inode),
1729                                                 data_reserved, pos,
1730                                                 write_bytes);
1731                         else
1732                                 btrfs_check_nocow_unlock(BTRFS_I(inode));
1733                         break;
1734                 }
1735
1736                 release_bytes = reserve_bytes;
1737 again:
1738                 /*
1739                  * This is going to setup the pages array with the number of
1740                  * pages we want, so we don't really need to worry about the
1741                  * contents of pages from loop to loop
1742                  */
1743                 ret = prepare_pages(inode, pages, num_pages,
1744                                     pos, write_bytes,
1745                                     force_page_uptodate);
1746                 if (ret) {
1747                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1748                                                        reserve_bytes);
1749                         break;
1750                 }
1751
1752                 extents_locked = lock_and_cleanup_extent_if_need(
1753                                 BTRFS_I(inode), pages,
1754                                 num_pages, pos, write_bytes, &lockstart,
1755                                 &lockend, &cached_state);
1756                 if (extents_locked < 0) {
1757                         if (extents_locked == -EAGAIN)
1758                                 goto again;
1759                         btrfs_delalloc_release_extents(BTRFS_I(inode),
1760                                                        reserve_bytes);
1761                         ret = extents_locked;
1762                         break;
1763                 }
1764
1765                 copied = btrfs_copy_from_user(pos, write_bytes, pages, i);
1766
1767                 num_sectors = BTRFS_BYTES_TO_BLKS(fs_info, reserve_bytes);
1768                 dirty_sectors = round_up(copied + sector_offset,
1769                                         fs_info->sectorsize);
1770                 dirty_sectors = BTRFS_BYTES_TO_BLKS(fs_info, dirty_sectors);
1771
1772                 /*
1773                  * if we have trouble faulting in the pages, fall
1774                  * back to one page at a time
1775                  */
1776                 if (copied < write_bytes)
1777                         nrptrs = 1;
1778
1779                 if (copied == 0) {
1780                         force_page_uptodate = true;
1781                         dirty_sectors = 0;
1782                         dirty_pages = 0;
1783                 } else {
1784                         force_page_uptodate = false;
1785                         dirty_pages = DIV_ROUND_UP(copied + offset,
1786                                                    PAGE_SIZE);
1787                 }
1788
1789                 if (num_sectors > dirty_sectors) {
1790                         /* release everything except the sectors we dirtied */
1791                         release_bytes -= dirty_sectors << fs_info->sectorsize_bits;
1792                         if (only_release_metadata) {
1793                                 btrfs_delalloc_release_metadata(BTRFS_I(inode),
1794                                                         release_bytes, true);
1795                         } else {
1796                                 u64 __pos;
1797
1798                                 __pos = round_down(pos,
1799                                                    fs_info->sectorsize) +
1800                                         (dirty_pages << PAGE_SHIFT);
1801                                 btrfs_delalloc_release_space(BTRFS_I(inode),
1802                                                 data_reserved, __pos,
1803                                                 release_bytes, true);
1804                         }
1805                 }
1806
1807                 release_bytes = round_up(copied + sector_offset,
1808                                         fs_info->sectorsize);
1809
1810                 ret = btrfs_dirty_pages(BTRFS_I(inode), pages,
1811                                         dirty_pages, pos, copied,
1812                                         &cached_state, only_release_metadata);
1813
1814                 /*
1815                  * If we have not locked the extent range, because the range's
1816                  * start offset is >= i_size, we might still have a non-NULL
1817                  * cached extent state, acquired while marking the extent range
1818                  * as delalloc through btrfs_dirty_pages(). Therefore free any
1819                  * possible cached extent state to avoid a memory leak.
1820                  */
1821                 if (extents_locked)
1822                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
1823                                              lockstart, lockend, &cached_state);
1824                 else
1825                         free_extent_state(cached_state);
1826
1827                 btrfs_delalloc_release_extents(BTRFS_I(inode), reserve_bytes);
1828                 if (ret) {
1829                         btrfs_drop_pages(fs_info, pages, num_pages, pos, copied);
1830                         break;
1831                 }
1832
1833                 release_bytes = 0;
1834                 if (only_release_metadata)
1835                         btrfs_check_nocow_unlock(BTRFS_I(inode));
1836
1837                 btrfs_drop_pages(fs_info, pages, num_pages, pos, copied);
1838
1839                 cond_resched();
1840
1841                 balance_dirty_pages_ratelimited(inode->i_mapping);
1842
1843                 pos += copied;
1844                 num_written += copied;
1845         }
1846
1847         kfree(pages);
1848
1849         if (release_bytes) {
1850                 if (only_release_metadata) {
1851                         btrfs_check_nocow_unlock(BTRFS_I(inode));
1852                         btrfs_delalloc_release_metadata(BTRFS_I(inode),
1853                                         release_bytes, true);
1854                 } else {
1855                         btrfs_delalloc_release_space(BTRFS_I(inode),
1856                                         data_reserved,
1857                                         round_down(pos, fs_info->sectorsize),
1858                                         release_bytes, true);
1859                 }
1860         }
1861
1862         extent_changeset_free(data_reserved);
1863         if (num_written > 0) {
1864                 pagecache_isize_extended(inode, old_isize, iocb->ki_pos);
1865                 iocb->ki_pos += num_written;
1866         }
1867 out:
1868         btrfs_inode_unlock(inode, ilock_flags);
1869         return num_written ? num_written : ret;
1870 }
1871
1872 static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
1873                                const struct iov_iter *iter, loff_t offset)
1874 {
1875         const u32 blocksize_mask = fs_info->sectorsize - 1;
1876
1877         if (offset & blocksize_mask)
1878                 return -EINVAL;
1879
1880         if (iov_iter_alignment(iter) & blocksize_mask)
1881                 return -EINVAL;
1882
1883         return 0;
1884 }
1885
1886 static ssize_t btrfs_direct_write(struct kiocb *iocb, struct iov_iter *from)
1887 {
1888         const bool is_sync_write = (iocb->ki_flags & IOCB_DSYNC);
1889         struct file *file = iocb->ki_filp;
1890         struct inode *inode = file_inode(file);
1891         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
1892         loff_t pos;
1893         ssize_t written = 0;
1894         ssize_t written_buffered;
1895         size_t prev_left = 0;
1896         loff_t endbyte;
1897         ssize_t err;
1898         unsigned int ilock_flags = 0;
1899
1900         if (iocb->ki_flags & IOCB_NOWAIT)
1901                 ilock_flags |= BTRFS_ILOCK_TRY;
1902
1903         /* If the write DIO is within EOF, use a shared lock */
1904         if (iocb->ki_pos + iov_iter_count(from) <= i_size_read(inode))
1905                 ilock_flags |= BTRFS_ILOCK_SHARED;
1906
1907 relock:
1908         err = btrfs_inode_lock(inode, ilock_flags);
1909         if (err < 0)
1910                 return err;
1911
1912         err = generic_write_checks(iocb, from);
1913         if (err <= 0) {
1914                 btrfs_inode_unlock(inode, ilock_flags);
1915                 return err;
1916         }
1917
1918         err = btrfs_write_check(iocb, from, err);
1919         if (err < 0) {
1920                 btrfs_inode_unlock(inode, ilock_flags);
1921                 goto out;
1922         }
1923
1924         pos = iocb->ki_pos;
1925         /*
1926          * Re-check since file size may have changed just before taking the
1927          * lock or pos may have changed because of O_APPEND in generic_write_check()
1928          */
1929         if ((ilock_flags & BTRFS_ILOCK_SHARED) &&
1930             pos + iov_iter_count(from) > i_size_read(inode)) {
1931                 btrfs_inode_unlock(inode, ilock_flags);
1932                 ilock_flags &= ~BTRFS_ILOCK_SHARED;
1933                 goto relock;
1934         }
1935
1936         if (check_direct_IO(fs_info, from, pos)) {
1937                 btrfs_inode_unlock(inode, ilock_flags);
1938                 goto buffered;
1939         }
1940
1941         /*
1942          * We remove IOCB_DSYNC so that we don't deadlock when iomap_dio_rw()
1943          * calls generic_write_sync() (through iomap_dio_complete()), because
1944          * that results in calling fsync (btrfs_sync_file()) which will try to
1945          * lock the inode in exclusive/write mode.
1946          */
1947         if (is_sync_write)
1948                 iocb->ki_flags &= ~IOCB_DSYNC;
1949
1950         /*
1951          * The iov_iter can be mapped to the same file range we are writing to.
1952          * If that's the case, then we will deadlock in the iomap code, because
1953          * it first calls our callback btrfs_dio_iomap_begin(), which will create
1954          * an ordered extent, and after that it will fault in the pages that the
1955          * iov_iter refers to. During the fault in we end up in the readahead
1956          * pages code (starting at btrfs_readahead()), which will lock the range,
1957          * find that ordered extent and then wait for it to complete (at
1958          * btrfs_lock_and_flush_ordered_range()), resulting in a deadlock since
1959          * obviously the ordered extent can never complete as we didn't submit
1960          * yet the respective bio(s). This always happens when the buffer is
1961          * memory mapped to the same file range, since the iomap DIO code always
1962          * invalidates pages in the target file range (after starting and waiting
1963          * for any writeback).
1964          *
1965          * So here we disable page faults in the iov_iter and then retry if we
1966          * got -EFAULT, faulting in the pages before the retry.
1967          */
1968 again:
1969         from->nofault = true;
1970         err = iomap_dio_rw(iocb, from, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
1971                            IOMAP_DIO_PARTIAL, written);
1972         from->nofault = false;
1973
1974         /* No increment (+=) because iomap returns a cumulative value. */
1975         if (err > 0)
1976                 written = err;
1977
1978         if (iov_iter_count(from) > 0 && (err == -EFAULT || err > 0)) {
1979                 const size_t left = iov_iter_count(from);
1980                 /*
1981                  * We have more data left to write. Try to fault in as many as
1982                  * possible of the remainder pages and retry. We do this without
1983                  * releasing and locking again the inode, to prevent races with
1984                  * truncate.
1985                  *
1986                  * Also, in case the iov refers to pages in the file range of the
1987                  * file we want to write to (due to a mmap), we could enter an
1988                  * infinite loop if we retry after faulting the pages in, since
1989                  * iomap will invalidate any pages in the range early on, before
1990                  * it tries to fault in the pages of the iov. So we keep track of
1991                  * how much was left of iov in the previous EFAULT and fallback
1992                  * to buffered IO in case we haven't made any progress.
1993                  */
1994                 if (left == prev_left) {
1995                         err = -ENOTBLK;
1996                 } else {
1997                         fault_in_iov_iter_readable(from, left);
1998                         prev_left = left;
1999                         goto again;
2000                 }
2001         }
2002
2003         btrfs_inode_unlock(inode, ilock_flags);
2004
2005         /*
2006          * Add back IOCB_DSYNC. Our caller, btrfs_file_write_iter(), will do
2007          * the fsync (call generic_write_sync()).
2008          */
2009         if (is_sync_write)
2010                 iocb->ki_flags |= IOCB_DSYNC;
2011
2012         /* If 'err' is -ENOTBLK then it means we must fallback to buffered IO. */
2013         if ((err < 0 && err != -ENOTBLK) || !iov_iter_count(from))
2014                 goto out;
2015
2016 buffered:
2017         pos = iocb->ki_pos;
2018         written_buffered = btrfs_buffered_write(iocb, from);
2019         if (written_buffered < 0) {
2020                 err = written_buffered;
2021                 goto out;
2022         }
2023         /*
2024          * Ensure all data is persisted. We want the next direct IO read to be
2025          * able to read what was just written.
2026          */
2027         endbyte = pos + written_buffered - 1;
2028         err = btrfs_fdatawrite_range(inode, pos, endbyte);
2029         if (err)
2030                 goto out;
2031         err = filemap_fdatawait_range(inode->i_mapping, pos, endbyte);
2032         if (err)
2033                 goto out;
2034         written += written_buffered;
2035         iocb->ki_pos = pos + written_buffered;
2036         invalidate_mapping_pages(file->f_mapping, pos >> PAGE_SHIFT,
2037                                  endbyte >> PAGE_SHIFT);
2038 out:
2039         return err < 0 ? err : written;
2040 }
2041
2042 static ssize_t btrfs_file_write_iter(struct kiocb *iocb,
2043                                     struct iov_iter *from)
2044 {
2045         struct file *file = iocb->ki_filp;
2046         struct btrfs_inode *inode = BTRFS_I(file_inode(file));
2047         ssize_t num_written = 0;
2048         const bool sync = iocb->ki_flags & IOCB_DSYNC;
2049
2050         /*
2051          * If the fs flips readonly due to some impossible error, although we
2052          * have opened a file as writable, we have to stop this write operation
2053          * to ensure consistency.
2054          */
2055         if (BTRFS_FS_ERROR(inode->root->fs_info))
2056                 return -EROFS;
2057
2058         if (!(iocb->ki_flags & IOCB_DIRECT) &&
2059             (iocb->ki_flags & IOCB_NOWAIT))
2060                 return -EOPNOTSUPP;
2061
2062         if (sync)
2063                 atomic_inc(&inode->sync_writers);
2064
2065         if (iocb->ki_flags & IOCB_DIRECT)
2066                 num_written = btrfs_direct_write(iocb, from);
2067         else
2068                 num_written = btrfs_buffered_write(iocb, from);
2069
2070         btrfs_set_inode_last_sub_trans(inode);
2071
2072         if (num_written > 0)
2073                 num_written = generic_write_sync(iocb, num_written);
2074
2075         if (sync)
2076                 atomic_dec(&inode->sync_writers);
2077
2078         current->backing_dev_info = NULL;
2079         return num_written;
2080 }
2081
2082 int btrfs_release_file(struct inode *inode, struct file *filp)
2083 {
2084         struct btrfs_file_private *private = filp->private_data;
2085
2086         if (private && private->filldir_buf)
2087                 kfree(private->filldir_buf);
2088         kfree(private);
2089         filp->private_data = NULL;
2090
2091         /*
2092          * Set by setattr when we are about to truncate a file from a non-zero
2093          * size to a zero size.  This tries to flush down new bytes that may
2094          * have been written if the application were using truncate to replace
2095          * a file in place.
2096          */
2097         if (test_and_clear_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
2098                                &BTRFS_I(inode)->runtime_flags))
2099                         filemap_flush(inode->i_mapping);
2100         return 0;
2101 }
2102
2103 static int start_ordered_ops(struct inode *inode, loff_t start, loff_t end)
2104 {
2105         int ret;
2106         struct blk_plug plug;
2107
2108         /*
2109          * This is only called in fsync, which would do synchronous writes, so
2110          * a plug can merge adjacent IOs as much as possible.  Esp. in case of
2111          * multiple disks using raid profile, a large IO can be split to
2112          * several segments of stripe length (currently 64K).
2113          */
2114         blk_start_plug(&plug);
2115         atomic_inc(&BTRFS_I(inode)->sync_writers);
2116         ret = btrfs_fdatawrite_range(inode, start, end);
2117         atomic_dec(&BTRFS_I(inode)->sync_writers);
2118         blk_finish_plug(&plug);
2119
2120         return ret;
2121 }
2122
2123 static inline bool skip_inode_logging(const struct btrfs_log_ctx *ctx)
2124 {
2125         struct btrfs_inode *inode = BTRFS_I(ctx->inode);
2126         struct btrfs_fs_info *fs_info = inode->root->fs_info;
2127
2128         if (btrfs_inode_in_log(inode, fs_info->generation) &&
2129             list_empty(&ctx->ordered_extents))
2130                 return true;
2131
2132         /*
2133          * If we are doing a fast fsync we can not bail out if the inode's
2134          * last_trans is <= then the last committed transaction, because we only
2135          * update the last_trans of the inode during ordered extent completion,
2136          * and for a fast fsync we don't wait for that, we only wait for the
2137          * writeback to complete.
2138          */
2139         if (inode->last_trans <= fs_info->last_trans_committed &&
2140             (test_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags) ||
2141              list_empty(&ctx->ordered_extents)))
2142                 return true;
2143
2144         return false;
2145 }
2146
2147 /*
2148  * fsync call for both files and directories.  This logs the inode into
2149  * the tree log instead of forcing full commits whenever possible.
2150  *
2151  * It needs to call filemap_fdatawait so that all ordered extent updates are
2152  * in the metadata btree are up to date for copying to the log.
2153  *
2154  * It drops the inode mutex before doing the tree log commit.  This is an
2155  * important optimization for directories because holding the mutex prevents
2156  * new operations on the dir while we write to disk.
2157  */
2158 int btrfs_sync_file(struct file *file, loff_t start, loff_t end, int datasync)
2159 {
2160         struct dentry *dentry = file_dentry(file);
2161         struct inode *inode = d_inode(dentry);
2162         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2163         struct btrfs_root *root = BTRFS_I(inode)->root;
2164         struct btrfs_trans_handle *trans;
2165         struct btrfs_log_ctx ctx;
2166         int ret = 0, err;
2167         u64 len;
2168         bool full_sync;
2169
2170         trace_btrfs_sync_file(file, datasync);
2171
2172         btrfs_init_log_ctx(&ctx, inode);
2173
2174         /*
2175          * Always set the range to a full range, otherwise we can get into
2176          * several problems, from missing file extent items to represent holes
2177          * when not using the NO_HOLES feature, to log tree corruption due to
2178          * races between hole detection during logging and completion of ordered
2179          * extents outside the range, to missing checksums due to ordered extents
2180          * for which we flushed only a subset of their pages.
2181          */
2182         start = 0;
2183         end = LLONG_MAX;
2184         len = (u64)LLONG_MAX + 1;
2185
2186         /*
2187          * We write the dirty pages in the range and wait until they complete
2188          * out of the ->i_mutex. If so, we can flush the dirty pages by
2189          * multi-task, and make the performance up.  See
2190          * btrfs_wait_ordered_range for an explanation of the ASYNC check.
2191          */
2192         ret = start_ordered_ops(inode, start, end);
2193         if (ret)
2194                 goto out;
2195
2196         btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
2197
2198         atomic_inc(&root->log_batch);
2199
2200         /*
2201          * Always check for the full sync flag while holding the inode's lock,
2202          * to avoid races with other tasks. The flag must be either set all the
2203          * time during logging or always off all the time while logging.
2204          */
2205         full_sync = test_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2206                              &BTRFS_I(inode)->runtime_flags);
2207
2208         /*
2209          * Before we acquired the inode's lock and the mmap lock, someone may
2210          * have dirtied more pages in the target range. We need to make sure
2211          * that writeback for any such pages does not start while we are logging
2212          * the inode, because if it does, any of the following might happen when
2213          * we are not doing a full inode sync:
2214          *
2215          * 1) We log an extent after its writeback finishes but before its
2216          *    checksums are added to the csum tree, leading to -EIO errors
2217          *    when attempting to read the extent after a log replay.
2218          *
2219          * 2) We can end up logging an extent before its writeback finishes.
2220          *    Therefore after the log replay we will have a file extent item
2221          *    pointing to an unwritten extent (and no data checksums as well).
2222          *
2223          * So trigger writeback for any eventual new dirty pages and then we
2224          * wait for all ordered extents to complete below.
2225          */
2226         ret = start_ordered_ops(inode, start, end);
2227         if (ret) {
2228                 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2229                 goto out;
2230         }
2231
2232         /*
2233          * We have to do this here to avoid the priority inversion of waiting on
2234          * IO of a lower priority task while holding a transaction open.
2235          *
2236          * For a full fsync we wait for the ordered extents to complete while
2237          * for a fast fsync we wait just for writeback to complete, and then
2238          * attach the ordered extents to the transaction so that a transaction
2239          * commit waits for their completion, to avoid data loss if we fsync,
2240          * the current transaction commits before the ordered extents complete
2241          * and a power failure happens right after that.
2242          *
2243          * For zoned filesystem, if a write IO uses a ZONE_APPEND command, the
2244          * logical address recorded in the ordered extent may change. We need
2245          * to wait for the IO to stabilize the logical address.
2246          */
2247         if (full_sync || btrfs_is_zoned(fs_info)) {
2248                 ret = btrfs_wait_ordered_range(inode, start, len);
2249         } else {
2250                 /*
2251                  * Get our ordered extents as soon as possible to avoid doing
2252                  * checksum lookups in the csum tree, and use instead the
2253                  * checksums attached to the ordered extents.
2254                  */
2255                 btrfs_get_ordered_extents_for_logging(BTRFS_I(inode),
2256                                                       &ctx.ordered_extents);
2257                 ret = filemap_fdatawait_range(inode->i_mapping, start, end);
2258         }
2259
2260         if (ret)
2261                 goto out_release_extents;
2262
2263         atomic_inc(&root->log_batch);
2264
2265         smp_mb();
2266         if (skip_inode_logging(&ctx)) {
2267                 /*
2268                  * We've had everything committed since the last time we were
2269                  * modified so clear this flag in case it was set for whatever
2270                  * reason, it's no longer relevant.
2271                  */
2272                 clear_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2273                           &BTRFS_I(inode)->runtime_flags);
2274                 /*
2275                  * An ordered extent might have started before and completed
2276                  * already with io errors, in which case the inode was not
2277                  * updated and we end up here. So check the inode's mapping
2278                  * for any errors that might have happened since we last
2279                  * checked called fsync.
2280                  */
2281                 ret = filemap_check_wb_err(inode->i_mapping, file->f_wb_err);
2282                 goto out_release_extents;
2283         }
2284
2285         /*
2286          * We use start here because we will need to wait on the IO to complete
2287          * in btrfs_sync_log, which could require joining a transaction (for
2288          * example checking cross references in the nocow path).  If we use join
2289          * here we could get into a situation where we're waiting on IO to
2290          * happen that is blocked on a transaction trying to commit.  With start
2291          * we inc the extwriter counter, so we wait for all extwriters to exit
2292          * before we start blocking joiners.  This comment is to keep somebody
2293          * from thinking they are super smart and changing this to
2294          * btrfs_join_transaction *cough*Josef*cough*.
2295          */
2296         trans = btrfs_start_transaction(root, 0);
2297         if (IS_ERR(trans)) {
2298                 ret = PTR_ERR(trans);
2299                 goto out_release_extents;
2300         }
2301         trans->in_fsync = true;
2302
2303         ret = btrfs_log_dentry_safe(trans, dentry, &ctx);
2304         btrfs_release_log_ctx_extents(&ctx);
2305         if (ret < 0) {
2306                 /* Fallthrough and commit/free transaction. */
2307                 ret = 1;
2308         }
2309
2310         /* we've logged all the items and now have a consistent
2311          * version of the file in the log.  It is possible that
2312          * someone will come in and modify the file, but that's
2313          * fine because the log is consistent on disk, and we
2314          * have references to all of the file's extents
2315          *
2316          * It is possible that someone will come in and log the
2317          * file again, but that will end up using the synchronization
2318          * inside btrfs_sync_log to keep things safe.
2319          */
2320         btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2321
2322         if (ret != BTRFS_NO_LOG_SYNC) {
2323                 if (!ret) {
2324                         ret = btrfs_sync_log(trans, root, &ctx);
2325                         if (!ret) {
2326                                 ret = btrfs_end_transaction(trans);
2327                                 goto out;
2328                         }
2329                 }
2330                 if (!full_sync) {
2331                         ret = btrfs_wait_ordered_range(inode, start, len);
2332                         if (ret) {
2333                                 btrfs_end_transaction(trans);
2334                                 goto out;
2335                         }
2336                 }
2337                 ret = btrfs_commit_transaction(trans);
2338         } else {
2339                 ret = btrfs_end_transaction(trans);
2340         }
2341 out:
2342         ASSERT(list_empty(&ctx.list));
2343         err = file_check_and_advance_wb_err(file);
2344         if (!ret)
2345                 ret = err;
2346         return ret > 0 ? -EIO : ret;
2347
2348 out_release_extents:
2349         btrfs_release_log_ctx_extents(&ctx);
2350         btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2351         goto out;
2352 }
2353
2354 static const struct vm_operations_struct btrfs_file_vm_ops = {
2355         .fault          = filemap_fault,
2356         .map_pages      = filemap_map_pages,
2357         .page_mkwrite   = btrfs_page_mkwrite,
2358 };
2359
2360 static int btrfs_file_mmap(struct file  *filp, struct vm_area_struct *vma)
2361 {
2362         struct address_space *mapping = filp->f_mapping;
2363
2364         if (!mapping->a_ops->readpage)
2365                 return -ENOEXEC;
2366
2367         file_accessed(filp);
2368         vma->vm_ops = &btrfs_file_vm_ops;
2369
2370         return 0;
2371 }
2372
2373 static int hole_mergeable(struct btrfs_inode *inode, struct extent_buffer *leaf,
2374                           int slot, u64 start, u64 end)
2375 {
2376         struct btrfs_file_extent_item *fi;
2377         struct btrfs_key key;
2378
2379         if (slot < 0 || slot >= btrfs_header_nritems(leaf))
2380                 return 0;
2381
2382         btrfs_item_key_to_cpu(leaf, &key, slot);
2383         if (key.objectid != btrfs_ino(inode) ||
2384             key.type != BTRFS_EXTENT_DATA_KEY)
2385                 return 0;
2386
2387         fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2388
2389         if (btrfs_file_extent_type(leaf, fi) != BTRFS_FILE_EXTENT_REG)
2390                 return 0;
2391
2392         if (btrfs_file_extent_disk_bytenr(leaf, fi))
2393                 return 0;
2394
2395         if (key.offset == end)
2396                 return 1;
2397         if (key.offset + btrfs_file_extent_num_bytes(leaf, fi) == start)
2398                 return 1;
2399         return 0;
2400 }
2401
2402 static int fill_holes(struct btrfs_trans_handle *trans,
2403                 struct btrfs_inode *inode,
2404                 struct btrfs_path *path, u64 offset, u64 end)
2405 {
2406         struct btrfs_fs_info *fs_info = trans->fs_info;
2407         struct btrfs_root *root = inode->root;
2408         struct extent_buffer *leaf;
2409         struct btrfs_file_extent_item *fi;
2410         struct extent_map *hole_em;
2411         struct extent_map_tree *em_tree = &inode->extent_tree;
2412         struct btrfs_key key;
2413         int ret;
2414
2415         if (btrfs_fs_incompat(fs_info, NO_HOLES))
2416                 goto out;
2417
2418         key.objectid = btrfs_ino(inode);
2419         key.type = BTRFS_EXTENT_DATA_KEY;
2420         key.offset = offset;
2421
2422         ret = btrfs_search_slot(trans, root, &key, path, 0, 1);
2423         if (ret <= 0) {
2424                 /*
2425                  * We should have dropped this offset, so if we find it then
2426                  * something has gone horribly wrong.
2427                  */
2428                 if (ret == 0)
2429                         ret = -EINVAL;
2430                 return ret;
2431         }
2432
2433         leaf = path->nodes[0];
2434         if (hole_mergeable(inode, leaf, path->slots[0] - 1, offset, end)) {
2435                 u64 num_bytes;
2436
2437                 path->slots[0]--;
2438                 fi = btrfs_item_ptr(leaf, path->slots[0],
2439                                     struct btrfs_file_extent_item);
2440                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) +
2441                         end - offset;
2442                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2443                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2444                 btrfs_set_file_extent_offset(leaf, fi, 0);
2445                 btrfs_mark_buffer_dirty(leaf);
2446                 goto out;
2447         }
2448
2449         if (hole_mergeable(inode, leaf, path->slots[0], offset, end)) {
2450                 u64 num_bytes;
2451
2452                 key.offset = offset;
2453                 btrfs_set_item_key_safe(fs_info, path, &key);
2454                 fi = btrfs_item_ptr(leaf, path->slots[0],
2455                                     struct btrfs_file_extent_item);
2456                 num_bytes = btrfs_file_extent_num_bytes(leaf, fi) + end -
2457                         offset;
2458                 btrfs_set_file_extent_num_bytes(leaf, fi, num_bytes);
2459                 btrfs_set_file_extent_ram_bytes(leaf, fi, num_bytes);
2460                 btrfs_set_file_extent_offset(leaf, fi, 0);
2461                 btrfs_mark_buffer_dirty(leaf);
2462                 goto out;
2463         }
2464         btrfs_release_path(path);
2465
2466         ret = btrfs_insert_file_extent(trans, root, btrfs_ino(inode),
2467                         offset, 0, 0, end - offset, 0, end - offset, 0, 0, 0);
2468         if (ret)
2469                 return ret;
2470
2471 out:
2472         btrfs_release_path(path);
2473
2474         hole_em = alloc_extent_map();
2475         if (!hole_em) {
2476                 btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2477                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2478         } else {
2479                 hole_em->start = offset;
2480                 hole_em->len = end - offset;
2481                 hole_em->ram_bytes = hole_em->len;
2482                 hole_em->orig_start = offset;
2483
2484                 hole_em->block_start = EXTENT_MAP_HOLE;
2485                 hole_em->block_len = 0;
2486                 hole_em->orig_block_len = 0;
2487                 hole_em->compress_type = BTRFS_COMPRESS_NONE;
2488                 hole_em->generation = trans->transid;
2489
2490                 do {
2491                         btrfs_drop_extent_cache(inode, offset, end - 1, 0);
2492                         write_lock(&em_tree->lock);
2493                         ret = add_extent_mapping(em_tree, hole_em, 1);
2494                         write_unlock(&em_tree->lock);
2495                 } while (ret == -EEXIST);
2496                 free_extent_map(hole_em);
2497                 if (ret)
2498                         set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
2499                                         &inode->runtime_flags);
2500         }
2501
2502         return 0;
2503 }
2504
2505 /*
2506  * Find a hole extent on given inode and change start/len to the end of hole
2507  * extent.(hole/vacuum extent whose em->start <= start &&
2508  *         em->start + em->len > start)
2509  * When a hole extent is found, return 1 and modify start/len.
2510  */
2511 static int find_first_non_hole(struct btrfs_inode *inode, u64 *start, u64 *len)
2512 {
2513         struct btrfs_fs_info *fs_info = inode->root->fs_info;
2514         struct extent_map *em;
2515         int ret = 0;
2516
2517         em = btrfs_get_extent(inode, NULL, 0,
2518                               round_down(*start, fs_info->sectorsize),
2519                               round_up(*len, fs_info->sectorsize));
2520         if (IS_ERR(em))
2521                 return PTR_ERR(em);
2522
2523         /* Hole or vacuum extent(only exists in no-hole mode) */
2524         if (em->block_start == EXTENT_MAP_HOLE) {
2525                 ret = 1;
2526                 *len = em->start + em->len > *start + *len ?
2527                        0 : *start + *len - em->start - em->len;
2528                 *start = em->start + em->len;
2529         }
2530         free_extent_map(em);
2531         return ret;
2532 }
2533
2534 static int btrfs_punch_hole_lock_range(struct inode *inode,
2535                                        const u64 lockstart,
2536                                        const u64 lockend,
2537                                        struct extent_state **cached_state)
2538 {
2539         /*
2540          * For subpage case, if the range is not at page boundary, we could
2541          * have pages at the leading/tailing part of the range.
2542          * This could lead to dead loop since filemap_range_has_page()
2543          * will always return true.
2544          * So here we need to do extra page alignment for
2545          * filemap_range_has_page().
2546          */
2547         const u64 page_lockstart = round_up(lockstart, PAGE_SIZE);
2548         const u64 page_lockend = round_down(lockend + 1, PAGE_SIZE) - 1;
2549
2550         while (1) {
2551                 struct btrfs_ordered_extent *ordered;
2552                 int ret;
2553
2554                 truncate_pagecache_range(inode, lockstart, lockend);
2555
2556                 lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
2557                                  cached_state);
2558                 ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
2559                                                             lockend);
2560
2561                 /*
2562                  * We need to make sure we have no ordered extents in this range
2563                  * and nobody raced in and read a page in this range, if we did
2564                  * we need to try again.
2565                  */
2566                 if ((!ordered ||
2567                     (ordered->file_offset + ordered->num_bytes <= lockstart ||
2568                      ordered->file_offset > lockend)) &&
2569                      !filemap_range_has_page(inode->i_mapping,
2570                                              page_lockstart, page_lockend)) {
2571                         if (ordered)
2572                                 btrfs_put_ordered_extent(ordered);
2573                         break;
2574                 }
2575                 if (ordered)
2576                         btrfs_put_ordered_extent(ordered);
2577                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
2578                                      lockend, cached_state);
2579                 ret = btrfs_wait_ordered_range(inode, lockstart,
2580                                                lockend - lockstart + 1);
2581                 if (ret)
2582                         return ret;
2583         }
2584         return 0;
2585 }
2586
2587 static int btrfs_insert_replace_extent(struct btrfs_trans_handle *trans,
2588                                      struct btrfs_inode *inode,
2589                                      struct btrfs_path *path,
2590                                      struct btrfs_replace_extent_info *extent_info,
2591                                      const u64 replace_len,
2592                                      const u64 bytes_to_drop)
2593 {
2594         struct btrfs_fs_info *fs_info = trans->fs_info;
2595         struct btrfs_root *root = inode->root;
2596         struct btrfs_file_extent_item *extent;
2597         struct extent_buffer *leaf;
2598         struct btrfs_key key;
2599         int slot;
2600         struct btrfs_ref ref = { 0 };
2601         int ret;
2602
2603         if (replace_len == 0)
2604                 return 0;
2605
2606         if (extent_info->disk_offset == 0 &&
2607             btrfs_fs_incompat(fs_info, NO_HOLES)) {
2608                 btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2609                 return 0;
2610         }
2611
2612         key.objectid = btrfs_ino(inode);
2613         key.type = BTRFS_EXTENT_DATA_KEY;
2614         key.offset = extent_info->file_offset;
2615         ret = btrfs_insert_empty_item(trans, root, path, &key,
2616                                       sizeof(struct btrfs_file_extent_item));
2617         if (ret)
2618                 return ret;
2619         leaf = path->nodes[0];
2620         slot = path->slots[0];
2621         write_extent_buffer(leaf, extent_info->extent_buf,
2622                             btrfs_item_ptr_offset(leaf, slot),
2623                             sizeof(struct btrfs_file_extent_item));
2624         extent = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
2625         ASSERT(btrfs_file_extent_type(leaf, extent) != BTRFS_FILE_EXTENT_INLINE);
2626         btrfs_set_file_extent_offset(leaf, extent, extent_info->data_offset);
2627         btrfs_set_file_extent_num_bytes(leaf, extent, replace_len);
2628         if (extent_info->is_new_extent)
2629                 btrfs_set_file_extent_generation(leaf, extent, trans->transid);
2630         btrfs_mark_buffer_dirty(leaf);
2631         btrfs_release_path(path);
2632
2633         ret = btrfs_inode_set_file_extent_range(inode, extent_info->file_offset,
2634                                                 replace_len);
2635         if (ret)
2636                 return ret;
2637
2638         /* If it's a hole, nothing more needs to be done. */
2639         if (extent_info->disk_offset == 0) {
2640                 btrfs_update_inode_bytes(inode, 0, bytes_to_drop);
2641                 return 0;
2642         }
2643
2644         btrfs_update_inode_bytes(inode, replace_len, bytes_to_drop);
2645
2646         if (extent_info->is_new_extent && extent_info->insertions == 0) {
2647                 key.objectid = extent_info->disk_offset;
2648                 key.type = BTRFS_EXTENT_ITEM_KEY;
2649                 key.offset = extent_info->disk_len;
2650                 ret = btrfs_alloc_reserved_file_extent(trans, root,
2651                                                        btrfs_ino(inode),
2652                                                        extent_info->file_offset,
2653                                                        extent_info->qgroup_reserved,
2654                                                        &key);
2655         } else {
2656                 u64 ref_offset;
2657
2658                 btrfs_init_generic_ref(&ref, BTRFS_ADD_DELAYED_REF,
2659                                        extent_info->disk_offset,
2660                                        extent_info->disk_len, 0);
2661                 ref_offset = extent_info->file_offset - extent_info->data_offset;
2662                 btrfs_init_data_ref(&ref, root->root_key.objectid,
2663                                     btrfs_ino(inode), ref_offset, 0, false);
2664                 ret = btrfs_inc_extent_ref(trans, &ref);
2665         }
2666
2667         extent_info->insertions++;
2668
2669         return ret;
2670 }
2671
2672 /*
2673  * The respective range must have been previously locked, as well as the inode.
2674  * The end offset is inclusive (last byte of the range).
2675  * @extent_info is NULL for fallocate's hole punching and non-NULL when replacing
2676  * the file range with an extent.
2677  * When not punching a hole, we don't want to end up in a state where we dropped
2678  * extents without inserting a new one, so we must abort the transaction to avoid
2679  * a corruption.
2680  */
2681 int btrfs_replace_file_extents(struct btrfs_inode *inode,
2682                                struct btrfs_path *path, const u64 start,
2683                                const u64 end,
2684                                struct btrfs_replace_extent_info *extent_info,
2685                                struct btrfs_trans_handle **trans_out)
2686 {
2687         struct btrfs_drop_extents_args drop_args = { 0 };
2688         struct btrfs_root *root = inode->root;
2689         struct btrfs_fs_info *fs_info = root->fs_info;
2690         u64 min_size = btrfs_calc_insert_metadata_size(fs_info, 1);
2691         u64 ino_size = round_up(inode->vfs_inode.i_size, fs_info->sectorsize);
2692         struct btrfs_trans_handle *trans = NULL;
2693         struct btrfs_block_rsv *rsv;
2694         unsigned int rsv_count;
2695         u64 cur_offset;
2696         u64 len = end - start;
2697         int ret = 0;
2698
2699         if (end <= start)
2700                 return -EINVAL;
2701
2702         rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
2703         if (!rsv) {
2704                 ret = -ENOMEM;
2705                 goto out;
2706         }
2707         rsv->size = btrfs_calc_insert_metadata_size(fs_info, 1);
2708         rsv->failfast = 1;
2709
2710         /*
2711          * 1 - update the inode
2712          * 1 - removing the extents in the range
2713          * 1 - adding the hole extent if no_holes isn't set or if we are
2714          *     replacing the range with a new extent
2715          */
2716         if (!btrfs_fs_incompat(fs_info, NO_HOLES) || extent_info)
2717                 rsv_count = 3;
2718         else
2719                 rsv_count = 2;
2720
2721         trans = btrfs_start_transaction(root, rsv_count);
2722         if (IS_ERR(trans)) {
2723                 ret = PTR_ERR(trans);
2724                 trans = NULL;
2725                 goto out_free;
2726         }
2727
2728         ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
2729                                       min_size, false);
2730         BUG_ON(ret);
2731         trans->block_rsv = rsv;
2732
2733         cur_offset = start;
2734         drop_args.path = path;
2735         drop_args.end = end + 1;
2736         drop_args.drop_cache = true;
2737         while (cur_offset < end) {
2738                 drop_args.start = cur_offset;
2739                 ret = btrfs_drop_extents(trans, root, inode, &drop_args);
2740                 /* If we are punching a hole decrement the inode's byte count */
2741                 if (!extent_info)
2742                         btrfs_update_inode_bytes(inode, 0,
2743                                                  drop_args.bytes_found);
2744                 if (ret != -ENOSPC) {
2745                         /*
2746                          * The only time we don't want to abort is if we are
2747                          * attempting to clone a partial inline extent, in which
2748                          * case we'll get EOPNOTSUPP.  However if we aren't
2749                          * clone we need to abort no matter what, because if we
2750                          * got EOPNOTSUPP via prealloc then we messed up and
2751                          * need to abort.
2752                          */
2753                         if (ret &&
2754                             (ret != -EOPNOTSUPP ||
2755                              (extent_info && extent_info->is_new_extent)))
2756                                 btrfs_abort_transaction(trans, ret);
2757                         break;
2758                 }
2759
2760                 trans->block_rsv = &fs_info->trans_block_rsv;
2761
2762                 if (!extent_info && cur_offset < drop_args.drop_end &&
2763                     cur_offset < ino_size) {
2764                         ret = fill_holes(trans, inode, path, cur_offset,
2765                                          drop_args.drop_end);
2766                         if (ret) {
2767                                 /*
2768                                  * If we failed then we didn't insert our hole
2769                                  * entries for the area we dropped, so now the
2770                                  * fs is corrupted, so we must abort the
2771                                  * transaction.
2772                                  */
2773                                 btrfs_abort_transaction(trans, ret);
2774                                 break;
2775                         }
2776                 } else if (!extent_info && cur_offset < drop_args.drop_end) {
2777                         /*
2778                          * We are past the i_size here, but since we didn't
2779                          * insert holes we need to clear the mapped area so we
2780                          * know to not set disk_i_size in this area until a new
2781                          * file extent is inserted here.
2782                          */
2783                         ret = btrfs_inode_clear_file_extent_range(inode,
2784                                         cur_offset,
2785                                         drop_args.drop_end - cur_offset);
2786                         if (ret) {
2787                                 /*
2788                                  * We couldn't clear our area, so we could
2789                                  * presumably adjust up and corrupt the fs, so
2790                                  * we need to abort.
2791                                  */
2792                                 btrfs_abort_transaction(trans, ret);
2793                                 break;
2794                         }
2795                 }
2796
2797                 if (extent_info &&
2798                     drop_args.drop_end > extent_info->file_offset) {
2799                         u64 replace_len = drop_args.drop_end -
2800                                           extent_info->file_offset;
2801
2802                         ret = btrfs_insert_replace_extent(trans, inode, path,
2803                                         extent_info, replace_len,
2804                                         drop_args.bytes_found);
2805                         if (ret) {
2806                                 btrfs_abort_transaction(trans, ret);
2807                                 break;
2808                         }
2809                         extent_info->data_len -= replace_len;
2810                         extent_info->data_offset += replace_len;
2811                         extent_info->file_offset += replace_len;
2812                 }
2813
2814                 ret = btrfs_update_inode(trans, root, inode);
2815                 if (ret)
2816                         break;
2817
2818                 btrfs_end_transaction(trans);
2819                 btrfs_btree_balance_dirty(fs_info);
2820
2821                 trans = btrfs_start_transaction(root, rsv_count);
2822                 if (IS_ERR(trans)) {
2823                         ret = PTR_ERR(trans);
2824                         trans = NULL;
2825                         break;
2826                 }
2827
2828                 ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
2829                                               rsv, min_size, false);
2830                 BUG_ON(ret);    /* shouldn't happen */
2831                 trans->block_rsv = rsv;
2832
2833                 cur_offset = drop_args.drop_end;
2834                 len = end - cur_offset;
2835                 if (!extent_info && len) {
2836                         ret = find_first_non_hole(inode, &cur_offset, &len);
2837                         if (unlikely(ret < 0))
2838                                 break;
2839                         if (ret && !len) {
2840                                 ret = 0;
2841                                 break;
2842                         }
2843                 }
2844         }
2845
2846         /*
2847          * If we were cloning, force the next fsync to be a full one since we
2848          * we replaced (or just dropped in the case of cloning holes when
2849          * NO_HOLES is enabled) file extent items and did not setup new extent
2850          * maps for the replacement extents (or holes).
2851          */
2852         if (extent_info && !extent_info->is_new_extent)
2853                 set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
2854
2855         if (ret)
2856                 goto out_trans;
2857
2858         trans->block_rsv = &fs_info->trans_block_rsv;
2859         /*
2860          * If we are using the NO_HOLES feature we might have had already an
2861          * hole that overlaps a part of the region [lockstart, lockend] and
2862          * ends at (or beyond) lockend. Since we have no file extent items to
2863          * represent holes, drop_end can be less than lockend and so we must
2864          * make sure we have an extent map representing the existing hole (the
2865          * call to __btrfs_drop_extents() might have dropped the existing extent
2866          * map representing the existing hole), otherwise the fast fsync path
2867          * will not record the existence of the hole region
2868          * [existing_hole_start, lockend].
2869          */
2870         if (drop_args.drop_end <= end)
2871                 drop_args.drop_end = end + 1;
2872         /*
2873          * Don't insert file hole extent item if it's for a range beyond eof
2874          * (because it's useless) or if it represents a 0 bytes range (when
2875          * cur_offset == drop_end).
2876          */
2877         if (!extent_info && cur_offset < ino_size &&
2878             cur_offset < drop_args.drop_end) {
2879                 ret = fill_holes(trans, inode, path, cur_offset,
2880                                  drop_args.drop_end);
2881                 if (ret) {
2882                         /* Same comment as above. */
2883                         btrfs_abort_transaction(trans, ret);
2884                         goto out_trans;
2885                 }
2886         } else if (!extent_info && cur_offset < drop_args.drop_end) {
2887                 /* See the comment in the loop above for the reasoning here. */
2888                 ret = btrfs_inode_clear_file_extent_range(inode, cur_offset,
2889                                         drop_args.drop_end - cur_offset);
2890                 if (ret) {
2891                         btrfs_abort_transaction(trans, ret);
2892                         goto out_trans;
2893                 }
2894
2895         }
2896         if (extent_info) {
2897                 ret = btrfs_insert_replace_extent(trans, inode, path,
2898                                 extent_info, extent_info->data_len,
2899                                 drop_args.bytes_found);
2900                 if (ret) {
2901                         btrfs_abort_transaction(trans, ret);
2902                         goto out_trans;
2903                 }
2904         }
2905
2906 out_trans:
2907         if (!trans)
2908                 goto out_free;
2909
2910         trans->block_rsv = &fs_info->trans_block_rsv;
2911         if (ret)
2912                 btrfs_end_transaction(trans);
2913         else
2914                 *trans_out = trans;
2915 out_free:
2916         btrfs_free_block_rsv(fs_info, rsv);
2917 out:
2918         return ret;
2919 }
2920
2921 static int btrfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
2922 {
2923         struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
2924         struct btrfs_root *root = BTRFS_I(inode)->root;
2925         struct extent_state *cached_state = NULL;
2926         struct btrfs_path *path;
2927         struct btrfs_trans_handle *trans = NULL;
2928         u64 lockstart;
2929         u64 lockend;
2930         u64 tail_start;
2931         u64 tail_len;
2932         u64 orig_start = offset;
2933         int ret = 0;
2934         bool same_block;
2935         u64 ino_size;
2936         bool truncated_block = false;
2937         bool updated_inode = false;
2938
2939         ret = btrfs_wait_ordered_range(inode, offset, len);
2940         if (ret)
2941                 return ret;
2942
2943         btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
2944         ino_size = round_up(inode->i_size, fs_info->sectorsize);
2945         ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
2946         if (ret < 0)
2947                 goto out_only_mutex;
2948         if (ret && !len) {
2949                 /* Already in a large hole */
2950                 ret = 0;
2951                 goto out_only_mutex;
2952         }
2953
2954         lockstart = round_up(offset, btrfs_inode_sectorsize(BTRFS_I(inode)));
2955         lockend = round_down(offset + len,
2956                              btrfs_inode_sectorsize(BTRFS_I(inode))) - 1;
2957         same_block = (BTRFS_BYTES_TO_BLKS(fs_info, offset))
2958                 == (BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1));
2959         /*
2960          * We needn't truncate any block which is beyond the end of the file
2961          * because we are sure there is no data there.
2962          */
2963         /*
2964          * Only do this if we are in the same block and we aren't doing the
2965          * entire block.
2966          */
2967         if (same_block && len < fs_info->sectorsize) {
2968                 if (offset < ino_size) {
2969                         truncated_block = true;
2970                         ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
2971                                                    0);
2972                 } else {
2973                         ret = 0;
2974                 }
2975                 goto out_only_mutex;
2976         }
2977
2978         /* zero back part of the first block */
2979         if (offset < ino_size) {
2980                 truncated_block = true;
2981                 ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
2982                 if (ret) {
2983                         btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
2984                         return ret;
2985                 }
2986         }
2987
2988         /* Check the aligned pages after the first unaligned page,
2989          * if offset != orig_start, which means the first unaligned page
2990          * including several following pages are already in holes,
2991          * the extra check can be skipped */
2992         if (offset == orig_start) {
2993                 /* after truncate page, check hole again */
2994                 len = offset + len - lockstart;
2995                 offset = lockstart;
2996                 ret = find_first_non_hole(BTRFS_I(inode), &offset, &len);
2997                 if (ret < 0)
2998                         goto out_only_mutex;
2999                 if (ret && !len) {
3000                         ret = 0;
3001                         goto out_only_mutex;
3002                 }
3003                 lockstart = offset;
3004         }
3005
3006         /* Check the tail unaligned part is in a hole */
3007         tail_start = lockend + 1;
3008         tail_len = offset + len - tail_start;
3009         if (tail_len) {
3010                 ret = find_first_non_hole(BTRFS_I(inode), &tail_start, &tail_len);
3011                 if (unlikely(ret < 0))
3012                         goto out_only_mutex;
3013                 if (!ret) {
3014                         /* zero the front end of the last page */
3015                         if (tail_start + tail_len < ino_size) {
3016                                 truncated_block = true;
3017                                 ret = btrfs_truncate_block(BTRFS_I(inode),
3018                                                         tail_start + tail_len,
3019                                                         0, 1);
3020                                 if (ret)
3021                                         goto out_only_mutex;
3022                         }
3023                 }
3024         }
3025
3026         if (lockend < lockstart) {
3027                 ret = 0;
3028                 goto out_only_mutex;
3029         }
3030
3031         ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3032                                           &cached_state);
3033         if (ret)
3034                 goto out_only_mutex;
3035
3036         path = btrfs_alloc_path();
3037         if (!path) {
3038                 ret = -ENOMEM;
3039                 goto out;
3040         }
3041
3042         ret = btrfs_replace_file_extents(BTRFS_I(inode), path, lockstart,
3043                                          lockend, NULL, &trans);
3044         btrfs_free_path(path);
3045         if (ret)
3046                 goto out;
3047
3048         ASSERT(trans != NULL);
3049         inode_inc_iversion(inode);
3050         inode->i_mtime = inode->i_ctime = current_time(inode);
3051         ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3052         updated_inode = true;
3053         btrfs_end_transaction(trans);
3054         btrfs_btree_balance_dirty(fs_info);
3055 out:
3056         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
3057                              &cached_state);
3058 out_only_mutex:
3059         if (!updated_inode && truncated_block && !ret) {
3060                 /*
3061                  * If we only end up zeroing part of a page, we still need to
3062                  * update the inode item, so that all the time fields are
3063                  * updated as well as the necessary btrfs inode in memory fields
3064                  * for detecting, at fsync time, if the inode isn't yet in the
3065                  * log tree or it's there but not up to date.
3066                  */
3067                 struct timespec64 now = current_time(inode);
3068
3069                 inode_inc_iversion(inode);
3070                 inode->i_mtime = now;
3071                 inode->i_ctime = now;
3072                 trans = btrfs_start_transaction(root, 1);
3073                 if (IS_ERR(trans)) {
3074                         ret = PTR_ERR(trans);
3075                 } else {
3076                         int ret2;
3077
3078                         ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3079                         ret2 = btrfs_end_transaction(trans);
3080                         if (!ret)
3081                                 ret = ret2;
3082                 }
3083         }
3084         btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3085         return ret;
3086 }
3087
3088 /* Helper structure to record which range is already reserved */
3089 struct falloc_range {
3090         struct list_head list;
3091         u64 start;
3092         u64 len;
3093 };
3094
3095 /*
3096  * Helper function to add falloc range
3097  *
3098  * Caller should have locked the larger range of extent containing
3099  * [start, len)
3100  */
3101 static int add_falloc_range(struct list_head *head, u64 start, u64 len)
3102 {
3103         struct falloc_range *range = NULL;
3104
3105         if (!list_empty(head)) {
3106                 /*
3107                  * As fallocate iterates by bytenr order, we only need to check
3108                  * the last range.
3109                  */
3110                 range = list_last_entry(head, struct falloc_range, list);
3111                 if (range->start + range->len == start) {
3112                         range->len += len;
3113                         return 0;
3114                 }
3115         }
3116
3117         range = kmalloc(sizeof(*range), GFP_KERNEL);
3118         if (!range)
3119                 return -ENOMEM;
3120         range->start = start;
3121         range->len = len;
3122         list_add_tail(&range->list, head);
3123         return 0;
3124 }
3125
3126 static int btrfs_fallocate_update_isize(struct inode *inode,
3127                                         const u64 end,
3128                                         const int mode)
3129 {
3130         struct btrfs_trans_handle *trans;
3131         struct btrfs_root *root = BTRFS_I(inode)->root;
3132         int ret;
3133         int ret2;
3134
3135         if (mode & FALLOC_FL_KEEP_SIZE || end <= i_size_read(inode))
3136                 return 0;
3137
3138         trans = btrfs_start_transaction(root, 1);
3139         if (IS_ERR(trans))
3140                 return PTR_ERR(trans);
3141
3142         inode->i_ctime = current_time(inode);
3143         i_size_write(inode, end);
3144         btrfs_inode_safe_disk_i_size_write(BTRFS_I(inode), 0);
3145         ret = btrfs_update_inode(trans, root, BTRFS_I(inode));
3146         ret2 = btrfs_end_transaction(trans);
3147
3148         return ret ? ret : ret2;
3149 }
3150
3151 enum {
3152         RANGE_BOUNDARY_WRITTEN_EXTENT,
3153         RANGE_BOUNDARY_PREALLOC_EXTENT,
3154         RANGE_BOUNDARY_HOLE,
3155 };
3156
3157 static int btrfs_zero_range_check_range_boundary(struct btrfs_inode *inode,
3158                                                  u64 offset)
3159 {
3160         const u64 sectorsize = btrfs_inode_sectorsize(inode);
3161         struct extent_map *em;
3162         int ret;
3163
3164         offset = round_down(offset, sectorsize);
3165         em = btrfs_get_extent(inode, NULL, 0, offset, sectorsize);
3166         if (IS_ERR(em))
3167                 return PTR_ERR(em);
3168
3169         if (em->block_start == EXTENT_MAP_HOLE)
3170                 ret = RANGE_BOUNDARY_HOLE;
3171         else if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
3172                 ret = RANGE_BOUNDARY_PREALLOC_EXTENT;
3173         else
3174                 ret = RANGE_BOUNDARY_WRITTEN_EXTENT;
3175
3176         free_extent_map(em);
3177         return ret;
3178 }
3179
3180 static int btrfs_zero_range(struct inode *inode,
3181                             loff_t offset,
3182                             loff_t len,
3183                             const int mode)
3184 {
3185         struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
3186         struct extent_map *em;
3187         struct extent_changeset *data_reserved = NULL;
3188         int ret;
3189         u64 alloc_hint = 0;
3190         const u64 sectorsize = btrfs_inode_sectorsize(BTRFS_I(inode));
3191         u64 alloc_start = round_down(offset, sectorsize);
3192         u64 alloc_end = round_up(offset + len, sectorsize);
3193         u64 bytes_to_reserve = 0;
3194         bool space_reserved = false;
3195
3196         inode_dio_wait(inode);
3197
3198         em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3199                               alloc_end - alloc_start);
3200         if (IS_ERR(em)) {
3201                 ret = PTR_ERR(em);
3202                 goto out;
3203         }
3204
3205         /*
3206          * Avoid hole punching and extent allocation for some cases. More cases
3207          * could be considered, but these are unlikely common and we keep things
3208          * as simple as possible for now. Also, intentionally, if the target
3209          * range contains one or more prealloc extents together with regular
3210          * extents and holes, we drop all the existing extents and allocate a
3211          * new prealloc extent, so that we get a larger contiguous disk extent.
3212          */
3213         if (em->start <= alloc_start &&
3214             test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3215                 const u64 em_end = em->start + em->len;
3216
3217                 if (em_end >= offset + len) {
3218                         /*
3219                          * The whole range is already a prealloc extent,
3220                          * do nothing except updating the inode's i_size if
3221                          * needed.
3222                          */
3223                         free_extent_map(em);
3224                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3225                                                            mode);
3226                         goto out;
3227                 }
3228                 /*
3229                  * Part of the range is already a prealloc extent, so operate
3230                  * only on the remaining part of the range.
3231                  */
3232                 alloc_start = em_end;
3233                 ASSERT(IS_ALIGNED(alloc_start, sectorsize));
3234                 len = offset + len - alloc_start;
3235                 offset = alloc_start;
3236                 alloc_hint = em->block_start + em->len;
3237         }
3238         free_extent_map(em);
3239
3240         if (BTRFS_BYTES_TO_BLKS(fs_info, offset) ==
3241             BTRFS_BYTES_TO_BLKS(fs_info, offset + len - 1)) {
3242                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, alloc_start,
3243                                       sectorsize);
3244                 if (IS_ERR(em)) {
3245                         ret = PTR_ERR(em);
3246                         goto out;
3247                 }
3248
3249                 if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
3250                         free_extent_map(em);
3251                         ret = btrfs_fallocate_update_isize(inode, offset + len,
3252                                                            mode);
3253                         goto out;
3254                 }
3255                 if (len < sectorsize && em->block_start != EXTENT_MAP_HOLE) {
3256                         free_extent_map(em);
3257                         ret = btrfs_truncate_block(BTRFS_I(inode), offset, len,
3258                                                    0);
3259                         if (!ret)
3260                                 ret = btrfs_fallocate_update_isize(inode,
3261                                                                    offset + len,
3262                                                                    mode);
3263                         return ret;
3264                 }
3265                 free_extent_map(em);
3266                 alloc_start = round_down(offset, sectorsize);
3267                 alloc_end = alloc_start + sectorsize;
3268                 goto reserve_space;
3269         }
3270
3271         alloc_start = round_up(offset, sectorsize);
3272         alloc_end = round_down(offset + len, sectorsize);
3273
3274         /*
3275          * For unaligned ranges, check the pages at the boundaries, they might
3276          * map to an extent, in which case we need to partially zero them, or
3277          * they might map to a hole, in which case we need our allocation range
3278          * to cover them.
3279          */
3280         if (!IS_ALIGNED(offset, sectorsize)) {
3281                 ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3282                                                             offset);
3283                 if (ret < 0)
3284                         goto out;
3285                 if (ret == RANGE_BOUNDARY_HOLE) {
3286                         alloc_start = round_down(offset, sectorsize);
3287                         ret = 0;
3288                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3289                         ret = btrfs_truncate_block(BTRFS_I(inode), offset, 0, 0);
3290                         if (ret)
3291                                 goto out;
3292                 } else {
3293                         ret = 0;
3294                 }
3295         }
3296
3297         if (!IS_ALIGNED(offset + len, sectorsize)) {
3298                 ret = btrfs_zero_range_check_range_boundary(BTRFS_I(inode),
3299                                                             offset + len);
3300                 if (ret < 0)
3301                         goto out;
3302                 if (ret == RANGE_BOUNDARY_HOLE) {
3303                         alloc_end = round_up(offset + len, sectorsize);
3304                         ret = 0;
3305                 } else if (ret == RANGE_BOUNDARY_WRITTEN_EXTENT) {
3306                         ret = btrfs_truncate_block(BTRFS_I(inode), offset + len,
3307                                                    0, 1);
3308                         if (ret)
3309                                 goto out;
3310                 } else {
3311                         ret = 0;
3312                 }
3313         }
3314
3315 reserve_space:
3316         if (alloc_start < alloc_end) {
3317                 struct extent_state *cached_state = NULL;
3318                 const u64 lockstart = alloc_start;
3319                 const u64 lockend = alloc_end - 1;
3320
3321                 bytes_to_reserve = alloc_end - alloc_start;
3322                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3323                                                       bytes_to_reserve);
3324                 if (ret < 0)
3325                         goto out;
3326                 space_reserved = true;
3327                 ret = btrfs_punch_hole_lock_range(inode, lockstart, lockend,
3328                                                   &cached_state);
3329                 if (ret)
3330                         goto out;
3331                 ret = btrfs_qgroup_reserve_data(BTRFS_I(inode), &data_reserved,
3332                                                 alloc_start, bytes_to_reserve);
3333                 if (ret) {
3334                         unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3335                                              lockend, &cached_state);
3336                         goto out;
3337                 }
3338                 ret = btrfs_prealloc_file_range(inode, mode, alloc_start,
3339                                                 alloc_end - alloc_start,
3340                                                 i_blocksize(inode),
3341                                                 offset + len, &alloc_hint);
3342                 unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart,
3343                                      lockend, &cached_state);
3344                 /* btrfs_prealloc_file_range releases reserved space on error */
3345                 if (ret) {
3346                         space_reserved = false;
3347                         goto out;
3348                 }
3349         }
3350         ret = btrfs_fallocate_update_isize(inode, offset + len, mode);
3351  out:
3352         if (ret && space_reserved)
3353                 btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3354                                                alloc_start, bytes_to_reserve);
3355         extent_changeset_free(data_reserved);
3356
3357         return ret;
3358 }
3359
3360 static long btrfs_fallocate(struct file *file, int mode,
3361                             loff_t offset, loff_t len)
3362 {
3363         struct inode *inode = file_inode(file);
3364         struct extent_state *cached_state = NULL;
3365         struct extent_changeset *data_reserved = NULL;
3366         struct falloc_range *range;
3367         struct falloc_range *tmp;
3368         struct list_head reserve_list;
3369         u64 cur_offset;
3370         u64 last_byte;
3371         u64 alloc_start;
3372         u64 alloc_end;
3373         u64 alloc_hint = 0;
3374         u64 locked_end;
3375         u64 actual_end = 0;
3376         struct extent_map *em;
3377         int blocksize = btrfs_inode_sectorsize(BTRFS_I(inode));
3378         int ret;
3379
3380         /* Do not allow fallocate in ZONED mode */
3381         if (btrfs_is_zoned(btrfs_sb(inode->i_sb)))
3382                 return -EOPNOTSUPP;
3383
3384         alloc_start = round_down(offset, blocksize);
3385         alloc_end = round_up(offset + len, blocksize);
3386         cur_offset = alloc_start;
3387
3388         /* Make sure we aren't being give some crap mode */
3389         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE |
3390                      FALLOC_FL_ZERO_RANGE))
3391                 return -EOPNOTSUPP;
3392
3393         if (mode & FALLOC_FL_PUNCH_HOLE)
3394                 return btrfs_punch_hole(inode, offset, len);
3395
3396         /*
3397          * Only trigger disk allocation, don't trigger qgroup reserve
3398          *
3399          * For qgroup space, it will be checked later.
3400          */
3401         if (!(mode & FALLOC_FL_ZERO_RANGE)) {
3402                 ret = btrfs_alloc_data_chunk_ondemand(BTRFS_I(inode),
3403                                                       alloc_end - alloc_start);
3404                 if (ret < 0)
3405                         return ret;
3406         }
3407
3408         btrfs_inode_lock(inode, BTRFS_ILOCK_MMAP);
3409
3410         if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size) {
3411                 ret = inode_newsize_ok(inode, offset + len);
3412                 if (ret)
3413                         goto out;
3414         }
3415
3416         /*
3417          * TODO: Move these two operations after we have checked
3418          * accurate reserved space, or fallocate can still fail but
3419          * with page truncated or size expanded.
3420          *
3421          * But that's a minor problem and won't do much harm BTW.
3422          */
3423         if (alloc_start > inode->i_size) {
3424                 ret = btrfs_cont_expand(BTRFS_I(inode), i_size_read(inode),
3425                                         alloc_start);
3426                 if (ret)
3427                         goto out;
3428         } else if (offset + len > inode->i_size) {
3429                 /*
3430                  * If we are fallocating from the end of the file onward we
3431                  * need to zero out the end of the block if i_size lands in the
3432                  * middle of a block.
3433                  */
3434                 ret = btrfs_truncate_block(BTRFS_I(inode), inode->i_size, 0, 0);
3435                 if (ret)
3436                         goto out;
3437         }
3438
3439         /*
3440          * wait for ordered IO before we have any locks.  We'll loop again
3441          * below with the locks held.
3442          */
3443         ret = btrfs_wait_ordered_range(inode, alloc_start,
3444                                        alloc_end - alloc_start);
3445         if (ret)
3446                 goto out;
3447
3448         if (mode & FALLOC_FL_ZERO_RANGE) {
3449                 ret = btrfs_zero_range(inode, offset, len, mode);
3450                 btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3451                 return ret;
3452         }
3453
3454         locked_end = alloc_end - 1;
3455         while (1) {
3456                 struct btrfs_ordered_extent *ordered;
3457
3458                 /* the extent lock is ordered inside the running
3459                  * transaction
3460                  */
3461                 lock_extent_bits(&BTRFS_I(inode)->io_tree, alloc_start,
3462                                  locked_end, &cached_state);
3463                 ordered = btrfs_lookup_first_ordered_extent(BTRFS_I(inode),
3464                                                             locked_end);
3465
3466                 if (ordered &&
3467                     ordered->file_offset + ordered->num_bytes > alloc_start &&
3468                     ordered->file_offset < alloc_end) {
3469                         btrfs_put_ordered_extent(ordered);
3470                         unlock_extent_cached(&BTRFS_I(inode)->io_tree,
3471                                              alloc_start, locked_end,
3472                                              &cached_state);
3473                         /*
3474                          * we can't wait on the range with the transaction
3475                          * running or with the extent lock held
3476                          */
3477                         ret = btrfs_wait_ordered_range(inode, alloc_start,
3478                                                        alloc_end - alloc_start);
3479                         if (ret)
3480                                 goto out;
3481                 } else {
3482                         if (ordered)
3483                                 btrfs_put_ordered_extent(ordered);
3484                         break;
3485                 }
3486         }
3487
3488         /* First, check if we exceed the qgroup limit */
3489         INIT_LIST_HEAD(&reserve_list);
3490         while (cur_offset < alloc_end) {
3491                 em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
3492                                       alloc_end - cur_offset);
3493                 if (IS_ERR(em)) {
3494                         ret = PTR_ERR(em);
3495                         break;
3496                 }
3497                 last_byte = min(extent_map_end(em), alloc_end);
3498                 actual_end = min_t(u64, extent_map_end(em), offset + len);
3499                 last_byte = ALIGN(last_byte, blocksize);
3500                 if (em->block_start == EXTENT_MAP_HOLE ||
3501                     (cur_offset >= inode->i_size &&
3502                      !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))) {
3503                         ret = add_falloc_range(&reserve_list, cur_offset,
3504                                                last_byte - cur_offset);
3505                         if (ret < 0) {
3506                                 free_extent_map(em);
3507                                 break;
3508                         }
3509                         ret = btrfs_qgroup_reserve_data(BTRFS_I(inode),
3510                                         &data_reserved, cur_offset,
3511                                         last_byte - cur_offset);
3512                         if (ret < 0) {
3513                                 cur_offset = last_byte;
3514                                 free_extent_map(em);
3515                                 break;
3516                         }
3517                 } else {
3518                         /*
3519                          * Do not need to reserve unwritten extent for this
3520                          * range, free reserved data space first, otherwise
3521                          * it'll result in false ENOSPC error.
3522                          */
3523                         btrfs_free_reserved_data_space(BTRFS_I(inode),
3524                                 data_reserved, cur_offset,
3525                                 last_byte - cur_offset);
3526                 }
3527                 free_extent_map(em);
3528                 cur_offset = last_byte;
3529         }
3530
3531         /*
3532          * If ret is still 0, means we're OK to fallocate.
3533          * Or just cleanup the list and exit.
3534          */
3535         list_for_each_entry_safe(range, tmp, &reserve_list, list) {
3536                 if (!ret)
3537                         ret = btrfs_prealloc_file_range(inode, mode,
3538                                         range->start,
3539                                         range->len, i_blocksize(inode),
3540                                         offset + len, &alloc_hint);
3541                 else
3542                         btrfs_free_reserved_data_space(BTRFS_I(inode),
3543                                         data_reserved, range->start,
3544                                         range->len);
3545                 list_del(&range->list);
3546                 kfree(range);
3547         }
3548         if (ret < 0)
3549                 goto out_unlock;
3550
3551         /*
3552          * We didn't need to allocate any more space, but we still extended the
3553          * size of the file so we need to update i_size and the inode item.
3554          */
3555         ret = btrfs_fallocate_update_isize(inode, actual_end, mode);
3556 out_unlock:
3557         unlock_extent_cached(&BTRFS_I(inode)->io_tree, alloc_start, locked_end,
3558                              &cached_state);
3559 out:
3560         btrfs_inode_unlock(inode, BTRFS_ILOCK_MMAP);
3561         /* Let go of our reservation. */
3562         if (ret != 0 && !(mode & FALLOC_FL_ZERO_RANGE))
3563                 btrfs_free_reserved_data_space(BTRFS_I(inode), data_reserved,
3564                                 cur_offset, alloc_end - cur_offset);
3565         extent_changeset_free(data_reserved);
3566         return ret;
3567 }
3568
3569 static loff_t find_desired_extent(struct btrfs_inode *inode, loff_t offset,
3570                                   int whence)
3571 {
3572         struct btrfs_fs_info *fs_info = inode->root->fs_info;
3573         struct extent_map *em = NULL;
3574         struct extent_state *cached_state = NULL;
3575         loff_t i_size = inode->vfs_inode.i_size;
3576         u64 lockstart;
3577         u64 lockend;
3578         u64 start;
3579         u64 len;
3580         int ret = 0;
3581
3582         if (i_size == 0 || offset >= i_size)
3583                 return -ENXIO;
3584
3585         /*
3586          * offset can be negative, in this case we start finding DATA/HOLE from
3587          * the very start of the file.
3588          */
3589         start = max_t(loff_t, 0, offset);
3590
3591         lockstart = round_down(start, fs_info->sectorsize);
3592         lockend = round_up(i_size, fs_info->sectorsize);
3593         if (lockend <= lockstart)
3594                 lockend = lockstart + fs_info->sectorsize;
3595         lockend--;
3596         len = lockend - lockstart + 1;
3597
3598         lock_extent_bits(&inode->io_tree, lockstart, lockend, &cached_state);
3599
3600         while (start < i_size) {
3601                 em = btrfs_get_extent_fiemap(inode, start, len);
3602                 if (IS_ERR(em)) {
3603                         ret = PTR_ERR(em);
3604                         em = NULL;
3605                         break;
3606                 }
3607
3608                 if (whence == SEEK_HOLE &&
3609                     (em->block_start == EXTENT_MAP_HOLE ||
3610                      test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3611                         break;
3612                 else if (whence == SEEK_DATA &&
3613                            (em->block_start != EXTENT_MAP_HOLE &&
3614                             !test_bit(EXTENT_FLAG_PREALLOC, &em->flags)))
3615                         break;
3616
3617                 start = em->start + em->len;
3618                 free_extent_map(em);
3619                 em = NULL;
3620                 cond_resched();
3621         }
3622         free_extent_map(em);
3623         unlock_extent_cached(&inode->io_tree, lockstart, lockend,
3624                              &cached_state);
3625         if (ret) {
3626                 offset = ret;
3627         } else {
3628                 if (whence == SEEK_DATA && start >= i_size)
3629                         offset = -ENXIO;
3630                 else
3631                         offset = min_t(loff_t, start, i_size);
3632         }
3633
3634         return offset;
3635 }
3636
3637 static loff_t btrfs_file_llseek(struct file *file, loff_t offset, int whence)
3638 {
3639         struct inode *inode = file->f_mapping->host;
3640
3641         switch (whence) {
3642         default:
3643                 return generic_file_llseek(file, offset, whence);
3644         case SEEK_DATA:
3645         case SEEK_HOLE:
3646                 btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3647                 offset = find_desired_extent(BTRFS_I(inode), offset, whence);
3648                 btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3649                 break;
3650         }
3651
3652         if (offset < 0)
3653                 return offset;
3654
3655         return vfs_setpos(file, offset, inode->i_sb->s_maxbytes);
3656 }
3657
3658 static int btrfs_file_open(struct inode *inode, struct file *filp)
3659 {
3660         int ret;
3661
3662         filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
3663
3664         ret = fsverity_file_open(inode, filp);
3665         if (ret)
3666                 return ret;
3667         return generic_file_open(inode, filp);
3668 }
3669
3670 static int check_direct_read(struct btrfs_fs_info *fs_info,
3671                              const struct iov_iter *iter, loff_t offset)
3672 {
3673         int ret;
3674         int i, seg;
3675
3676         ret = check_direct_IO(fs_info, iter, offset);
3677         if (ret < 0)
3678                 return ret;
3679
3680         if (!iter_is_iovec(iter))
3681                 return 0;
3682
3683         for (seg = 0; seg < iter->nr_segs; seg++)
3684                 for (i = seg + 1; i < iter->nr_segs; i++)
3685                         if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
3686                                 return -EINVAL;
3687         return 0;
3688 }
3689
3690 static ssize_t btrfs_direct_read(struct kiocb *iocb, struct iov_iter *to)
3691 {
3692         struct inode *inode = file_inode(iocb->ki_filp);
3693         size_t prev_left = 0;
3694         ssize_t read = 0;
3695         ssize_t ret;
3696
3697         if (fsverity_active(inode))
3698                 return 0;
3699
3700         if (check_direct_read(btrfs_sb(inode->i_sb), to, iocb->ki_pos))
3701                 return 0;
3702
3703         btrfs_inode_lock(inode, BTRFS_ILOCK_SHARED);
3704 again:
3705         /*
3706          * This is similar to what we do for direct IO writes, see the comment
3707          * at btrfs_direct_write(), but we also disable page faults in addition
3708          * to disabling them only at the iov_iter level. This is because when
3709          * reading from a hole or prealloc extent, iomap calls iov_iter_zero(),
3710          * which can still trigger page fault ins despite having set ->nofault
3711          * to true of our 'to' iov_iter.
3712          *
3713          * The difference to direct IO writes is that we deadlock when trying
3714          * to lock the extent range in the inode's tree during he page reads
3715          * triggered by the fault in (while for writes it is due to waiting for
3716          * our own ordered extent). This is because for direct IO reads,
3717          * btrfs_dio_iomap_begin() returns with the extent range locked, which
3718          * is only unlocked in the endio callback (end_bio_extent_readpage()).
3719          */
3720         pagefault_disable();
3721         to->nofault = true;
3722         ret = iomap_dio_rw(iocb, to, &btrfs_dio_iomap_ops, &btrfs_dio_ops,
3723                            IOMAP_DIO_PARTIAL, read);
3724         to->nofault = false;
3725         pagefault_enable();
3726
3727         /* No increment (+=) because iomap returns a cumulative value. */
3728         if (ret > 0)
3729                 read = ret;
3730
3731         if (iov_iter_count(to) > 0 && (ret == -EFAULT || ret > 0)) {
3732                 const size_t left = iov_iter_count(to);
3733
3734                 if (left == prev_left) {
3735                         /*
3736                          * We didn't make any progress since the last attempt,
3737                          * fallback to a buffered read for the remainder of the
3738                          * range. This is just to avoid any possibility of looping
3739                          * for too long.
3740                          */
3741                         ret = read;
3742                 } else {
3743                         /*
3744                          * We made some progress since the last retry or this is
3745                          * the first time we are retrying. Fault in as many pages
3746                          * as possible and retry.
3747                          */
3748                         fault_in_iov_iter_writeable(to, left);
3749                         prev_left = left;
3750                         goto again;
3751                 }
3752         }
3753         btrfs_inode_unlock(inode, BTRFS_ILOCK_SHARED);
3754         return ret < 0 ? ret : read;
3755 }
3756
3757 static ssize_t btrfs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
3758 {
3759         ssize_t ret = 0;
3760
3761         if (iocb->ki_flags & IOCB_DIRECT) {
3762                 ret = btrfs_direct_read(iocb, to);
3763                 if (ret < 0 || !iov_iter_count(to) ||
3764                     iocb->ki_pos >= i_size_read(file_inode(iocb->ki_filp)))
3765                         return ret;
3766         }
3767
3768         return filemap_read(iocb, to, ret);
3769 }
3770
3771 const struct file_operations btrfs_file_operations = {
3772         .llseek         = btrfs_file_llseek,
3773         .read_iter      = btrfs_file_read_iter,
3774         .splice_read    = generic_file_splice_read,
3775         .write_iter     = btrfs_file_write_iter,
3776         .splice_write   = iter_file_splice_write,
3777         .mmap           = btrfs_file_mmap,
3778         .open           = btrfs_file_open,
3779         .release        = btrfs_release_file,
3780         .fsync          = btrfs_sync_file,
3781         .fallocate      = btrfs_fallocate,
3782         .unlocked_ioctl = btrfs_ioctl,
3783 #ifdef CONFIG_COMPAT
3784         .compat_ioctl   = btrfs_compat_ioctl,
3785 #endif
3786         .remap_file_range = btrfs_remap_file_range,
3787 };
3788
3789 void __cold btrfs_auto_defrag_exit(void)
3790 {
3791         kmem_cache_destroy(btrfs_inode_defrag_cachep);
3792 }
3793
3794 int __init btrfs_auto_defrag_init(void)
3795 {
3796         btrfs_inode_defrag_cachep = kmem_cache_create("btrfs_inode_defrag",
3797                                         sizeof(struct inode_defrag), 0,
3798                                         SLAB_MEM_SPREAD,
3799                                         NULL);
3800         if (!btrfs_inode_defrag_cachep)
3801                 return -ENOMEM;
3802
3803         return 0;
3804 }
3805
3806 int btrfs_fdatawrite_range(struct inode *inode, loff_t start, loff_t end)
3807 {
3808         int ret;
3809
3810         /*
3811          * So with compression we will find and lock a dirty page and clear the
3812          * first one as dirty, setup an async extent, and immediately return
3813          * with the entire range locked but with nobody actually marked with
3814          * writeback.  So we can't just filemap_write_and_wait_range() and
3815          * expect it to work since it will just kick off a thread to do the
3816          * actual work.  So we need to call filemap_fdatawrite_range _again_
3817          * since it will wait on the page lock, which won't be unlocked until
3818          * after the pages have been marked as writeback and so we're good to go
3819          * from there.  We have to do this otherwise we'll miss the ordered
3820          * extents and that results in badness.  Please Josef, do not think you
3821          * know better and pull this out at some point in the future, it is
3822          * right and you are wrong.
3823          */
3824         ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3825         if (!ret && test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
3826                              &BTRFS_I(inode)->runtime_flags))
3827                 ret = filemap_fdatawrite_range(inode->i_mapping, start, end);
3828
3829         return ret;
3830 }