fs/fuse/file.c

   1 /*
   2   FUSE: Filesystem in Userspace
   3   Copyright (C) 2001-2008  Miklos Szeredi <miklos@szeredi.hu>
   4
   5   This program can be distributed under the terms of the GNU GPL.
   6   See the file COPYING.
   7 */
   8
   9 #include "fuse_i.h"
  10
  11 #include <linux/pagemap.h>
  12 #include <linux/slab.h>
  13 #include <linux/kernel.h>
  14 #include <linux/sched.h>
  15 #include <linux/sched/signal.h>
  16 #include <linux/module.h>
  17 #include <linux/compat.h>
  18 #include <linux/swap.h>
  19 #include <linux/falloc.h>
  20 #include <linux/uio.h>
  21 #include <linux/fs.h>
  22
  23 static struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags,
  24                                       struct fuse_page_desc **desc)
  25 {
  26         struct page **pages;
  27
  28         pages = kzalloc(npages * (sizeof(struct page *) +
  29                                   sizeof(struct fuse_page_desc)), flags);
  30         *desc = (void *) (pages + npages);
  31
  32         return pages;
  33 }
  34
  35 static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
  36                           int opcode, struct fuse_open_out *outargp)
  37 {
  38         struct fuse_open_in inarg;
  39         FUSE_ARGS(args);
  40
  41         memset(&inarg, 0, sizeof(inarg));
  42         inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  43         if (!fm->fc->atomic_o_trunc)
  44                 inarg.flags &= ~O_TRUNC;
  45         args.opcode = opcode;
  46         args.nodeid = nodeid;
  47         args.in_numargs = 1;
  48         args.in_args[0].size = sizeof(inarg);
  49         args.in_args[0].value = &inarg;
  50         args.out_numargs = 1;
  51         args.out_args[0].size = sizeof(*outargp);
  52         args.out_args[0].value = outargp;
  53
  54         return fuse_simple_request(fm, &args);
  55 }
  56
  57 struct fuse_release_args {
  58         struct fuse_args args;
  59         struct fuse_release_in inarg;
  60         struct inode *inode;
  61 };
  62
  63 struct fuse_file *fuse_file_alloc(struct fuse_mount *fm)
  64 {
  65         struct fuse_file *ff;
  66
  67         ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
  68         if (unlikely(!ff))
  69                 return NULL;
  70
  71         ff->fm = fm;
  72         ff->release_args = kzalloc(sizeof(*ff->release_args),
  73                                    GFP_KERNEL_ACCOUNT);
  74         if (!ff->release_args) {
  75                 kfree(ff);
  76                 return NULL;
  77         }
  78
  79         INIT_LIST_HEAD(&ff->write_entry);
  80         mutex_init(&ff->readdir.lock);
  81         refcount_set(&ff->count, 1);
  82         RB_CLEAR_NODE(&ff->polled_node);
  83         init_waitqueue_head(&ff->poll_wait);
  84
  85         ff->kh = atomic64_inc_return(&fm->fc->khctr);
  86
  87         return ff;
  88 }
  89
  90 void fuse_file_free(struct fuse_file *ff)
  91 {
  92         kfree(ff->release_args);
  93         mutex_destroy(&ff->readdir.lock);
  94         kfree(ff);
  95 }
  96
  97 static struct fuse_file *fuse_file_get(struct fuse_file *ff)
  98 {
  99         refcount_inc(&ff->count);
 100         return ff;
 101 }
 102
 103 static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
 104                              int error)
 105 {
 106         struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
 107
 108         iput(ra->inode);
 109         kfree(ra);
 110 }
 111
 112 static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
 113 {
 114         if (refcount_dec_and_test(&ff->count)) {
 115                 struct fuse_args *args = &ff->release_args->args;
 116
 117                 if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) {
 118                         /* Do nothing when client does not implement 'open' */
 119                         fuse_release_end(ff->fm, args, 0);
 120                 } else if (sync) {
 121                         fuse_simple_request(ff->fm, args);
 122                         fuse_release_end(ff->fm, args, 0);
 123                 } else {
 124                         args->end = fuse_release_end;
 125                         if (fuse_simple_background(ff->fm, args,
 126                                                    GFP_KERNEL | __GFP_NOFAIL))
 127                                 fuse_release_end(ff->fm, args, -ENOTCONN);
 128                 }
 129                 kfree(ff);
 130         }
 131 }
 132
 133 int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
 134                  bool isdir)
 135 {
 136         struct fuse_conn *fc = fm->fc;
 137         struct fuse_file *ff;
 138         int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
 139
 140         ff = fuse_file_alloc(fm);
 141         if (!ff)
 142                 return -ENOMEM;
 143
 144         ff->fh = 0;
 145         /* Default for no-open */
 146         ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
 147         if (isdir ? !fc->no_opendir : !fc->no_open) {
 148                 struct fuse_open_out outarg;
 149                 int err;
 150
 151                 err = fuse_send_open(fm, nodeid, file, opcode, &outarg);
 152                 if (!err) {
 153                         ff->fh = outarg.fh;
 154                         ff->open_flags = outarg.open_flags;
 155
 156                 } else if (err != -ENOSYS) {
 157                         fuse_file_free(ff);
 158                         return err;
 159                 } else {
 160                         if (isdir)
 161                                 fc->no_opendir = 1;
 162                         else
 163                                 fc->no_open = 1;
 164                 }
 165         }
 166
 167         if (isdir)
 168                 ff->open_flags &= ~FOPEN_DIRECT_IO;
 169
 170         ff->nodeid = nodeid;
 171         file->private_data = ff;
 172
 173         return 0;
 174 }
 175 EXPORT_SYMBOL_GPL(fuse_do_open);
 176
 177 static void fuse_link_write_file(struct file *file)
 178 {
 179         struct inode *inode = file_inode(file);
 180         struct fuse_inode *fi = get_fuse_inode(inode);
 181         struct fuse_file *ff = file->private_data;
 182         /*
 183          * file may be written through mmap, so chain it onto the
 184          * inodes's write_file list
 185          */
 186         spin_lock(&fi->lock);
 187         if (list_empty(&ff->write_entry))
 188                 list_add(&ff->write_entry, &fi->write_files);
 189         spin_unlock(&fi->lock);
 190 }
 191
 192 void fuse_finish_open(struct inode *inode, struct file *file)
 193 {
 194         struct fuse_file *ff = file->private_data;
 195         struct fuse_conn *fc = get_fuse_conn(inode);
 196
 197         if (!(ff->open_flags & FOPEN_KEEP_CACHE))
 198                 invalidate_inode_pages2(inode->i_mapping);
 199         if (ff->open_flags & FOPEN_STREAM)
 200                 stream_open(inode, file);
 201         else if (ff->open_flags & FOPEN_NONSEEKABLE)
 202                 nonseekable_open(inode, file);
 203         if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
 204                 struct fuse_inode *fi = get_fuse_inode(inode);
 205
 206                 spin_lock(&fi->lock);
 207                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 208                 i_size_write(inode, 0);
 209                 spin_unlock(&fi->lock);
 210                 fuse_invalidate_attr(inode);
 211                 if (fc->writeback_cache)
 212                         file_update_time(file);
 213         }
 214         if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
 215                 fuse_link_write_file(file);
 216 }
 217
 218 int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
 219 {
 220         struct fuse_mount *fm = get_fuse_mount(inode);
 221         struct fuse_conn *fc = fm->fc;
 222         int err;
 223         bool is_wb_truncate = (file->f_flags & O_TRUNC) &&
 224                           fc->atomic_o_trunc &&
 225                           fc->writeback_cache;
 226         bool dax_truncate = (file->f_flags & O_TRUNC) &&
 227                           fc->atomic_o_trunc && FUSE_IS_DAX(inode);
 228
 229         err = generic_file_open(inode, file);
 230         if (err)
 231                 return err;
 232
 233         if (is_wb_truncate || dax_truncate) {
 234                 inode_lock(inode);
 235                 fuse_set_nowrite(inode);
 236         }
 237
 238         if (dax_truncate) {
 239                 down_write(&get_fuse_inode(inode)->i_mmap_sem);
 240                 err = fuse_dax_break_layouts(inode, 0, 0);
 241                 if (err)
 242                         goto out;
 243         }
 244
 245         err = fuse_do_open(fm, get_node_id(inode), file, isdir);
 246         if (!err)
 247                 fuse_finish_open(inode, file);
 248
 249 out:
 250         if (dax_truncate)
 251                 up_write(&get_fuse_inode(inode)->i_mmap_sem);
 252
 253         if (is_wb_truncate | dax_truncate) {
 254                 fuse_release_nowrite(inode);
 255                 inode_unlock(inode);
 256         }
 257
 258         return err;
 259 }
 260
 261 static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
 262                                  int flags, int opcode)
 263 {
 264         struct fuse_conn *fc = ff->fm->fc;
 265         struct fuse_release_args *ra = ff->release_args;
 266
 267         /* Inode is NULL on error path of fuse_create_open() */
 268         if (likely(fi)) {
 269                 spin_lock(&fi->lock);
 270                 list_del(&ff->write_entry);
 271                 spin_unlock(&fi->lock);
 272         }
 273         spin_lock(&fc->lock);
 274         if (!RB_EMPTY_NODE(&ff->polled_node))
 275                 rb_erase(&ff->polled_node, &fc->polled_files);
 276         spin_unlock(&fc->lock);
 277
 278         wake_up_interruptible_all(&ff->poll_wait);
 279
 280         ra->inarg.fh = ff->fh;
 281         ra->inarg.flags = flags;
 282         ra->args.in_numargs = 1;
 283         ra->args.in_args[0].size = sizeof(struct fuse_release_in);
 284         ra->args.in_args[0].value = &ra->inarg;
 285         ra->args.opcode = opcode;
 286         ra->args.nodeid = ff->nodeid;
 287         ra->args.force = true;
 288         ra->args.nocreds = true;
 289 }
 290
 291 void fuse_release_common(struct file *file, bool isdir)
 292 {
 293         struct fuse_inode *fi = get_fuse_inode(file_inode(file));
 294         struct fuse_file *ff = file->private_data;
 295         struct fuse_release_args *ra = ff->release_args;
 296         int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
 297
 298         fuse_prepare_release(fi, ff, file->f_flags, opcode);
 299
 300         if (ff->flock) {
 301                 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
 302                 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc,
 303                                                           (fl_owner_t) file);
 304         }
 305         /* Hold inode until release is finished */
 306         ra->inode = igrab(file_inode(file));
 307
 308         /*
 309          * Normally this will send the RELEASE request, however if
 310          * some asynchronous READ or WRITE requests are outstanding,
 311          * the sending will be delayed.
 312          *
 313          * Make the release synchronous if this is a fuseblk mount,
 314          * synchronous RELEASE is allowed (and desirable) in this case
 315          * because the server can be trusted not to screw up.
 316          */
 317         fuse_file_put(ff, ff->fm->fc->destroy, isdir);
 318 }
 319
 320 static int fuse_open(struct inode *inode, struct file *file)
 321 {
 322         return fuse_open_common(inode, file, false);
 323 }
 324
 325 static int fuse_release(struct inode *inode, struct file *file)
 326 {
 327         struct fuse_conn *fc = get_fuse_conn(inode);
 328
 329         /* see fuse_vma_close() for !writeback_cache case */
 330         if (fc->writeback_cache)
 331                 write_inode_now(inode, 1);
 332
 333         fuse_release_common(file, false);
 334
 335         /* return value is ignored by VFS */
 336         return 0;
 337 }
 338
 339 void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, int flags)
 340 {
 341         WARN_ON(refcount_read(&ff->count) > 1);
 342         fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
 343         /*
 344          * iput(NULL) is a no-op and since the refcount is 1 and everything's
 345          * synchronous, we are fine with not doing igrab() here"
 346          */
 347         fuse_file_put(ff, true, false);
 348 }
 349 EXPORT_SYMBOL_GPL(fuse_sync_release);
 350
 351 /*
 352  * Scramble the ID space with XTEA, so that the value of the files_struct
 353  * pointer is not exposed to userspace.
 354  */
 355 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
 356 {
 357         u32 *k = fc->scramble_key;
 358         u64 v = (unsigned long) id;
 359         u32 v0 = v;
 360         u32 v1 = v >> 32;
 361         u32 sum = 0;
 362         int i;
 363
 364         for (i = 0; i < 32; i++) {
 365                 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
 366                 sum += 0x9E3779B9;
 367                 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
 368         }
 369
 370         return (u64) v0 + ((u64) v1 << 32);
 371 }
 372
 373 struct fuse_writepage_args {
 374         struct fuse_io_args ia;
 375         struct rb_node writepages_entry;
 376         struct list_head queue_entry;
 377         struct fuse_writepage_args *next;
 378         struct inode *inode;
 379 };
 380
 381 static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi,
 382                                             pgoff_t idx_from, pgoff_t idx_to)
 383 {
 384         struct rb_node *n;
 385
 386         n = fi->writepages.rb_node;
 387
 388         while (n) {
 389                 struct fuse_writepage_args *wpa;
 390                 pgoff_t curr_index;
 391
 392                 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
 393                 WARN_ON(get_fuse_inode(wpa->inode) != fi);
 394                 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
 395                 if (idx_from >= curr_index + wpa->ia.ap.num_pages)
 396                         n = n->rb_right;
 397                 else if (idx_to < curr_index)
 398                         n = n->rb_left;
 399                 else
 400                         return wpa;
 401         }
 402         return NULL;
 403 }
 404
 405 /*
 406  * Check if any page in a range is under writeback
 407  *
 408  * This is currently done by walking the list of writepage requests
 409  * for the inode, which can be pretty inefficient.
 410  */
 411 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
 412                                    pgoff_t idx_to)
 413 {
 414         struct fuse_inode *fi = get_fuse_inode(inode);
 415         bool found;
 416
 417         spin_lock(&fi->lock);
 418         found = fuse_find_writeback(fi, idx_from, idx_to);
 419         spin_unlock(&fi->lock);
 420
 421         return found;
 422 }
 423
 424 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
 425 {
 426         return fuse_range_is_writeback(inode, index, index);
 427 }
 428
 429 /*
 430  * Wait for page writeback to be completed.
 431  *
 432  * Since fuse doesn't rely on the VM writeback tracking, this has to
 433  * use some other means.
 434  */
 435 static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
 436 {
 437         struct fuse_inode *fi = get_fuse_inode(inode);
 438
 439         wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
 440 }
 441
 442 /*
 443  * Wait for all pending writepages on the inode to finish.
 444  *
 445  * This is currently done by blocking further writes with FUSE_NOWRITE
 446  * and waiting for all sent writes to complete.
 447  *
 448  * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
 449  * could conflict with truncation.
 450  */
 451 static void fuse_sync_writes(struct inode *inode)
 452 {
 453         fuse_set_nowrite(inode);
 454         fuse_release_nowrite(inode);
 455 }
 456
 457 static int fuse_flush(struct file *file, fl_owner_t id)
 458 {
 459         struct inode *inode = file_inode(file);
 460         struct fuse_mount *fm = get_fuse_mount(inode);
 461         struct fuse_file *ff = file->private_data;
 462         struct fuse_flush_in inarg;
 463         FUSE_ARGS(args);
 464         int err;
 465
 466         if (is_bad_inode(inode))
 467                 return -EIO;
 468
 469         err = write_inode_now(inode, 1);
 470         if (err)
 471                 return err;
 472
 473         inode_lock(inode);
 474         fuse_sync_writes(inode);
 475         inode_unlock(inode);
 476
 477         err = filemap_check_errors(file->f_mapping);
 478         if (err)
 479                 return err;
 480
 481         err = 0;
 482         if (fm->fc->no_flush)
 483                 goto inval_attr_out;
 484
 485         memset(&inarg, 0, sizeof(inarg));
 486         inarg.fh = ff->fh;
 487         inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
 488         args.opcode = FUSE_FLUSH;
 489         args.nodeid = get_node_id(inode);
 490         args.in_numargs = 1;
 491         args.in_args[0].size = sizeof(inarg);
 492         args.in_args[0].value = &inarg;
 493         args.force = true;
 494
 495         err = fuse_simple_request(fm, &args);
 496         if (err == -ENOSYS) {
 497                 fm->fc->no_flush = 1;
 498                 err = 0;
 499         }
 500
 501 inval_attr_out:
 502         /*
 503          * In memory i_blocks is not maintained by fuse, if writeback cache is
 504          * enabled, i_blocks from cached attr may not be accurate.
 505          */
 506         if (!err && fm->fc->writeback_cache)
 507                 fuse_invalidate_attr(inode);
 508         return err;
 509 }
 510
 511 int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
 512                       int datasync, int opcode)
 513 {
 514         struct inode *inode = file->f_mapping->host;
 515         struct fuse_mount *fm = get_fuse_mount(inode);
 516         struct fuse_file *ff = file->private_data;
 517         FUSE_ARGS(args);
 518         struct fuse_fsync_in inarg;
 519
 520         memset(&inarg, 0, sizeof(inarg));
 521         inarg.fh = ff->fh;
 522         inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
 523         args.opcode = opcode;
 524         args.nodeid = get_node_id(inode);
 525         args.in_numargs = 1;
 526         args.in_args[0].size = sizeof(inarg);
 527         args.in_args[0].value = &inarg;
 528         return fuse_simple_request(fm, &args);
 529 }
 530
 531 static int fuse_fsync(struct file *file, loff_t start, loff_t end,
 532                       int datasync)
 533 {
 534         struct inode *inode = file->f_mapping->host;
 535         struct fuse_conn *fc = get_fuse_conn(inode);
 536         int err;
 537
 538         if (is_bad_inode(inode))
 539                 return -EIO;
 540
 541         inode_lock(inode);
 542
 543         /*
 544          * Start writeback against all dirty pages of the inode, then
 545          * wait for all outstanding writes, before sending the FSYNC
 546          * request.
 547          */
 548         err = file_write_and_wait_range(file, start, end);
 549         if (err)
 550                 goto out;
 551
 552         fuse_sync_writes(inode);
 553
 554         /*
 555          * Due to implementation of fuse writeback
 556          * file_write_and_wait_range() does not catch errors.
 557          * We have to do this directly after fuse_sync_writes()
 558          */
 559         err = file_check_and_advance_wb_err(file);
 560         if (err)
 561                 goto out;
 562
 563         err = sync_inode_metadata(inode, 1);
 564         if (err)
 565                 goto out;
 566
 567         if (fc->no_fsync)
 568                 goto out;
 569
 570         err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
 571         if (err == -ENOSYS) {
 572                 fc->no_fsync = 1;
 573                 err = 0;
 574         }
 575 out:
 576         inode_unlock(inode);
 577
 578         return err;
 579 }
 580
 581 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
 582                          size_t count, int opcode)
 583 {
 584         struct fuse_file *ff = file->private_data;
 585         struct fuse_args *args = &ia->ap.args;
 586
 587         ia->read.in.fh = ff->fh;
 588         ia->read.in.offset = pos;
 589         ia->read.in.size = count;
 590         ia->read.in.flags = file->f_flags;
 591         args->opcode = opcode;
 592         args->nodeid = ff->nodeid;
 593         args->in_numargs = 1;
 594         args->in_args[0].size = sizeof(ia->read.in);
 595         args->in_args[0].value = &ia->read.in;
 596         args->out_argvar = true;
 597         args->out_numargs = 1;
 598         args->out_args[0].size = count;
 599 }
 600
 601 static void fuse_release_user_pages(struct fuse_args_pages *ap,
 602                                     bool should_dirty)
 603 {
 604         unsigned int i;
 605
 606         for (i = 0; i < ap->num_pages; i++) {
 607                 if (should_dirty)
 608                         set_page_dirty_lock(ap->pages[i]);
 609                 put_page(ap->pages[i]);
 610         }
 611 }
 612
 613 static void fuse_io_release(struct kref *kref)
 614 {
 615         kfree(container_of(kref, struct fuse_io_priv, refcnt));
 616 }
 617
 618 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
 619 {
 620         if (io->err)
 621                 return io->err;
 622
 623         if (io->bytes >= 0 && io->write)
 624                 return -EIO;
 625
 626         return io->bytes < 0 ? io->size : io->bytes;
 627 }
 628
 629 /**
 630  * In case of short read, the caller sets 'pos' to the position of
 631  * actual end of fuse request in IO request. Otherwise, if bytes_requested
 632  * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
 633  *
 634  * An example:
 635  * User requested DIO read of 64K. It was splitted into two 32K fuse requests,
 636  * both submitted asynchronously. The first of them was ACKed by userspace as
 637  * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
 638  * second request was ACKed as short, e.g. only 1K was read, resulting in
 639  * pos == 33K.
 640  *
 641  * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
 642  * will be equal to the length of the longest contiguous fragment of
 643  * transferred data starting from the beginning of IO request.
 644  */
 645 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
 646 {
 647         int left;
 648
 649         spin_lock(&io->lock);
 650         if (err)
 651                 io->err = io->err ? : err;
 652         else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
 653                 io->bytes = pos;
 654
 655         left = --io->reqs;
 656         if (!left && io->blocking)
 657                 complete(io->done);
 658         spin_unlock(&io->lock);
 659
 660         if (!left && !io->blocking) {
 661                 ssize_t res = fuse_get_res_by_io(io);
 662
 663                 if (res >= 0) {
 664                         struct inode *inode = file_inode(io->iocb->ki_filp);
 665                         struct fuse_conn *fc = get_fuse_conn(inode);
 666                         struct fuse_inode *fi = get_fuse_inode(inode);
 667
 668                         spin_lock(&fi->lock);
 669                         fi->attr_version = atomic64_inc_return(&fc->attr_version);
 670                         spin_unlock(&fi->lock);
 671                 }
 672
 673                 io->iocb->ki_complete(io->iocb, res, 0);
 674         }
 675
 676         kref_put(&io->refcnt, fuse_io_release);
 677 }
 678
 679 static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
 680                                           unsigned int npages)
 681 {
 682         struct fuse_io_args *ia;
 683
 684         ia = kzalloc(sizeof(*ia), GFP_KERNEL);
 685         if (ia) {
 686                 ia->io = io;
 687                 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,
 688                                                 &ia->ap.descs);
 689                 if (!ia->ap.pages) {
 690                         kfree(ia);
 691                         ia = NULL;
 692                 }
 693         }
 694         return ia;
 695 }
 696
 697 static void fuse_io_free(struct fuse_io_args *ia)
 698 {
 699         kfree(ia->ap.pages);
 700         kfree(ia);
 701 }
 702
 703 static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
 704                                   int err)
 705 {
 706         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 707         struct fuse_io_priv *io = ia->io;
 708         ssize_t pos = -1;
 709
 710         fuse_release_user_pages(&ia->ap, io->should_dirty);
 711
 712         if (err) {
 713                 /* Nothing */
 714         } else if (io->write) {
 715                 if (ia->write.out.size > ia->write.in.size) {
 716                         err = -EIO;
 717                 } else if (ia->write.in.size != ia->write.out.size) {
 718                         pos = ia->write.in.offset - io->offset +
 719                                 ia->write.out.size;
 720                 }
 721         } else {
 722                 u32 outsize = args->out_args[0].size;
 723
 724                 if (ia->read.in.size != outsize)
 725                         pos = ia->read.in.offset - io->offset + outsize;
 726         }
 727
 728         fuse_aio_complete(io, err, pos);
 729         fuse_io_free(ia);
 730 }
 731
 732 static ssize_t fuse_async_req_send(struct fuse_mount *fm,
 733                                    struct fuse_io_args *ia, size_t num_bytes)
 734 {
 735         ssize_t err;
 736         struct fuse_io_priv *io = ia->io;
 737
 738         spin_lock(&io->lock);
 739         kref_get(&io->refcnt);
 740         io->size += num_bytes;
 741         io->reqs++;
 742         spin_unlock(&io->lock);
 743
 744         ia->ap.args.end = fuse_aio_complete_req;
 745         ia->ap.args.may_block = io->should_dirty;
 746         err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
 747         if (err)
 748                 fuse_aio_complete_req(fm, &ia->ap.args, err);
 749
 750         return num_bytes;
 751 }
 752
 753 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
 754                               fl_owner_t owner)
 755 {
 756         struct file *file = ia->io->iocb->ki_filp;
 757         struct fuse_file *ff = file->private_data;
 758         struct fuse_mount *fm = ff->fm;
 759
 760         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 761         if (owner != NULL) {
 762                 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
 763                 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
 764         }
 765
 766         if (ia->io->async)
 767                 return fuse_async_req_send(fm, ia, count);
 768
 769         return fuse_simple_request(fm, &ia->ap.args);
 770 }
 771
 772 static void fuse_read_update_size(struct inode *inode, loff_t size,
 773                                   u64 attr_ver)
 774 {
 775         struct fuse_conn *fc = get_fuse_conn(inode);
 776         struct fuse_inode *fi = get_fuse_inode(inode);
 777
 778         spin_lock(&fi->lock);
 779         if (attr_ver == fi->attr_version && size < inode->i_size &&
 780             !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
 781                 fi->attr_version = atomic64_inc_return(&fc->attr_version);
 782                 i_size_write(inode, size);
 783         }
 784         spin_unlock(&fi->lock);
 785 }
 786
 787 static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
 788                             struct fuse_args_pages *ap)
 789 {
 790         struct fuse_conn *fc = get_fuse_conn(inode);
 791
 792         if (fc->writeback_cache) {
 793                 /*
 794                  * A hole in a file. Some data after the hole are in page cache,
 795                  * but have not reached the client fs yet. So, the hole is not
 796                  * present there.
 797                  */
 798                 int i;
 799                 int start_idx = num_read >> PAGE_SHIFT;
 800                 size_t off = num_read & (PAGE_SIZE - 1);
 801
 802                 for (i = start_idx; i < ap->num_pages; i++) {
 803                         zero_user_segment(ap->pages[i], off, PAGE_SIZE);
 804                         off = 0;
 805                 }
 806         } else {
 807                 loff_t pos = page_offset(ap->pages[0]) + num_read;
 808                 fuse_read_update_size(inode, pos, attr_ver);
 809         }
 810 }
 811
 812 static int fuse_do_readpage(struct file *file, struct page *page)
 813 {
 814         struct inode *inode = page->mapping->host;
 815         struct fuse_mount *fm = get_fuse_mount(inode);
 816         loff_t pos = page_offset(page);
 817         struct fuse_page_desc desc = { .length = PAGE_SIZE };
 818         struct fuse_io_args ia = {
 819                 .ap.args.page_zeroing = true,
 820                 .ap.args.out_pages = true,
 821                 .ap.num_pages = 1,
 822                 .ap.pages = &page,
 823                 .ap.descs = &desc,
 824         };
 825         ssize_t res;
 826         u64 attr_ver;
 827
 828         /*
 829          * Page writeback can extend beyond the lifetime of the
 830          * page-cache page, so make sure we read a properly synced
 831          * page.
 832          */
 833         fuse_wait_on_page_writeback(inode, page->index);
 834
 835         attr_ver = fuse_get_attr_version(fm->fc);
 836
 837         /* Don't overflow end offset */
 838         if (pos + (desc.length - 1) == LLONG_MAX)
 839                 desc.length--;
 840
 841         fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
 842         res = fuse_simple_request(fm, &ia.ap.args);
 843         if (res < 0)
 844                 return res;
 845         /*
 846          * Short read means EOF.  If file size is larger, truncate it
 847          */
 848         if (res < desc.length)
 849                 fuse_short_read(inode, attr_ver, res, &ia.ap);
 850
 851         SetPageUptodate(page);
 852
 853         return 0;
 854 }
 855
 856 static int fuse_readpage(struct file *file, struct page *page)
 857 {
 858         struct inode *inode = page->mapping->host;
 859         int err;
 860
 861         err = -EIO;
 862         if (is_bad_inode(inode))
 863                 goto out;
 864
 865         err = fuse_do_readpage(file, page);
 866         fuse_invalidate_atime(inode);
 867  out:
 868         unlock_page(page);
 869         return err;
 870 }
 871
 872 static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
 873                                int err)
 874 {
 875         int i;
 876         struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
 877         struct fuse_args_pages *ap = &ia->ap;
 878         size_t count = ia->read.in.size;
 879         size_t num_read = args->out_args[0].size;
 880         struct address_space *mapping = NULL;
 881
 882         for (i = 0; mapping == NULL && i < ap->num_pages; i++)
 883                 mapping = ap->pages[i]->mapping;
 884
 885         if (mapping) {
 886                 struct inode *inode = mapping->host;
 887
 888                 /*
 889                  * Short read means EOF. If file size is larger, truncate it
 890                  */
 891                 if (!err && num_read < count)
 892                         fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
 893
 894                 fuse_invalidate_atime(inode);
 895         }
 896
 897         for (i = 0; i < ap->num_pages; i++) {
 898                 struct page *page = ap->pages[i];
 899
 900                 if (!err)
 901                         SetPageUptodate(page);
 902                 else
 903                         SetPageError(page);
 904                 unlock_page(page);
 905                 put_page(page);
 906         }
 907         if (ia->ff)
 908                 fuse_file_put(ia->ff, false, false);
 909
 910         fuse_io_free(ia);
 911 }
 912
 913 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)
 914 {
 915         struct fuse_file *ff = file->private_data;
 916         struct fuse_mount *fm = ff->fm;
 917         struct fuse_args_pages *ap = &ia->ap;
 918         loff_t pos = page_offset(ap->pages[0]);
 919         size_t count = ap->num_pages << PAGE_SHIFT;
 920         ssize_t res;
 921         int err;
 922
 923         ap->args.out_pages = true;
 924         ap->args.page_zeroing = true;
 925         ap->args.page_replace = true;
 926
 927         /* Don't overflow end offset */
 928         if (pos + (count - 1) == LLONG_MAX) {
 929                 count--;
 930                 ap->descs[ap->num_pages - 1].length--;
 931         }
 932         WARN_ON((loff_t) (pos + count) < 0);
 933
 934         fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
 935         ia->read.attr_ver = fuse_get_attr_version(fm->fc);
 936         if (fm->fc->async_read) {
 937                 ia->ff = fuse_file_get(ff);
 938                 ap->args.end = fuse_readpages_end;
 939                 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
 940                 if (!err)
 941                         return;
 942         } else {
 943                 res = fuse_simple_request(fm, &ap->args);
 944                 err = res < 0 ? res : 0;
 945         }
 946         fuse_readpages_end(fm, &ap->args, err);
 947 }
 948
 949 static void fuse_readahead(struct readahead_control *rac)
 950 {
 951         struct inode *inode = rac->mapping->host;
 952         struct fuse_conn *fc = get_fuse_conn(inode);
 953         unsigned int i, max_pages, nr_pages = 0;
 954
 955         if (is_bad_inode(inode))
 956                 return;
 957
 958         max_pages = min_t(unsigned int, fc->max_pages,
 959                         fc->max_read / PAGE_SIZE);
 960
 961         for (;;) {
 962                 struct fuse_io_args *ia;
 963                 struct fuse_args_pages *ap;
 964
 965                 nr_pages = readahead_count(rac) - nr_pages;
 966                 if (nr_pages > max_pages)
 967                         nr_pages = max_pages;
 968                 if (nr_pages == 0)
 969                         break;
 970                 ia = fuse_io_alloc(NULL, nr_pages);
 971                 if (!ia)
 972                         return;
 973                 ap = &ia->ap;
 974                 nr_pages = __readahead_batch(rac, ap->pages, nr_pages);
 975                 for (i = 0; i < nr_pages; i++) {
 976                         fuse_wait_on_page_writeback(inode,
 977                                                     readahead_index(rac) + i);
 978                         ap->descs[i].length = PAGE_SIZE;
 979                 }
 980                 ap->num_pages = nr_pages;
 981                 fuse_send_readpages(ia, rac->file);
 982         }
 983 }
 984
 985 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
 986 {
 987         struct inode *inode = iocb->ki_filp->f_mapping->host;
 988         struct fuse_conn *fc = get_fuse_conn(inode);
 989
 990         /*
 991          * In auto invalidate mode, always update attributes on read.
 992          * Otherwise, only update if we attempt to read past EOF (to ensure
 993          * i_size is up to date).
 994          */
 995         if (fc->auto_inval_data ||
 996             (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
 997                 int err;
 998                 err = fuse_update_attributes(inode, iocb->ki_filp);
 999                 if (err)
1000                         return err;
1001         }
1002
1003         return generic_file_read_iter(iocb, to);
1004 }
1005
1006 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
1007                                  loff_t pos, size_t count)
1008 {
1009         struct fuse_args *args = &ia->ap.args;
1010
1011         ia->write.in.fh = ff->fh;
1012         ia->write.in.offset = pos;
1013         ia->write.in.size = count;
1014         args->opcode = FUSE_WRITE;
1015         args->nodeid = ff->nodeid;
1016         args->in_numargs = 2;
1017         if (ff->fm->fc->minor < 9)
1018                 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
1019         else
1020                 args->in_args[0].size = sizeof(ia->write.in);
1021         args->in_args[0].value = &ia->write.in;
1022         args->in_args[1].size = count;
1023         args->out_numargs = 1;
1024         args->out_args[0].size = sizeof(ia->write.out);
1025         args->out_args[0].value = &ia->write.out;
1026 }
1027
1028 static unsigned int fuse_write_flags(struct kiocb *iocb)
1029 {
1030         unsigned int flags = iocb->ki_filp->f_flags;
1031
1032         if (iocb->ki_flags & IOCB_DSYNC)
1033                 flags |= O_DSYNC;
1034         if (iocb->ki_flags & IOCB_SYNC)
1035                 flags |= O_SYNC;
1036
1037         return flags;
1038 }
1039
1040 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
1041                                size_t count, fl_owner_t owner)
1042 {
1043         struct kiocb *iocb = ia->io->iocb;
1044         struct file *file = iocb->ki_filp;
1045         struct fuse_file *ff = file->private_data;
1046         struct fuse_mount *fm = ff->fm;
1047         struct fuse_write_in *inarg = &ia->write.in;
1048         ssize_t err;
1049
1050         fuse_write_args_fill(ia, ff, pos, count);
1051         inarg->flags = fuse_write_flags(iocb);
1052         if (owner != NULL) {
1053                 inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
1054                 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
1055         }
1056
1057         if (ia->io->async)
1058                 return fuse_async_req_send(fm, ia, count);
1059
1060         err = fuse_simple_request(fm, &ia->ap.args);
1061         if (!err && ia->write.out.size > count)
1062                 err = -EIO;
1063
1064         return err ?: ia->write.out.size;
1065 }
1066
1067 bool fuse_write_update_size(struct inode *inode, loff_t pos)
1068 {
1069         struct fuse_conn *fc = get_fuse_conn(inode);
1070         struct fuse_inode *fi = get_fuse_inode(inode);
1071         bool ret = false;
1072
1073         spin_lock(&fi->lock);
1074         fi->attr_version = atomic64_inc_return(&fc->attr_version);
1075         if (pos > inode->i_size) {
1076                 i_size_write(inode, pos);
1077                 ret = true;
1078         }
1079         spin_unlock(&fi->lock);
1080
1081         return ret;
1082 }
1083
1084 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
1085                                      struct kiocb *iocb, struct inode *inode,
1086                                      loff_t pos, size_t count)
1087 {
1088         struct fuse_args_pages *ap = &ia->ap;
1089         struct file *file = iocb->ki_filp;
1090         struct fuse_file *ff = file->private_data;
1091         struct fuse_mount *fm = ff->fm;
1092         unsigned int offset, i;
1093         int err;
1094
1095         for (i = 0; i < ap->num_pages; i++)
1096                 fuse_wait_on_page_writeback(inode, ap->pages[i]->index);
1097
1098         fuse_write_args_fill(ia, ff, pos, count);
1099         ia->write.in.flags = fuse_write_flags(iocb);
1100
1101         err = fuse_simple_request(fm, &ap->args);
1102         if (!err && ia->write.out.size > count)
1103                 err = -EIO;
1104
1105         offset = ap->descs[0].offset;
1106         count = ia->write.out.size;
1107         for (i = 0; i < ap->num_pages; i++) {
1108                 struct page *page = ap->pages[i];
1109
1110                 if (!err && !offset && count >= PAGE_SIZE)
1111                         SetPageUptodate(page);
1112
1113                 if (count > PAGE_SIZE - offset)
1114                         count -= PAGE_SIZE - offset;
1115                 else
1116                         count = 0;
1117                 offset = 0;
1118
1119                 unlock_page(page);
1120                 put_page(page);
1121         }
1122
1123         return err;
1124 }
1125
1126 static ssize_t fuse_fill_write_pages(struct fuse_args_pages *ap,
1127                                      struct address_space *mapping,
1128                                      struct iov_iter *ii, loff_t pos,
1129                                      unsigned int max_pages)
1130 {
1131         struct fuse_conn *fc = get_fuse_conn(mapping->host);
1132         unsigned offset = pos & (PAGE_SIZE - 1);
1133         size_t count = 0;
1134         int err;
1135
1136         ap->args.in_pages = true;
1137         ap->descs[0].offset = offset;
1138
1139         do {
1140                 size_t tmp;
1141                 struct page *page;
1142                 pgoff_t index = pos >> PAGE_SHIFT;
1143                 size_t bytes = min_t(size_t, PAGE_SIZE - offset,
1144                                      iov_iter_count(ii));
1145
1146                 bytes = min_t(size_t, bytes, fc->max_write - count);
1147
1148  again:
1149                 err = -EFAULT;
1150                 if (iov_iter_fault_in_readable(ii, bytes))
1151                         break;
1152
1153                 err = -ENOMEM;
1154                 page = grab_cache_page_write_begin(mapping, index, 0);
1155                 if (!page)
1156                         break;
1157
1158                 if (mapping_writably_mapped(mapping))
1159                         flush_dcache_page(page);
1160
1161                 tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
1162                 flush_dcache_page(page);
1163
1164                 iov_iter_advance(ii, tmp);
1165                 if (!tmp) {
1166                         unlock_page(page);
1167                         put_page(page);
1168                         bytes = min(bytes, iov_iter_single_seg_count(ii));
1169                         goto again;
1170                 }
1171
1172                 err = 0;
1173                 ap->pages[ap->num_pages] = page;
1174                 ap->descs[ap->num_pages].length = tmp;
1175                 ap->num_pages++;
1176
1177                 count += tmp;
1178                 pos += tmp;
1179                 offset += tmp;
1180                 if (offset == PAGE_SIZE)
1181                         offset = 0;
1182
1183                 if (!fc->big_writes)
1184                         break;
1185         } while (iov_iter_count(ii) && count < fc->max_write &&
1186                  ap->num_pages < max_pages && offset == 0);
1187
1188         return count > 0 ? count : err;
1189 }
1190
1191 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
1192                                      unsigned int max_pages)
1193 {
1194         return min_t(unsigned int,
1195                      ((pos + len - 1) >> PAGE_SHIFT) -
1196                      (pos >> PAGE_SHIFT) + 1,
1197                      max_pages);
1198 }
1199
1200 static ssize_t fuse_perform_write(struct kiocb *iocb,
1201                                   struct address_space *mapping,
1202                                   struct iov_iter *ii, loff_t pos)
1203 {
1204         struct inode *inode = mapping->host;
1205         struct fuse_conn *fc = get_fuse_conn(inode);
1206         struct fuse_inode *fi = get_fuse_inode(inode);
1207         int err = 0;
1208         ssize_t res = 0;
1209
1210         if (inode->i_size < pos + iov_iter_count(ii))
1211                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1212
1213         do {
1214                 ssize_t count;
1215                 struct fuse_io_args ia = {};
1216                 struct fuse_args_pages *ap = &ia.ap;
1217                 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
1218                                                       fc->max_pages);
1219
1220                 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);
1221                 if (!ap->pages) {
1222                         err = -ENOMEM;
1223                         break;
1224                 }
1225
1226                 count = fuse_fill_write_pages(ap, mapping, ii, pos, nr_pages);
1227                 if (count <= 0) {
1228                         err = count;
1229                 } else {
1230                         err = fuse_send_write_pages(&ia, iocb, inode,
1231                                                     pos, count);
1232                         if (!err) {
1233                                 size_t num_written = ia.write.out.size;
1234
1235                                 res += num_written;
1236                                 pos += num_written;
1237
1238                                 /* break out of the loop on short write */
1239                                 if (num_written != count)
1240                                         err = -EIO;
1241                         }
1242                 }
1243                 kfree(ap->pages);
1244         } while (!err && iov_iter_count(ii));
1245
1246         if (res > 0)
1247                 fuse_write_update_size(inode, pos);
1248
1249         clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
1250         fuse_invalidate_attr(inode);
1251
1252         return res > 0 ? res : err;
1253 }
1254
1255 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
1256 {
1257         struct file *file = iocb->ki_filp;
1258         struct address_space *mapping = file->f_mapping;
1259         ssize_t written = 0;
1260         ssize_t written_buffered = 0;
1261         struct inode *inode = mapping->host;
1262         ssize_t err;
1263         loff_t endbyte = 0;
1264
1265         if (get_fuse_conn(inode)->writeback_cache) {
1266                 /* Update size (EOF optimization) and mode (SUID clearing) */
1267                 err = fuse_update_attributes(mapping->host, file);
1268                 if (err)
1269                         return err;
1270
1271                 return generic_file_write_iter(iocb, from);
1272         }
1273
1274         inode_lock(inode);
1275
1276         /* We can write back this queue in page reclaim */
1277         current->backing_dev_info = inode_to_bdi(inode);
1278
1279         err = generic_write_checks(iocb, from);
1280         if (err <= 0)
1281                 goto out;
1282
1283         err = file_remove_privs(file);
1284         if (err)
1285                 goto out;
1286
1287         err = file_update_time(file);
1288         if (err)
1289                 goto out;
1290
1291         if (iocb->ki_flags & IOCB_DIRECT) {
1292                 loff_t pos = iocb->ki_pos;
1293                 written = generic_file_direct_write(iocb, from);
1294                 if (written < 0 || !iov_iter_count(from))
1295                         goto out;
1296
1297                 pos += written;
1298
1299                 written_buffered = fuse_perform_write(iocb, mapping, from, pos);
1300                 if (written_buffered < 0) {
1301                         err = written_buffered;
1302                         goto out;
1303                 }
1304                 endbyte = pos + written_buffered - 1;
1305
1306                 err = filemap_write_and_wait_range(file->f_mapping, pos,
1307                                                    endbyte);
1308                 if (err)
1309                         goto out;
1310
1311                 invalidate_mapping_pages(file->f_mapping,
1312                                          pos >> PAGE_SHIFT,
1313                                          endbyte >> PAGE_SHIFT);
1314
1315                 written += written_buffered;
1316                 iocb->ki_pos = pos + written_buffered;
1317         } else {
1318                 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
1319                 if (written >= 0)
1320                         iocb->ki_pos += written;
1321         }
1322 out:
1323         current->backing_dev_info = NULL;
1324         inode_unlock(inode);
1325         if (written > 0)
1326                 written = generic_write_sync(iocb, written);
1327
1328         return written ? written : err;
1329 }
1330
1331 static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs,
1332                                                unsigned int index,
1333                                                unsigned int nr_pages)
1334 {
1335         int i;
1336
1337         for (i = index; i < index + nr_pages; i++)
1338                 descs[i].length = PAGE_SIZE - descs[i].offset;
1339 }
1340
1341 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
1342 {
1343         return (unsigned long)ii->iov->iov_base + ii->iov_offset;
1344 }
1345
1346 static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
1347                                         size_t max_size)
1348 {
1349         return min(iov_iter_single_seg_count(ii), max_size);
1350 }
1351
1352 static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
1353                                size_t *nbytesp, int write,
1354                                unsigned int max_pages)
1355 {
1356         size_t nbytes = 0;  /* # bytes already packed in req */
1357         ssize_t ret = 0;
1358
1359         /* Special case for kernel I/O: can copy directly into the buffer */
1360         if (iov_iter_is_kvec(ii)) {
1361                 unsigned long user_addr = fuse_get_user_addr(ii);
1362                 size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
1363
1364                 if (write)
1365                         ap->args.in_args[1].value = (void *) user_addr;
1366                 else
1367                         ap->args.out_args[0].value = (void *) user_addr;
1368
1369                 iov_iter_advance(ii, frag_size);
1370                 *nbytesp = frag_size;
1371                 return 0;
1372         }
1373
1374         while (nbytes < *nbytesp && ap->num_pages < max_pages) {
1375                 unsigned npages;
1376                 size_t start;
1377                 ret = iov_iter_get_pages(ii, &ap->pages[ap->num_pages],
1378                                         *nbytesp - nbytes,
1379                                         max_pages - ap->num_pages,
1380                                         &start);
1381                 if (ret < 0)
1382                         break;
1383
1384                 iov_iter_advance(ii, ret);
1385                 nbytes += ret;
1386
1387                 ret += start;
1388                 npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
1389
1390                 ap->descs[ap->num_pages].offset = start;
1391                 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);
1392
1393                 ap->num_pages += npages;
1394                 ap->descs[ap->num_pages - 1].length -=
1395                         (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
1396         }
1397
1398         if (write)
1399                 ap->args.in_pages = true;
1400         else
1401                 ap->args.out_pages = true;
1402
1403         *nbytesp = nbytes;
1404
1405         return ret < 0 ? ret : 0;
1406 }
1407
1408 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
1409                        loff_t *ppos, int flags)
1410 {
1411         int write = flags & FUSE_DIO_WRITE;
1412         int cuse = flags & FUSE_DIO_CUSE;
1413         struct file *file = io->iocb->ki_filp;
1414         struct inode *inode = file->f_mapping->host;
1415         struct fuse_file *ff = file->private_data;
1416         struct fuse_conn *fc = ff->fm->fc;
1417         size_t nmax = write ? fc->max_write : fc->max_read;
1418         loff_t pos = *ppos;
1419         size_t count = iov_iter_count(iter);
1420         pgoff_t idx_from = pos >> PAGE_SHIFT;
1421         pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
1422         ssize_t res = 0;
1423         int err = 0;
1424         struct fuse_io_args *ia;
1425         unsigned int max_pages;
1426
1427         max_pages = iov_iter_npages(iter, fc->max_pages);
1428         ia = fuse_io_alloc(io, max_pages);
1429         if (!ia)
1430                 return -ENOMEM;
1431
1432         ia->io = io;
1433         if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
1434                 if (!write)
1435                         inode_lock(inode);
1436                 fuse_sync_writes(inode);
1437                 if (!write)
1438                         inode_unlock(inode);
1439         }
1440
1441         io->should_dirty = !write && iter_is_iovec(iter);
1442         while (count) {
1443                 ssize_t nres;
1444                 fl_owner_t owner = current->files;
1445                 size_t nbytes = min(count, nmax);
1446
1447                 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
1448                                           max_pages);
1449                 if (err && !nbytes)
1450                         break;
1451
1452                 if (write) {
1453                         if (!capable(CAP_FSETID))
1454                                 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID;
1455
1456                         nres = fuse_send_write(ia, pos, nbytes, owner);
1457                 } else {
1458                         nres = fuse_send_read(ia, pos, nbytes, owner);
1459                 }
1460
1461                 if (!io->async || nres < 0) {
1462                         fuse_release_user_pages(&ia->ap, io->should_dirty);
1463                         fuse_io_free(ia);
1464                 }
1465                 ia = NULL;
1466                 if (nres < 0) {
1467                         iov_iter_revert(iter, nbytes);
1468                         err = nres;
1469                         break;
1470                 }
1471                 WARN_ON(nres > nbytes);
1472
1473                 count -= nres;
1474                 res += nres;
1475                 pos += nres;
1476                 if (nres != nbytes) {
1477                         iov_iter_revert(iter, nbytes - nres);
1478                         break;
1479                 }
1480                 if (count) {
1481                         max_pages = iov_iter_npages(iter, fc->max_pages);
1482                         ia = fuse_io_alloc(io, max_pages);
1483                         if (!ia)
1484                                 break;
1485                 }
1486         }
1487         if (ia)
1488                 fuse_io_free(ia);
1489         if (res > 0)
1490                 *ppos = pos;
1491
1492         return res > 0 ? res : err;
1493 }
1494 EXPORT_SYMBOL_GPL(fuse_direct_io);
1495
1496 static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
1497                                   struct iov_iter *iter,
1498                                   loff_t *ppos)
1499 {
1500         ssize_t res;
1501         struct inode *inode = file_inode(io->iocb->ki_filp);
1502
1503         res = fuse_direct_io(io, iter, ppos, 0);
1504
1505         fuse_invalidate_atime(inode);
1506
1507         return res;
1508 }
1509
1510 static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
1511
1512 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
1513 {
1514         ssize_t res;
1515
1516         if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1517                 res = fuse_direct_IO(iocb, to);
1518         } else {
1519                 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1520
1521                 res = __fuse_direct_read(&io, to, &iocb->ki_pos);
1522         }
1523
1524         return res;
1525 }
1526
1527 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
1528 {
1529         struct inode *inode = file_inode(iocb->ki_filp);
1530         struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
1531         ssize_t res;
1532
1533         /* Don't allow parallel writes to the same file */
1534         inode_lock(inode);
1535         res = generic_write_checks(iocb, from);
1536         if (res > 0) {
1537                 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
1538                         res = fuse_direct_IO(iocb, from);
1539                 } else {
1540                         res = fuse_direct_io(&io, from, &iocb->ki_pos,
1541                                              FUSE_DIO_WRITE);
1542                 }
1543         }
1544         fuse_invalidate_attr(inode);
1545         if (res > 0)
1546                 fuse_write_update_size(inode, iocb->ki_pos);
1547         inode_unlock(inode);
1548
1549         return res;
1550 }
1551
1552 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
1553 {
1554         struct file *file = iocb->ki_filp;
1555         struct fuse_file *ff = file->private_data;
1556         struct inode *inode = file_inode(file);
1557
1558         if (is_bad_inode(inode))
1559                 return -EIO;
1560
1561         if (FUSE_IS_DAX(inode))
1562                 return fuse_dax_read_iter(iocb, to);
1563
1564         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1565                 return fuse_cache_read_iter(iocb, to);
1566         else
1567                 return fuse_direct_read_iter(iocb, to);
1568 }
1569
1570 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
1571 {
1572         struct file *file = iocb->ki_filp;
1573         struct fuse_file *ff = file->private_data;
1574         struct inode *inode = file_inode(file);
1575
1576         if (is_bad_inode(inode))
1577                 return -EIO;
1578
1579         if (FUSE_IS_DAX(inode))
1580                 return fuse_dax_write_iter(iocb, from);
1581
1582         if (!(ff->open_flags & FOPEN_DIRECT_IO))
1583                 return fuse_cache_write_iter(iocb, from);
1584         else
1585                 return fuse_direct_write_iter(iocb, from);
1586 }
1587
1588 static void fuse_writepage_free(struct fuse_writepage_args *wpa)
1589 {
1590         struct fuse_args_pages *ap = &wpa->ia.ap;
1591         int i;
1592
1593         for (i = 0; i < ap->num_pages; i++)
1594                 __free_page(ap->pages[i]);
1595
1596         if (wpa->ia.ff)
1597                 fuse_file_put(wpa->ia.ff, false, false);
1598
1599         kfree(ap->pages);
1600         kfree(wpa);
1601 }
1602
1603 static void fuse_writepage_finish(struct fuse_mount *fm,
1604                                   struct fuse_writepage_args *wpa)
1605 {
1606         struct fuse_args_pages *ap = &wpa->ia.ap;
1607         struct inode *inode = wpa->inode;
1608         struct fuse_inode *fi = get_fuse_inode(inode);
1609         struct backing_dev_info *bdi = inode_to_bdi(inode);
1610         int i;
1611
1612         for (i = 0; i < ap->num_pages; i++) {
1613                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
1614                 dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP);
1615                 wb_writeout_inc(&bdi->wb);
1616         }
1617         wake_up(&fi->page_waitq);
1618 }
1619
1620 /* Called under fi->lock, may release and reacquire it */
1621 static void fuse_send_writepage(struct fuse_mount *fm,
1622                                 struct fuse_writepage_args *wpa, loff_t size)
1623 __releases(fi->lock)
1624 __acquires(fi->lock)
1625 {
1626         struct fuse_writepage_args *aux, *next;
1627         struct fuse_inode *fi = get_fuse_inode(wpa->inode);
1628         struct fuse_write_in *inarg = &wpa->ia.write.in;
1629         struct fuse_args *args = &wpa->ia.ap.args;
1630         __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;
1631         int err;
1632
1633         fi->writectr++;
1634         if (inarg->offset + data_size <= size) {
1635                 inarg->size = data_size;
1636         } else if (inarg->offset < size) {
1637                 inarg->size = size - inarg->offset;
1638         } else {
1639                 /* Got truncated off completely */
1640                 goto out_free;
1641         }
1642
1643         args->in_args[1].size = inarg->size;
1644         args->force = true;
1645         args->nocreds = true;
1646
1647         err = fuse_simple_background(fm, args, GFP_ATOMIC);
1648         if (err == -ENOMEM) {
1649                 spin_unlock(&fi->lock);
1650                 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
1651                 spin_lock(&fi->lock);
1652         }
1653
1654         /* Fails on broken connection only */
1655         if (unlikely(err))
1656                 goto out_free;
1657
1658         return;
1659
1660  out_free:
1661         fi->writectr--;
1662         rb_erase(&wpa->writepages_entry, &fi->writepages);
1663         fuse_writepage_finish(fm, wpa);
1664         spin_unlock(&fi->lock);
1665
1666         /* After fuse_writepage_finish() aux request list is private */
1667         for (aux = wpa->next; aux; aux = next) {
1668                 next = aux->next;
1669                 aux->next = NULL;
1670                 fuse_writepage_free(aux);
1671         }
1672
1673         fuse_writepage_free(wpa);
1674         spin_lock(&fi->lock);
1675 }
1676
1677 /*
1678  * If fi->writectr is positive (no truncate or fsync going on) send
1679  * all queued writepage requests.
1680  *
1681  * Called with fi->lock
1682  */
1683 void fuse_flush_writepages(struct inode *inode)
1684 __releases(fi->lock)
1685 __acquires(fi->lock)
1686 {
1687         struct fuse_mount *fm = get_fuse_mount(inode);
1688         struct fuse_inode *fi = get_fuse_inode(inode);
1689         loff_t crop = i_size_read(inode);
1690         struct fuse_writepage_args *wpa;
1691
1692         while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
1693                 wpa = list_entry(fi->queued_writes.next,
1694                                  struct fuse_writepage_args, queue_entry);
1695                 list_del_init(&wpa->queue_entry);
1696                 fuse_send_writepage(fm, wpa, crop);
1697         }
1698 }
1699
1700 static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root,
1701                                                 struct fuse_writepage_args *wpa)
1702 {
1703         pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
1704         pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;
1705         struct rb_node **p = &root->rb_node;
1706         struct rb_node  *parent = NULL;
1707
1708         WARN_ON(!wpa->ia.ap.num_pages);
1709         while (*p) {
1710                 struct fuse_writepage_args *curr;
1711                 pgoff_t curr_index;
1712
1713                 parent = *p;
1714                 curr = rb_entry(parent, struct fuse_writepage_args,
1715                                 writepages_entry);
1716                 WARN_ON(curr->inode != wpa->inode);
1717                 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
1718
1719                 if (idx_from >= curr_index + curr->ia.ap.num_pages)
1720                         p = &(*p)->rb_right;
1721                 else if (idx_to < curr_index)
1722                         p = &(*p)->rb_left;
1723                 else
1724                         return curr;
1725         }
1726
1727         rb_link_node(&wpa->writepages_entry, parent, p);
1728         rb_insert_color(&wpa->writepages_entry, root);
1729         return NULL;
1730 }
1731
1732 static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa)
1733 {
1734         WARN_ON(fuse_insert_writeback(root, wpa));
1735 }
1736
1737 static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
1738                                int error)
1739 {
1740         struct fuse_writepage_args *wpa =
1741                 container_of(args, typeof(*wpa), ia.ap.args);
1742         struct inode *inode = wpa->inode;
1743         struct fuse_inode *fi = get_fuse_inode(inode);
1744
1745         mapping_set_error(inode->i_mapping, error);
1746         spin_lock(&fi->lock);
1747         rb_erase(&wpa->writepages_entry, &fi->writepages);
1748         while (wpa->next) {
1749                 struct fuse_mount *fm = get_fuse_mount(inode);
1750                 struct fuse_write_in *inarg = &wpa->ia.write.in;
1751                 struct fuse_writepage_args *next = wpa->next;
1752
1753                 wpa->next = next->next;
1754                 next->next = NULL;
1755                 next->ia.ff = fuse_file_get(wpa->ia.ff);
1756                 tree_insert(&fi->writepages, next);
1757
1758                 /*
1759                  * Skip fuse_flush_writepages() to make it easy to crop requests
1760                  * based on primary request size.
1761                  *
1762                  * 1st case (trivial): there are no concurrent activities using
1763                  * fuse_set/release_nowrite.  Then we're on safe side because
1764                  * fuse_flush_writepages() would call fuse_send_writepage()
1765                  * anyway.
1766                  *
1767                  * 2nd case: someone called fuse_set_nowrite and it is waiting
1768                  * now for completion of all in-flight requests.  This happens
1769                  * rarely and no more than once per page, so this should be
1770                  * okay.
1771                  *
1772                  * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle
1773                  * of fuse_set_nowrite..fuse_release_nowrite section.  The fact
1774                  * that fuse_set_nowrite returned implies that all in-flight
1775                  * requests were completed along with all of their secondary
1776                  * requests.  Further primary requests are blocked by negative
1777                  * writectr.  Hence there cannot be any in-flight requests and
1778                  * no invocations of fuse_writepage_end() while we're in
1779                  * fuse_set_nowrite..fuse_release_nowrite section.
1780                  */
1781                 fuse_send_writepage(fm, next, inarg->offset + inarg->size);
1782         }
1783         fi->writectr--;
1784         fuse_writepage_finish(fm, wpa);
1785         spin_unlock(&fi->lock);
1786         fuse_writepage_free(wpa);
1787 }
1788
1789 static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
1790                                                struct fuse_inode *fi)
1791 {
1792         struct fuse_file *ff = NULL;
1793
1794         spin_lock(&fi->lock);
1795         if (!list_empty(&fi->write_files)) {
1796                 ff = list_entry(fi->write_files.next, struct fuse_file,
1797                                 write_entry);
1798                 fuse_file_get(ff);
1799         }
1800         spin_unlock(&fi->lock);
1801
1802         return ff;
1803 }
1804
1805 static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
1806                                              struct fuse_inode *fi)
1807 {
1808         struct fuse_file *ff = __fuse_write_file_get(fc, fi);
1809         WARN_ON(!ff);
1810         return ff;
1811 }
1812
1813 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
1814 {
1815         struct fuse_conn *fc = get_fuse_conn(inode);
1816         struct fuse_inode *fi = get_fuse_inode(inode);
1817         struct fuse_file *ff;
1818         int err;
1819
1820         ff = __fuse_write_file_get(fc, fi);
1821         err = fuse_flush_times(inode, ff);
1822         if (ff)
1823                 fuse_file_put(ff, false, false);
1824
1825         return err;
1826 }
1827
1828 static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
1829 {
1830         struct fuse_writepage_args *wpa;
1831         struct fuse_args_pages *ap;
1832
1833         wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
1834         if (wpa) {
1835                 ap = &wpa->ia.ap;
1836                 ap->num_pages = 0;
1837                 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);
1838                 if (!ap->pages) {
1839                         kfree(wpa);
1840                         wpa = NULL;
1841                 }
1842         }
1843         return wpa;
1844
1845 }
1846
1847 static int fuse_writepage_locked(struct page *page)
1848 {
1849         struct address_space *mapping = page->mapping;
1850         struct inode *inode = mapping->host;
1851         struct fuse_conn *fc = get_fuse_conn(inode);
1852         struct fuse_inode *fi = get_fuse_inode(inode);
1853         struct fuse_writepage_args *wpa;
1854         struct fuse_args_pages *ap;
1855         struct page *tmp_page;
1856         int error = -ENOMEM;
1857
1858         set_page_writeback(page);
1859
1860         wpa = fuse_writepage_args_alloc();
1861         if (!wpa)
1862                 goto err;
1863         ap = &wpa->ia.ap;
1864
1865         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
1866         if (!tmp_page)
1867                 goto err_free;
1868
1869         error = -EIO;
1870         wpa->ia.ff = fuse_write_file_get(fc, fi);
1871         if (!wpa->ia.ff)
1872                 goto err_nofile;
1873
1874         fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);
1875
1876         copy_highpage(tmp_page, page);
1877         wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
1878         wpa->next = NULL;
1879         ap->args.in_pages = true;
1880         ap->num_pages = 1;
1881         ap->pages[0] = tmp_page;
1882         ap->descs[0].offset = 0;
1883         ap->descs[0].length = PAGE_SIZE;
1884         ap->args.end = fuse_writepage_end;
1885         wpa->inode = inode;
1886
1887         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
1888         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
1889
1890         spin_lock(&fi->lock);
1891         tree_insert(&fi->writepages, wpa);
1892         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
1893         fuse_flush_writepages(inode);
1894         spin_unlock(&fi->lock);
1895
1896         end_page_writeback(page);
1897
1898         return 0;
1899
1900 err_nofile:
1901         __free_page(tmp_page);
1902 err_free:
1903         kfree(wpa);
1904 err:
1905         mapping_set_error(page->mapping, error);
1906         end_page_writeback(page);
1907         return error;
1908 }
1909
1910 static int fuse_writepage(struct page *page, struct writeback_control *wbc)
1911 {
1912         int err;
1913
1914         if (fuse_page_is_writeback(page->mapping->host, page->index)) {
1915                 /*
1916                  * ->writepages() should be called for sync() and friends.  We
1917                  * should only get here on direct reclaim and then we are
1918                  * allowed to skip a page which is already in flight
1919                  */
1920                 WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
1921
1922                 redirty_page_for_writepage(wbc, page);
1923                 unlock_page(page);
1924                 return 0;
1925         }
1926
1927         err = fuse_writepage_locked(page);
1928         unlock_page(page);
1929
1930         return err;
1931 }
1932
1933 struct fuse_fill_wb_data {
1934         struct fuse_writepage_args *wpa;
1935         struct fuse_file *ff;
1936         struct inode *inode;
1937         struct page **orig_pages;
1938         unsigned int max_pages;
1939 };
1940
1941 static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
1942 {
1943         struct fuse_args_pages *ap = &data->wpa->ia.ap;
1944         struct fuse_conn *fc = get_fuse_conn(data->inode);
1945         struct page **pages;
1946         struct fuse_page_desc *descs;
1947         unsigned int npages = min_t(unsigned int,
1948                                     max_t(unsigned int, data->max_pages * 2,
1949                                           FUSE_DEFAULT_MAX_PAGES_PER_REQ),
1950                                     fc->max_pages);
1951         WARN_ON(npages <= data->max_pages);
1952
1953         pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);
1954         if (!pages)
1955                 return false;
1956
1957         memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);
1958         memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);
1959         kfree(ap->pages);
1960         ap->pages = pages;
1961         ap->descs = descs;
1962         data->max_pages = npages;
1963
1964         return true;
1965 }
1966
1967 static void fuse_writepages_send(struct fuse_fill_wb_data *data)
1968 {
1969         struct fuse_writepage_args *wpa = data->wpa;
1970         struct inode *inode = data->inode;
1971         struct fuse_inode *fi = get_fuse_inode(inode);
1972         int num_pages = wpa->ia.ap.num_pages;
1973         int i;
1974
1975         wpa->ia.ff = fuse_file_get(data->ff);
1976         spin_lock(&fi->lock);
1977         list_add_tail(&wpa->queue_entry, &fi->queued_writes);
1978         fuse_flush_writepages(inode);
1979         spin_unlock(&fi->lock);
1980
1981         for (i = 0; i < num_pages; i++)
1982                 end_page_writeback(data->orig_pages[i]);
1983 }
1984
1985 /*
1986  * Check under fi->lock if the page is under writeback, and insert it onto the
1987  * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's
1988  * one already added for a page at this offset.  If there's none, then insert
1989  * this new request onto the auxiliary list, otherwise reuse the existing one by
1990  * swapping the new temp page with the old one.
1991  */
1992 static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
1993                                struct page *page)
1994 {
1995         struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
1996         struct fuse_writepage_args *tmp;
1997         struct fuse_writepage_args *old_wpa;
1998         struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
1999
2000         WARN_ON(new_ap->num_pages != 0);
2001         new_ap->num_pages = 1;
2002
2003         spin_lock(&fi->lock);
2004         old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
2005         if (!old_wpa) {
2006                 spin_unlock(&fi->lock);
2007                 return true;
2008         }
2009
2010         for (tmp = old_wpa->next; tmp; tmp = tmp->next) {
2011                 pgoff_t curr_index;
2012
2013                 WARN_ON(tmp->inode != new_wpa->inode);
2014                 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
2015                 if (curr_index == page->index) {
2016                         WARN_ON(tmp->ia.ap.num_pages != 1);
2017                         swap(tmp->ia.ap.pages[0], new_ap->pages[0]);
2018                         break;
2019                 }
2020         }
2021
2022         if (!tmp) {
2023                 new_wpa->next = old_wpa->next;
2024                 old_wpa->next = new_wpa;
2025         }
2026
2027         spin_unlock(&fi->lock);
2028
2029         if (tmp) {
2030                 struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode);
2031
2032                 dec_wb_stat(&bdi->wb, WB_WRITEBACK);
2033                 dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP);
2034                 wb_writeout_inc(&bdi->wb);
2035                 fuse_writepage_free(new_wpa);
2036         }
2037
2038         return false;
2039 }
2040
2041 static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,
2042                                      struct fuse_args_pages *ap,
2043                                      struct fuse_fill_wb_data *data)
2044 {
2045         WARN_ON(!ap->num_pages);
2046
2047         /*
2048          * Being under writeback is unlikely but possible.  For example direct
2049          * read to an mmaped fuse file will set the page dirty twice; once when
2050          * the pages are faulted with get_user_pages(), and then after the read
2051          * completed.
2052          */
2053         if (fuse_page_is_writeback(data->inode, page->index))
2054                 return true;
2055
2056         /* Reached max pages */
2057         if (ap->num_pages == fc->max_pages)
2058                 return true;
2059
2060         /* Reached max write bytes */
2061         if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)
2062                 return true;
2063
2064         /* Discontinuity */
2065         if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)
2066                 return true;
2067
2068         /* Need to grow the pages array?  If so, did the expansion fail? */
2069         if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))
2070                 return true;
2071
2072         return false;
2073 }
2074
2075 static int fuse_writepages_fill(struct page *page,
2076                 struct writeback_control *wbc, void *_data)
2077 {
2078         struct fuse_fill_wb_data *data = _data;
2079         struct fuse_writepage_args *wpa = data->wpa;
2080         struct fuse_args_pages *ap = &wpa->ia.ap;
2081         struct inode *inode = data->inode;
2082         struct fuse_inode *fi = get_fuse_inode(inode);
2083         struct fuse_conn *fc = get_fuse_conn(inode);
2084         struct page *tmp_page;
2085         int err;
2086
2087         if (!data->ff) {
2088                 err = -EIO;
2089                 data->ff = fuse_write_file_get(fc, fi);
2090                 if (!data->ff)
2091                         goto out_unlock;
2092         }
2093
2094         if (wpa && fuse_writepage_need_send(fc, page, ap, data)) {
2095                 fuse_writepages_send(data);
2096                 data->wpa = NULL;
2097         }
2098
2099         err = -ENOMEM;
2100         tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
2101         if (!tmp_page)
2102                 goto out_unlock;
2103
2104         /*
2105          * The page must not be redirtied until the writeout is completed
2106          * (i.e. userspace has sent a reply to the write request).  Otherwise
2107          * there could be more than one temporary page instance for each real
2108          * page.
2109          *
2110          * This is ensured by holding the page lock in page_mkwrite() while
2111          * checking fuse_page_is_writeback().  We already hold the page lock
2112          * since clear_page_dirty_for_io() and keep it held until we add the
2113          * request to the fi->writepages list and increment ap->num_pages.
2114          * After this fuse_page_is_writeback() will indicate that the page is
2115          * under writeback, so we can release the page lock.
2116          */
2117         if (data->wpa == NULL) {
2118                 err = -ENOMEM;
2119                 wpa = fuse_writepage_args_alloc();
2120                 if (!wpa) {
2121                         __free_page(tmp_page);
2122                         goto out_unlock;
2123                 }
2124                 data->max_pages = 1;
2125
2126                 ap = &wpa->ia.ap;
2127                 fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);
2128                 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
2129                 wpa->next = NULL;
2130                 ap->args.in_pages = true;
2131                 ap->args.end = fuse_writepage_end;
2132                 ap->num_pages = 0;
2133                 wpa->inode = inode;
2134         }
2135         set_page_writeback(page);
2136
2137         copy_highpage(tmp_page, page);
2138         ap->pages[ap->num_pages] = tmp_page;
2139         ap->descs[ap->num_pages].offset = 0;
2140         ap->descs[ap->num_pages].length = PAGE_SIZE;
2141         data->orig_pages[ap->num_pages] = page;
2142
2143         inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
2144         inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
2145
2146         err = 0;
2147         if (data->wpa) {
2148                 /*
2149                  * Protected by fi->lock against concurrent access by
2150                  * fuse_page_is_writeback().
2151                  */
2152                 spin_lock(&fi->lock);
2153                 ap->num_pages++;
2154                 spin_unlock(&fi->lock);
2155         } else if (fuse_writepage_add(wpa, page)) {
2156                 data->wpa = wpa;
2157         } else {
2158                 end_page_writeback(page);
2159         }
2160 out_unlock:
2161         unlock_page(page);
2162
2163         return err;
2164 }
2165
2166 static int fuse_writepages(struct address_space *mapping,
2167                            struct writeback_control *wbc)
2168 {
2169         struct inode *inode = mapping->host;
2170         struct fuse_conn *fc = get_fuse_conn(inode);
2171         struct fuse_fill_wb_data data;
2172         int err;
2173
2174         err = -EIO;
2175         if (is_bad_inode(inode))
2176                 goto out;
2177
2178         data.inode = inode;
2179         data.wpa = NULL;
2180         data.ff = NULL;
2181
2182         err = -ENOMEM;
2183         data.orig_pages = kcalloc(fc->max_pages,
2184                                   sizeof(struct page *),
2185                                   GFP_NOFS);
2186         if (!data.orig_pages)
2187                 goto out;
2188
2189         err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
2190         if (data.wpa) {
2191                 WARN_ON(!data.wpa->ia.ap.num_pages);
2192                 fuse_writepages_send(&data);
2193         }
2194         if (data.ff)
2195                 fuse_file_put(data.ff, false, false);
2196
2197         kfree(data.orig_pages);
2198 out:
2199         return err;
2200 }
2201
2202 /*
2203  * It's worthy to make sure that space is reserved on disk for the write,
2204  * but how to implement it without killing performance need more thinking.
2205  */
2206 static int fuse_write_begin(struct file *file, struct address_space *mapping,
2207                 loff_t pos, unsigned len, unsigned flags,
2208                 struct page **pagep, void **fsdata)
2209 {
2210         pgoff_t index = pos >> PAGE_SHIFT;
2211         struct fuse_conn *fc = get_fuse_conn(file_inode(file));
2212         struct page *page;
2213         loff_t fsize;
2214         int err = -ENOMEM;
2215
2216         WARN_ON(!fc->writeback_cache);
2217
2218         page = grab_cache_page_write_begin(mapping, index, flags);
2219         if (!page)
2220                 goto error;
2221
2222         fuse_wait_on_page_writeback(mapping->host, page->index);
2223
2224         if (PageUptodate(page) || len == PAGE_SIZE)
2225                 goto success;
2226         /*
2227          * Check if the start this page comes after the end of file, in which
2228          * case the readpage can be optimized away.
2229          */
2230         fsize = i_size_read(mapping->host);
2231         if (fsize <= (pos & PAGE_MASK)) {
2232                 size_t off = pos & ~PAGE_MASK;
2233                 if (off)
2234                         zero_user_segment(page, 0, off);
2235                 goto success;
2236         }
2237         err = fuse_do_readpage(file, page);
2238         if (err)
2239                 goto cleanup;
2240 success:
2241         *pagep = page;
2242         return 0;
2243
2244 cleanup:
2245         unlock_page(page);
2246         put_page(page);
2247 error:
2248         return err;
2249 }
2250
2251 static int fuse_write_end(struct file *file, struct address_space *mapping,
2252                 loff_t pos, unsigned len, unsigned copied,
2253                 struct page *page, void *fsdata)
2254 {
2255         struct inode *inode = page->mapping->host;
2256
2257         /* Haven't copied anything?  Skip zeroing, size extending, dirtying. */
2258         if (!copied)
2259                 goto unlock;
2260
2261         if (!PageUptodate(page)) {
2262                 /* Zero any unwritten bytes at the end of the page */
2263                 size_t endoff = (pos + copied) & ~PAGE_MASK;
2264                 if (endoff)
2265                         zero_user_segment(page, endoff, PAGE_SIZE);
2266                 SetPageUptodate(page);
2267         }
2268
2269         fuse_write_update_size(inode, pos + copied);
2270         set_page_dirty(page);
2271
2272 unlock:
2273         unlock_page(page);
2274         put_page(page);
2275
2276         return copied;
2277 }
2278
2279 static int fuse_launder_page(struct page *page)
2280 {
2281         int err = 0;
2282         if (clear_page_dirty_for_io(page)) {
2283                 struct inode *inode = page->mapping->host;
2284
2285                 /* Serialize with pending writeback for the same page */
2286                 fuse_wait_on_page_writeback(inode, page->index);
2287                 err = fuse_writepage_locked(page);
2288                 if (!err)
2289                         fuse_wait_on_page_writeback(inode, page->index);
2290         }
2291         return err;
2292 }
2293
2294 /*
2295  * Write back dirty pages now, because there may not be any suitable
2296  * open files later
2297  */
2298 static void fuse_vma_close(struct vm_area_struct *vma)
2299 {
2300         filemap_write_and_wait(vma->vm_file->f_mapping);
2301 }
2302
2303 /*
2304  * Wait for writeback against this page to complete before allowing it
2305  * to be marked dirty again, and hence written back again, possibly
2306  * before the previous writepage completed.
2307  *
2308  * Block here, instead of in ->writepage(), so that the userspace fs
2309  * can only block processes actually operating on the filesystem.
2310  *
2311  * Otherwise unprivileged userspace fs would be able to block
2312  * unrelated:
2313  *
2314  * - page migration
2315  * - sync(2)
2316  * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
2317  */
2318 static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
2319 {
2320         struct page *page = vmf->page;
2321         struct inode *inode = file_inode(vmf->vma->vm_file);
2322
2323         file_update_time(vmf->vma->vm_file);
2324         lock_page(page);
2325         if (page->mapping != inode->i_mapping) {
2326                 unlock_page(page);
2327                 return VM_FAULT_NOPAGE;
2328         }
2329
2330         fuse_wait_on_page_writeback(inode, page->index);
2331         return VM_FAULT_LOCKED;
2332 }
2333
2334 static const struct vm_operations_struct fuse_file_vm_ops = {
2335         .close          = fuse_vma_close,
2336         .fault          = filemap_fault,
2337         .map_pages      = filemap_map_pages,
2338         .page_mkwrite   = fuse_page_mkwrite,
2339 };
2340
2341 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
2342 {
2343         struct fuse_file *ff = file->private_data;
2344
2345         /* DAX mmap is superior to direct_io mmap */
2346         if (FUSE_IS_DAX(file_inode(file)))
2347                 return fuse_dax_mmap(file, vma);
2348
2349         if (ff->open_flags & FOPEN_DIRECT_IO) {
2350                 /* Can't provide the coherency needed for MAP_SHARED */
2351                 if (vma->vm_flags & VM_MAYSHARE)
2352                         return -ENODEV;
2353
2354                 invalidate_inode_pages2(file->f_mapping);
2355
2356                 return generic_file_mmap(file, vma);
2357         }
2358
2359         if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
2360                 fuse_link_write_file(file);
2361
2362         file_accessed(file);
2363         vma->vm_ops = &fuse_file_vm_ops;
2364         return 0;
2365 }
2366
2367 static int convert_fuse_file_lock(struct fuse_conn *fc,
2368                                   const struct fuse_file_lock *ffl,
2369                                   struct file_lock *fl)
2370 {
2371         switch (ffl->type) {
2372         case F_UNLCK:
2373                 break;
2374
2375         case F_RDLCK:
2376         case F_WRLCK:
2377                 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
2378                     ffl->end < ffl->start)
2379                         return -EIO;
2380
2381                 fl->fl_start = ffl->start;
2382                 fl->fl_end = ffl->end;
2383
2384                 /*
2385                  * Convert pid into init's pid namespace.  The locks API will
2386                  * translate it into the caller's pid namespace.
2387                  */
2388                 rcu_read_lock();
2389                 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
2390                 rcu_read_unlock();
2391                 break;
2392
2393         default:
2394                 return -EIO;
2395         }
2396         fl->fl_type = ffl->type;
2397         return 0;
2398 }
2399
2400 static void fuse_lk_fill(struct fuse_args *args, struct file *file,
2401                          const struct file_lock *fl, int opcode, pid_t pid,
2402                          int flock, struct fuse_lk_in *inarg)
2403 {
2404         struct inode *inode = file_inode(file);
2405         struct fuse_conn *fc = get_fuse_conn(inode);
2406         struct fuse_file *ff = file->private_data;
2407
2408         memset(inarg, 0, sizeof(*inarg));
2409         inarg->fh = ff->fh;
2410         inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
2411         inarg->lk.start = fl->fl_start;
2412         inarg->lk.end = fl->fl_end;
2413         inarg->lk.type = fl->fl_type;
2414         inarg->lk.pid = pid;
2415         if (flock)
2416                 inarg->lk_flags |= FUSE_LK_FLOCK;
2417         args->opcode = opcode;
2418         args->nodeid = get_node_id(inode);
2419         args->in_numargs = 1;
2420         args->in_args[0].size = sizeof(*inarg);
2421         args->in_args[0].value = inarg;
2422 }
2423
2424 static int fuse_getlk(struct file *file, struct file_lock *fl)
2425 {
2426         struct inode *inode = file_inode(file);
2427         struct fuse_mount *fm = get_fuse_mount(inode);
2428         FUSE_ARGS(args);
2429         struct fuse_lk_in inarg;
2430         struct fuse_lk_out outarg;
2431         int err;
2432
2433         fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
2434         args.out_numargs = 1;
2435         args.out_args[0].size = sizeof(outarg);
2436         args.out_args[0].value = &outarg;
2437         err = fuse_simple_request(fm, &args);
2438         if (!err)
2439                 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
2440
2441         return err;
2442 }
2443
2444 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
2445 {
2446         struct inode *inode = file_inode(file);
2447         struct fuse_mount *fm = get_fuse_mount(inode);
2448         FUSE_ARGS(args);
2449         struct fuse_lk_in inarg;
2450         int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
2451         struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
2452         pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
2453         int err;
2454
2455         if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
2456                 /* NLM needs asynchronous locks, which we don't support yet */
2457                 return -ENOLCK;
2458         }
2459
2460         /* Unlock on close is handled by the flush method */
2461         if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
2462                 return 0;
2463
2464         fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
2465         err = fuse_simple_request(fm, &args);
2466
2467         /* locking is restartable */
2468         if (err == -EINTR)
2469                 err = -ERESTARTSYS;
2470
2471         return err;
2472 }
2473
2474 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
2475 {
2476         struct inode *inode = file_inode(file);
2477         struct fuse_conn *fc = get_fuse_conn(inode);
2478         int err;
2479
2480         if (cmd == F_CANCELLK) {
2481                 err = 0;
2482         } else if (cmd == F_GETLK) {
2483                 if (fc->no_lock) {
2484                         posix_test_lock(file, fl);
2485                         err = 0;
2486                 } else
2487                         err = fuse_getlk(file, fl);
2488         } else {
2489                 if (fc->no_lock)
2490                         err = posix_lock_file(file, fl, NULL);
2491                 else
2492                         err = fuse_setlk(file, fl, 0);
2493         }
2494         return err;
2495 }
2496
2497 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
2498 {
2499         struct inode *inode = file_inode(file);
2500         struct fuse_conn *fc = get_fuse_conn(inode);
2501         int err;
2502
2503         if (fc->no_flock) {
2504                 err = locks_lock_file_wait(file, fl);
2505         } else {
2506                 struct fuse_file *ff = file->private_data;
2507
2508                 /* emulate flock with POSIX locks */
2509                 ff->flock = true;
2510                 err = fuse_setlk(file, fl, 1);
2511         }
2512
2513         return err;
2514 }
2515
2516 static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
2517 {
2518         struct inode *inode = mapping->host;
2519         struct fuse_mount *fm = get_fuse_mount(inode);
2520         FUSE_ARGS(args);
2521         struct fuse_bmap_in inarg;
2522         struct fuse_bmap_out outarg;
2523         int err;
2524
2525         if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
2526                 return 0;
2527
2528         memset(&inarg, 0, sizeof(inarg));
2529         inarg.block = block;
2530         inarg.blocksize = inode->i_sb->s_blocksize;
2531         args.opcode = FUSE_BMAP;
2532         args.nodeid = get_node_id(inode);
2533         args.in_numargs = 1;
2534         args.in_args[0].size = sizeof(inarg);
2535         args.in_args[0].value = &inarg;
2536         args.out_numargs = 1;
2537         args.out_args[0].size = sizeof(outarg);
2538         args.out_args[0].value = &outarg;
2539         err = fuse_simple_request(fm, &args);
2540         if (err == -ENOSYS)
2541                 fm->fc->no_bmap = 1;
2542
2543         return err ? 0 : outarg.block;
2544 }
2545
2546 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
2547 {
2548         struct inode *inode = file->f_mapping->host;
2549         struct fuse_mount *fm = get_fuse_mount(inode);
2550         struct fuse_file *ff = file->private_data;
2551         FUSE_ARGS(args);
2552         struct fuse_lseek_in inarg = {
2553                 .fh = ff->fh,
2554                 .offset = offset,
2555                 .whence = whence
2556         };
2557         struct fuse_lseek_out outarg;
2558         int err;
2559
2560         if (fm->fc->no_lseek)
2561                 goto fallback;
2562
2563         args.opcode = FUSE_LSEEK;
2564         args.nodeid = ff->nodeid;
2565         args.in_numargs = 1;
2566         args.in_args[0].size = sizeof(inarg);
2567         args.in_args[0].value = &inarg;
2568         args.out_numargs = 1;
2569         args.out_args[0].size = sizeof(outarg);
2570         args.out_args[0].value = &outarg;
2571         err = fuse_simple_request(fm, &args);
2572         if (err) {
2573                 if (err == -ENOSYS) {
2574                         fm->fc->no_lseek = 1;
2575                         goto fallback;
2576                 }
2577                 return err;
2578         }
2579
2580         return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
2581
2582 fallback:
2583         err = fuse_update_attributes(inode, file);
2584         if (!err)
2585                 return generic_file_llseek(file, offset, whence);
2586         else
2587                 return err;
2588 }
2589
2590 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
2591 {
2592         loff_t retval;
2593         struct inode *inode = file_inode(file);
2594
2595         switch (whence) {
2596         case SEEK_SET:
2597         case SEEK_CUR:
2598                  /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
2599                 retval = generic_file_llseek(file, offset, whence);
2600                 break;
2601         case SEEK_END:
2602                 inode_lock(inode);
2603                 retval = fuse_update_attributes(inode, file);
2604                 if (!retval)
2605                         retval = generic_file_llseek(file, offset, whence);
2606                 inode_unlock(inode);
2607                 break;
2608         case SEEK_HOLE:
2609         case SEEK_DATA:
2610                 inode_lock(inode);
2611                 retval = fuse_lseek(file, offset, whence);
2612                 inode_unlock(inode);
2613                 break;
2614         default:
2615                 retval = -EINVAL;
2616         }
2617
2618         return retval;
2619 }
2620
2621 /*
2622  * CUSE servers compiled on 32bit broke on 64bit kernels because the
2623  * ABI was defined to be 'struct iovec' which is different on 32bit
2624  * and 64bit.  Fortunately we can determine which structure the server
2625  * used from the size of the reply.
2626  */
2627 static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
2628                                      size_t transferred, unsigned count,
2629                                      bool is_compat)
2630 {
2631 #ifdef CONFIG_COMPAT
2632         if (count * sizeof(struct compat_iovec) == transferred) {
2633                 struct compat_iovec *ciov = src;
2634                 unsigned i;
2635
2636                 /*
2637                  * With this interface a 32bit server cannot support
2638                  * non-compat (i.e. ones coming from 64bit apps) ioctl
2639                  * requests
2640                  */
2641                 if (!is_compat)
2642                         return -EINVAL;
2643
2644                 for (i = 0; i < count; i++) {
2645                         dst[i].iov_base = compat_ptr(ciov[i].iov_base);
2646                         dst[i].iov_len = ciov[i].iov_len;
2647                 }
2648                 return 0;
2649         }
2650 #endif
2651
2652         if (count * sizeof(struct iovec) != transferred)
2653                 return -EIO;
2654
2655         memcpy(dst, src, transferred);
2656         return 0;
2657 }
2658
2659 /* Make sure iov_length() won't overflow */
2660 static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov,
2661                                  size_t count)
2662 {
2663         size_t n;
2664         u32 max = fc->max_pages << PAGE_SHIFT;
2665
2666         for (n = 0; n < count; n++, iov++) {
2667                 if (iov->iov_len > (size_t) max)
2668                         return -ENOMEM;
2669                 max -= iov->iov_len;
2670         }
2671         return 0;
2672 }
2673
2674 static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
2675                                  void *src, size_t transferred, unsigned count,
2676                                  bool is_compat)
2677 {
2678         unsigned i;
2679         struct fuse_ioctl_iovec *fiov = src;
2680
2681         if (fc->minor < 16) {
2682                 return fuse_copy_ioctl_iovec_old(dst, src, transferred,
2683                                                  count, is_compat);
2684         }
2685
2686         if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
2687                 return -EIO;
2688
2689         for (i = 0; i < count; i++) {
2690                 /* Did the server supply an inappropriate value? */
2691                 if (fiov[i].base != (unsigned long) fiov[i].base ||
2692                     fiov[i].len != (unsigned long) fiov[i].len)
2693                         return -EIO;
2694
2695                 dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
2696                 dst[i].iov_len = (size_t) fiov[i].len;
2697
2698 #ifdef CONFIG_COMPAT
2699                 if (is_compat &&
2700                     (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
2701                      (compat_size_t) dst[i].iov_len != fiov[i].len))
2702                         return -EIO;
2703 #endif
2704         }
2705
2706         return 0;
2707 }
2708
2709
2710 /*
2711  * For ioctls, there is no generic way to determine how much memory
2712  * needs to be read and/or written.  Furthermore, ioctls are allowed
2713  * to dereference the passed pointer, so the parameter requires deep
2714  * copying but FUSE has no idea whatsoever about what to copy in or
2715  * out.
2716  *
2717  * This is solved by allowing FUSE server to retry ioctl with
2718  * necessary in/out iovecs.  Let's assume the ioctl implementation
2719  * needs to read in the following structure.
2720  *
2721  * struct a {
2722  *      char    *buf;
2723  *      size_t  buflen;
2724  * }
2725  *
2726  * On the first callout to FUSE server, inarg->in_size and
2727  * inarg->out_size will be NULL; then, the server completes the ioctl
2728  * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and
2729  * the actual iov array to
2730  *
2731  * { { .iov_base = inarg.arg,   .iov_len = sizeof(struct a) } }
2732  *
2733  * which tells FUSE to copy in the requested area and retry the ioctl.
2734  * On the second round, the server has access to the structure and
2735  * from that it can tell what to look for next, so on the invocation,
2736  * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to
2737  *
2738  * { { .iov_base = inarg.arg,   .iov_len = sizeof(struct a)     },
2739  *   { .iov_base = a.buf,       .iov_len = a.buflen             } }
2740  *
2741  * FUSE will copy both struct a and the pointed buffer from the
2742  * process doing the ioctl and retry ioctl with both struct a and the
2743  * buffer.
2744  *
2745  * This time, FUSE server has everything it needs and completes ioctl
2746  * without FUSE_IOCTL_RETRY which finishes the ioctl call.
2747  *
2748  * Copying data out works the same way.
2749  *
2750  * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel
2751  * automatically initializes in and out iovs by decoding @cmd with
2752  * _IOC_* macros and the server is not allowed to request RETRY.  This
2753  * limits ioctl data transfers to well-formed ioctls and is the forced
2754  * behavior for all FUSE servers.
2755  */
2756 long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
2757                    unsigned int flags)
2758 {
2759         struct fuse_file *ff = file->private_data;
2760         struct fuse_mount *fm = ff->fm;
2761         struct fuse_ioctl_in inarg = {
2762                 .fh = ff->fh,
2763                 .cmd = cmd,
2764                 .arg = arg,
2765                 .flags = flags
2766         };
2767         struct fuse_ioctl_out outarg;
2768         struct iovec *iov_page = NULL;
2769         struct iovec *in_iov = NULL, *out_iov = NULL;
2770         unsigned int in_iovs = 0, out_iovs = 0, max_pages;
2771         size_t in_size, out_size, c;
2772         ssize_t transferred;
2773         int err, i;
2774         struct iov_iter ii;
2775         struct fuse_args_pages ap = {};
2776
2777 #if BITS_PER_LONG == 32
2778         inarg.flags |= FUSE_IOCTL_32BIT;
2779 #else
2780         if (flags & FUSE_IOCTL_COMPAT) {
2781                 inarg.flags |= FUSE_IOCTL_32BIT;
2782 #ifdef CONFIG_X86_X32
2783                 if (in_x32_syscall())
2784                         inarg.flags |= FUSE_IOCTL_COMPAT_X32;
2785 #endif
2786         }
2787 #endif
2788
2789         /* assume all the iovs returned by client always fits in a page */
2790         BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
2791
2792         err = -ENOMEM;
2793         ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs);
2794         iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
2795         if (!ap.pages || !iov_page)
2796                 goto out;
2797
2798         fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages);
2799
2800         /*
2801          * If restricted, initialize IO parameters as encoded in @cmd.
2802          * RETRY from server is not allowed.
2803          */
2804         if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
2805                 struct iovec *iov = iov_page;
2806
2807                 iov->iov_base = (void __user *)arg;
2808
2809                 switch (cmd) {
2810                 case FS_IOC_GETFLAGS:
2811                 case FS_IOC_SETFLAGS:
2812                         iov->iov_len = sizeof(int);
2813                         break;
2814                 default:
2815                         iov->iov_len = _IOC_SIZE(cmd);
2816                         break;
2817                 }
2818
2819                 if (_IOC_DIR(cmd) & _IOC_WRITE) {
2820                         in_iov = iov;
2821                         in_iovs = 1;
2822                 }
2823
2824                 if (_IOC_DIR(cmd) & _IOC_READ) {
2825                         out_iov = iov;
2826                         out_iovs = 1;
2827                 }
2828         }
2829
2830  retry:
2831         inarg.in_size = in_size = iov_length(in_iov, in_iovs);
2832         inarg.out_size = out_size = iov_length(out_iov, out_iovs);
2833
2834         /*
2835          * Out data can be used either for actual out data or iovs,
2836          * make sure there always is at least one page.
2837          */
2838         out_size = max_t(size_t, out_size, PAGE_SIZE);
2839         max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
2840
2841         /* make sure there are enough buffer pages and init request with them */
2842         err = -ENOMEM;
2843         if (max_pages > fm->fc->max_pages)
2844                 goto out;
2845         while (ap.num_pages < max_pages) {
2846                 ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
2847                 if (!ap.pages[ap.num_pages])
2848                         goto out;
2849                 ap.num_pages++;
2850         }
2851
2852
2853         /* okay, let's send it to the client */
2854         ap.args.opcode = FUSE_IOCTL;
2855         ap.args.nodeid = ff->nodeid;
2856         ap.args.in_numargs = 1;
2857         ap.args.in_args[0].size = sizeof(inarg);
2858         ap.args.in_args[0].value = &inarg;
2859         if (in_size) {
2860                 ap.args.in_numargs++;
2861                 ap.args.in_args[1].size = in_size;
2862                 ap.args.in_pages = true;
2863
2864                 err = -EFAULT;
2865                 iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
2866                 for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
2867                         c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
2868                         if (c != PAGE_SIZE && iov_iter_count(&ii))
2869                                 goto out;
2870                 }
2871         }
2872
2873         ap.args.out_numargs = 2;
2874         ap.args.out_args[0].size = sizeof(outarg);
2875         ap.args.out_args[0].value = &outarg;
2876         ap.args.out_args[1].size = out_size;
2877         ap.args.out_pages = true;
2878         ap.args.out_argvar = true;
2879
2880         transferred = fuse_simple_request(fm, &ap.args);
2881         err = transferred;
2882         if (transferred < 0)
2883                 goto out;
2884
2885         /* did it ask for retry? */
2886         if (outarg.flags & FUSE_IOCTL_RETRY) {
2887                 void *vaddr;
2888
2889                 /* no retry if in restricted mode */
2890                 err = -EIO;
2891                 if (!(flags & FUSE_IOCTL_UNRESTRICTED))
2892                         goto out;
2893
2894                 in_iovs = outarg.in_iovs;
2895                 out_iovs = outarg.out_iovs;
2896
2897                 /*
2898                  * Make sure things are in boundary, separate checks
2899                  * are to protect against overflow.
2900                  */
2901                 err = -ENOMEM;
2902                 if (in_iovs > FUSE_IOCTL_MAX_IOV ||
2903                     out_iovs > FUSE_IOCTL_MAX_IOV ||
2904                     in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
2905                         goto out;
2906
2907                 vaddr = kmap_atomic(ap.pages[0]);
2908                 err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr,
2909                                             transferred, in_iovs + out_iovs,
2910                                             (flags & FUSE_IOCTL_COMPAT) != 0);
2911                 kunmap_atomic(vaddr);
2912                 if (err)
2913                         goto out;
2914
2915                 in_iov = iov_page;
2916                 out_iov = in_iov + in_iovs;
2917
2918                 err = fuse_verify_ioctl_iov(fm->fc, in_iov, in_iovs);
2919                 if (err)
2920                         goto out;
2921
2922                 err = fuse_verify_ioctl_iov(fm->fc, out_iov, out_iovs);
2923                 if (err)
2924                         goto out;
2925
2926                 goto retry;
2927         }
2928
2929         err = -EIO;
2930         if (transferred > inarg.out_size)
2931                 goto out;
2932
2933         err = -EFAULT;
2934         iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
2935         for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
2936                 c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
2937                 if (c != PAGE_SIZE && iov_iter_count(&ii))
2938                         goto out;
2939         }
2940         err = 0;
2941  out:
2942         free_page((unsigned long) iov_page);
2943         while (ap.num_pages)
2944                 __free_page(ap.pages[--ap.num_pages]);
2945         kfree(ap.pages);
2946
2947         return err ? err : outarg.result;
2948 }
2949 EXPORT_SYMBOL_GPL(fuse_do_ioctl);
2950
2951 long fuse_ioctl_common(struct file *file, unsigned int cmd,
2952                        unsigned long arg, unsigned int flags)
2953 {
2954         struct inode *inode = file_inode(file);
2955         struct fuse_conn *fc = get_fuse_conn(inode);
2956
2957         if (!fuse_allow_current_process(fc))
2958                 return -EACCES;
2959
2960         if (is_bad_inode(inode))
2961                 return -EIO;
2962
2963         return fuse_do_ioctl(file, cmd, arg, flags);
2964 }
2965
2966 static long fuse_file_ioctl(struct file *file, unsigned int cmd,
2967                             unsigned long arg)
2968 {
2969         return fuse_ioctl_common(file, cmd, arg, 0);
2970 }
2971
2972 static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
2973                                    unsigned long arg)
2974 {
2975         return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
2976 }
2977
2978 /*
2979  * All files which have been polled are linked to RB tree
2980  * fuse_conn->polled_files which is indexed by kh.  Walk the tree and
2981  * find the matching one.
2982  */
2983 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
2984                                               struct rb_node **parent_out)
2985 {
2986         struct rb_node **link = &fc->polled_files.rb_node;
2987         struct rb_node *last = NULL;
2988
2989         while (*link) {
2990                 struct fuse_file *ff;
2991
2992                 last = *link;
2993                 ff = rb_entry(last, struct fuse_file, polled_node);
2994
2995                 if (kh < ff->kh)
2996                         link = &last->rb_left;
2997                 else if (kh > ff->kh)
2998                         link = &last->rb_right;
2999                 else
3000                         return link;
3001         }
3002
3003         if (parent_out)
3004                 *parent_out = last;
3005         return link;
3006 }
3007
3008 /*
3009  * The file is about to be polled.  Make sure it's on the polled_files
3010  * RB tree.  Note that files once added to the polled_files tree are
3011  * not removed before the file is released.  This is because a file
3012  * polled once is likely to be polled again.
3013  */
3014 static void fuse_register_polled_file(struct fuse_conn *fc,
3015                                       struct fuse_file *ff)
3016 {
3017         spin_lock(&fc->lock);
3018         if (RB_EMPTY_NODE(&ff->polled_node)) {
3019                 struct rb_node **link, *parent;
3020
3021                 link = fuse_find_polled_node(fc, ff->kh, &parent);
3022                 BUG_ON(*link);
3023                 rb_link_node(&ff->polled_node, parent, link);
3024                 rb_insert_color(&ff->polled_node, &fc->polled_files);
3025         }
3026         spin_unlock(&fc->lock);
3027 }
3028
3029 __poll_t fuse_file_poll(struct file *file, poll_table *wait)
3030 {
3031         struct fuse_file *ff = file->private_data;
3032         struct fuse_mount *fm = ff->fm;
3033         struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
3034         struct fuse_poll_out outarg;
3035         FUSE_ARGS(args);
3036         int err;
3037
3038         if (fm->fc->no_poll)
3039                 return DEFAULT_POLLMASK;
3040
3041         poll_wait(file, &ff->poll_wait, wait);
3042         inarg.events = mangle_poll(poll_requested_events(wait));
3043
3044         /*
3045          * Ask for notification iff there's someone waiting for it.
3046          * The client may ignore the flag and always notify.
3047          */
3048         if (waitqueue_active(&ff->poll_wait)) {
3049                 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
3050                 fuse_register_polled_file(fm->fc, ff);
3051         }
3052
3053         args.opcode = FUSE_POLL;
3054         args.nodeid = ff->nodeid;
3055         args.in_numargs = 1;
3056         args.in_args[0].size = sizeof(inarg);
3057         args.in_args[0].value = &inarg;
3058         args.out_numargs = 1;
3059         args.out_args[0].size = sizeof(outarg);
3060         args.out_args[0].value = &outarg;
3061         err = fuse_simple_request(fm, &args);
3062
3063         if (!err)
3064                 return demangle_poll(outarg.revents);
3065         if (err == -ENOSYS) {
3066                 fm->fc->no_poll = 1;
3067                 return DEFAULT_POLLMASK;
3068         }
3069         return EPOLLERR;
3070 }
3071 EXPORT_SYMBOL_GPL(fuse_file_poll);
3072
3073 /*
3074  * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
3075  * wakes up the poll waiters.
3076  */
3077 int fuse_notify_poll_wakeup(struct fuse_conn *fc,
3078                             struct fuse_notify_poll_wakeup_out *outarg)
3079 {
3080         u64 kh = outarg->kh;
3081         struct rb_node **link;
3082
3083         spin_lock(&fc->lock);
3084
3085         link = fuse_find_polled_node(fc, kh, NULL);
3086         if (*link) {
3087                 struct fuse_file *ff;
3088
3089                 ff = rb_entry(*link, struct fuse_file, polled_node);
3090                 wake_up_interruptible_sync(&ff->poll_wait);
3091         }
3092
3093         spin_unlock(&fc->lock);
3094         return 0;
3095 }
3096
3097 static void fuse_do_truncate(struct file *file)
3098 {
3099         struct inode *inode = file->f_mapping->host;
3100         struct iattr attr;
3101
3102         attr.ia_valid = ATTR_SIZE;
3103         attr.ia_size = i_size_read(inode);
3104
3105         attr.ia_file = file;
3106         attr.ia_valid |= ATTR_FILE;
3107
3108         fuse_do_setattr(file_dentry(file), &attr, file);
3109 }
3110
3111 static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
3112 {
3113         return round_up(off, fc->max_pages << PAGE_SHIFT);
3114 }
3115
3116 static ssize_t
3117 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
3118 {
3119         DECLARE_COMPLETION_ONSTACK(wait);
3120         ssize_t ret = 0;
3121         struct file *file = iocb->ki_filp;
3122         struct fuse_file *ff = file->private_data;
3123         loff_t pos = 0;
3124         struct inode *inode;
3125         loff_t i_size;
3126         size_t count = iov_iter_count(iter), shortened = 0;
3127         loff_t offset = iocb->ki_pos;
3128         struct fuse_io_priv *io;
3129
3130         pos = offset;
3131         inode = file->f_mapping->host;
3132         i_size = i_size_read(inode);
3133
3134         if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
3135                 return 0;
3136
3137         io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
3138         if (!io)
3139                 return -ENOMEM;
3140         spin_lock_init(&io->lock);
3141         kref_init(&io->refcnt);
3142         io->reqs = 1;
3143         io->bytes = -1;
3144         io->size = 0;
3145         io->offset = offset;
3146         io->write = (iov_iter_rw(iter) == WRITE);
3147         io->err = 0;
3148         /*
3149          * By default, we want to optimize all I/Os with async request
3150          * submission to the client filesystem if supported.
3151          */
3152         io->async = ff->fm->fc->async_dio;
3153         io->iocb = iocb;
3154         io->blocking = is_sync_kiocb(iocb);
3155
3156         /* optimization for short read */
3157         if (io->async && !io->write && offset + count > i_size) {
3158                 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
3159                 shortened = count - iov_iter_count(iter);
3160                 count -= shortened;
3161         }
3162
3163         /*
3164          * We cannot asynchronously extend the size of a file.
3165          * In such case the aio will behave exactly like sync io.
3166          */
3167         if ((offset + count > i_size) && io->write)
3168                 io->blocking = true;
3169
3170         if (io->async && io->blocking) {
3171                 /*
3172                  * Additional reference to keep io around after
3173                  * calling fuse_aio_complete()
3174                  */
3175                 kref_get(&io->refcnt);
3176                 io->done = &wait;
3177         }
3178
3179         if (iov_iter_rw(iter) == WRITE) {
3180                 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
3181                 fuse_invalidate_attr(inode);
3182         } else {
3183                 ret = __fuse_direct_read(io, iter, &pos);
3184         }
3185         iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
3186
3187         if (io->async) {
3188                 bool blocking = io->blocking;
3189
3190                 fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
3191
3192                 /* we have a non-extending, async request, so return */
3193                 if (!blocking)
3194                         return -EIOCBQUEUED;
3195
3196                 wait_for_completion(&wait);
3197                 ret = fuse_get_res_by_io(io);
3198         }
3199
3200         kref_put(&io->refcnt, fuse_io_release);
3201
3202         if (iov_iter_rw(iter) == WRITE) {
3203                 if (ret > 0)
3204                         fuse_write_update_size(inode, pos);
3205                 else if (ret < 0 && offset + count > i_size)
3206                         fuse_do_truncate(file);
3207         }
3208
3209         return ret;
3210 }
3211
3212 static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
3213 {
3214         int err = filemap_write_and_wait_range(inode->i_mapping, start, end);
3215
3216         if (!err)
3217                 fuse_sync_writes(inode);
3218
3219         return err;
3220 }
3221
3222 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
3223                                 loff_t length)
3224 {
3225         struct fuse_file *ff = file->private_data;
3226         struct inode *inode = file_inode(file);
3227         struct fuse_inode *fi = get_fuse_inode(inode);
3228         struct fuse_mount *fm = ff->fm;
3229         FUSE_ARGS(args);
3230         struct fuse_fallocate_in inarg = {
3231                 .fh = ff->fh,
3232                 .offset = offset,
3233                 .length = length,
3234                 .mode = mode
3235         };
3236         int err;
3237         bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
3238                            (mode & FALLOC_FL_PUNCH_HOLE);
3239
3240         bool block_faults = FUSE_IS_DAX(inode) && lock_inode;
3241
3242         if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
3243                 return -EOPNOTSUPP;
3244
3245         if (fm->fc->no_fallocate)
3246                 return -EOPNOTSUPP;
3247
3248         if (lock_inode) {
3249                 inode_lock(inode);
3250                 if (block_faults) {
3251                         down_write(&fi->i_mmap_sem);
3252                         err = fuse_dax_break_layouts(inode, 0, 0);
3253                         if (err)
3254                                 goto out;
3255                 }
3256
3257                 if (mode & FALLOC_FL_PUNCH_HOLE) {
3258                         loff_t endbyte = offset + length - 1;
3259
3260                         err = fuse_writeback_range(inode, offset, endbyte);
3261                         if (err)
3262                                 goto out;
3263                 }
3264         }
3265
3266         if (!(mode & FALLOC_FL_KEEP_SIZE) &&
3267             offset + length > i_size_read(inode)) {
3268                 err = inode_newsize_ok(inode, offset + length);
3269                 if (err)
3270                         goto out;
3271         }
3272
3273         if (!(mode & FALLOC_FL_KEEP_SIZE))
3274                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3275
3276         args.opcode = FUSE_FALLOCATE;
3277         args.nodeid = ff->nodeid;
3278         args.in_numargs = 1;
3279         args.in_args[0].size = sizeof(inarg);
3280         args.in_args[0].value = &inarg;
3281         err = fuse_simple_request(fm, &args);
3282         if (err == -ENOSYS) {
3283                 fm->fc->no_fallocate = 1;
3284                 err = -EOPNOTSUPP;
3285         }
3286         if (err)
3287                 goto out;
3288
3289         /* we could have extended the file */
3290         if (!(mode & FALLOC_FL_KEEP_SIZE)) {
3291                 bool changed = fuse_write_update_size(inode, offset + length);
3292
3293                 if (changed && fm->fc->writeback_cache)
3294                         file_update_time(file);
3295         }
3296
3297         if (mode & FALLOC_FL_PUNCH_HOLE)
3298                 truncate_pagecache_range(inode, offset, offset + length - 1);
3299
3300         fuse_invalidate_attr(inode);
3301
3302 out:
3303         if (!(mode & FALLOC_FL_KEEP_SIZE))
3304                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
3305
3306         if (block_faults)
3307                 up_write(&fi->i_mmap_sem);
3308
3309         if (lock_inode)
3310                 inode_unlock(inode);
3311
3312         return err;
3313 }
3314
3315 static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
3316                                       struct file *file_out, loff_t pos_out,
3317                                       size_t len, unsigned int flags)
3318 {
3319         struct fuse_file *ff_in = file_in->private_data;
3320         struct fuse_file *ff_out = file_out->private_data;
3321         struct inode *inode_in = file_inode(file_in);
3322         struct inode *inode_out = file_inode(file_out);
3323         struct fuse_inode *fi_out = get_fuse_inode(inode_out);
3324         struct fuse_mount *fm = ff_in->fm;
3325         struct fuse_conn *fc = fm->fc;
3326         FUSE_ARGS(args);
3327         struct fuse_copy_file_range_in inarg = {
3328                 .fh_in = ff_in->fh,
3329                 .off_in = pos_in,
3330                 .nodeid_out = ff_out->nodeid,
3331                 .fh_out = ff_out->fh,
3332                 .off_out = pos_out,
3333                 .len = len,
3334                 .flags = flags
3335         };
3336         struct fuse_write_out outarg;
3337         ssize_t err;
3338         /* mark unstable when write-back is not used, and file_out gets
3339          * extended */
3340         bool is_unstable = (!fc->writeback_cache) &&
3341                            ((pos_out + len) > inode_out->i_size);
3342
3343         if (fc->no_copy_file_range)
3344                 return -EOPNOTSUPP;
3345
3346         if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
3347                 return -EXDEV;
3348
3349         inode_lock(inode_in);
3350         err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
3351         inode_unlock(inode_in);
3352         if (err)
3353                 return err;
3354
3355         inode_lock(inode_out);
3356
3357         err = file_modified(file_out);
3358         if (err)
3359                 goto out;
3360
3361         /*
3362          * Write out dirty pages in the destination file before sending the COPY
3363          * request to userspace.  After the request is completed, truncate off
3364          * pages (including partial ones) from the cache that have been copied,
3365          * since these contain stale data at that point.
3366          *
3367          * This should be mostly correct, but if the COPY writes to partial
3368          * pages (at the start or end) and the parts not covered by the COPY are
3369          * written through a memory map after calling fuse_writeback_range(),
3370          * then these partial page modifications will be lost on truncation.
3371          *
3372          * It is unlikely that someone would rely on such mixed style
3373          * modifications.  Yet this does give less guarantees than if the
3374          * copying was performed with write(2).
3375          *
3376          * To fix this a i_mmap_sem style lock could be used to prevent new
3377          * faults while the copy is ongoing.
3378          */
3379         err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
3380         if (err)
3381                 goto out;
3382
3383         if (is_unstable)
3384                 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3385
3386         args.opcode = FUSE_COPY_FILE_RANGE;
3387         args.nodeid = ff_in->nodeid;
3388         args.in_numargs = 1;
3389         args.in_args[0].size = sizeof(inarg);
3390         args.in_args[0].value = &inarg;
3391         args.out_numargs = 1;
3392         args.out_args[0].size = sizeof(outarg);
3393         args.out_args[0].value = &outarg;
3394         err = fuse_simple_request(fm, &args);
3395         if (err == -ENOSYS) {
3396                 fc->no_copy_file_range = 1;
3397                 err = -EOPNOTSUPP;
3398         }
3399         if (err)
3400                 goto out;
3401
3402         truncate_inode_pages_range(inode_out->i_mapping,
3403                                    ALIGN_DOWN(pos_out, PAGE_SIZE),
3404                                    ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
3405
3406         if (fc->writeback_cache) {
3407                 fuse_write_update_size(inode_out, pos_out + outarg.size);
3408                 file_update_time(file_out);
3409         }
3410
3411         fuse_invalidate_attr(inode_out);
3412
3413         err = outarg.size;
3414 out:
3415         if (is_unstable)
3416                 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
3417
3418         inode_unlock(inode_out);
3419         file_accessed(file_in);
3420
3421         return err;
3422 }
3423
3424 static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
3425                                     struct file *dst_file, loff_t dst_off,
3426                                     size_t len, unsigned int flags)
3427 {
3428         ssize_t ret;
3429
3430         ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
3431                                      len, flags);
3432
3433         if (ret == -EOPNOTSUPP || ret == -EXDEV)
3434                 ret = generic_copy_file_range(src_file, src_off, dst_file,
3435                                               dst_off, len, flags);
3436         return ret;
3437 }
3438
3439 static const struct file_operations fuse_file_operations = {
3440         .llseek         = fuse_file_llseek,
3441         .read_iter      = fuse_file_read_iter,
3442         .write_iter     = fuse_file_write_iter,
3443         .mmap           = fuse_file_mmap,
3444         .open           = fuse_open,
3445         .flush          = fuse_flush,
3446         .release        = fuse_release,
3447         .fsync          = fuse_fsync,
3448         .lock           = fuse_file_lock,
3449         .get_unmapped_area = thp_get_unmapped_area,
3450         .flock          = fuse_file_flock,
3451         .splice_read    = generic_file_splice_read,
3452         .splice_write   = iter_file_splice_write,
3453         .unlocked_ioctl = fuse_file_ioctl,
3454         .compat_ioctl   = fuse_file_compat_ioctl,
3455         .poll           = fuse_file_poll,
3456         .fallocate      = fuse_file_fallocate,
3457         .copy_file_range = fuse_copy_file_range,
3458 };
3459
3460 static const struct address_space_operations fuse_file_aops  = {
3461         .readpage       = fuse_readpage,
3462         .readahead      = fuse_readahead,
3463         .writepage      = fuse_writepage,
3464         .writepages     = fuse_writepages,
3465         .launder_page   = fuse_launder_page,
3466         .set_page_dirty = __set_page_dirty_nobuffers,
3467         .bmap           = fuse_bmap,
3468         .direct_IO      = fuse_direct_IO,
3469         .write_begin    = fuse_write_begin,
3470         .write_end      = fuse_write_end,
3471 };
3472
3473 void fuse_init_file_inode(struct inode *inode)
3474 {
3475         struct fuse_inode *fi = get_fuse_inode(inode);
3476
3477         inode->i_fop = &fuse_file_operations;
3478         inode->i_data.a_ops = &fuse_file_aops;
3479
3480         INIT_LIST_HEAD(&fi->write_files);
3481         INIT_LIST_HEAD(&fi->queued_writes);
3482         fi->writectr = 0;
3483         init_waitqueue_head(&fi->page_waitq);
3484         fi->writepages = RB_ROOT;
3485
3486         if (IS_ENABLED(CONFIG_FUSE_DAX))
3487                 fuse_dax_inode_init(inode);
3488 }