struct file *file = iocb->ki_filp;
struct fuse_file *ff = file->private_data;
struct inode *inode = file_inode(iocb->ki_filp);
+ struct fuse_inode *fi = get_fuse_inode(inode);
/* Server side has to advise that it supports parallel dio writes. */
if (!(ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES))
if (iocb->ki_flags & IOCB_APPEND)
return true;
- /*
- * Combination of page access and direct-io is difficult, shared locks
- * actually introduce a conflict.
- */
- if (get_fuse_conn(inode)->direct_io_allow_mmap)
- return true;
+ /* shared locks are not allowed with parallel page cache IO */
+ if (test_bit(FUSE_I_CACHE_IO_MODE, &fi->state))
+ return false;
/* Parallel dio beyond EOF is not supported, at least for now. */
if (fuse_io_past_eof(iocb, from))
bool *exclusive)
{
struct inode *inode = file_inode(iocb->ki_filp);
+ struct fuse_file *ff = iocb->ki_filp->private_data;
*exclusive = fuse_dio_wr_exclusive_lock(iocb, from);
if (*exclusive) {
} else {
inode_lock_shared(inode);
/*
- * Previous check was without inode lock and might have raced,
- * check again.
+ * New parallal dio allowed only if inode is not in caching
+ * mode and denies new opens in caching mode. This check
+ * should be performed only after taking shared inode lock.
+ * Previous past eof check was without inode lock and might
+ * have raced, so check it again.
*/
- if (fuse_io_past_eof(iocb, from)) {
+ if (fuse_io_past_eof(iocb, from) ||
+ fuse_file_uncached_io_start(inode, ff) != 0) {
inode_unlock_shared(inode);
inode_lock(inode);
*exclusive = true;
}
}
-static void fuse_dio_unlock(struct inode *inode, bool exclusive)
+static void fuse_dio_unlock(struct kiocb *iocb, bool exclusive)
{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ struct fuse_file *ff = iocb->ki_filp->private_data;
+
if (exclusive) {
inode_unlock(inode);
} else {
+ /* Allow opens in caching mode after last parallel dio end */
+ fuse_file_uncached_io_end(inode, ff);
inode_unlock_shared(inode);
}
}
fuse_write_update_attr(inode, iocb->ki_pos, res);
}
}
- fuse_dio_unlock(inode, exclusive);
+ fuse_dio_unlock(iocb, exclusive);
return res;
}
if (FUSE_IS_DAX(file_inode(file)))
return fuse_dax_mmap(file, vma);
+ /*
+ * FOPEN_DIRECT_IO handling is special compared to O_DIRECT,
+ * as does not allow MAP_SHARED mmap without FUSE_DIRECT_IO_ALLOW_MMAP.
+ */
if (ff->open_flags & FOPEN_DIRECT_IO) {
/*
* Can't provide the coherency needed for MAP_SHARED
return generic_file_mmap(file, vma);
}
- /* First mmap of direct_io file enters caching inode io mode. */
+ /*
+ * First mmap of direct_io file enters caching inode io mode.
+ * Also waits for parallel dio writers to go into serial mode
+ * (exclusive instead of shared lock).
+ */
rc = fuse_file_cached_io_start(file_inode(file), ff);
if (rc)
return rc;
fi->writectr = 0;
fi->iocachectr = 0;
init_waitqueue_head(&fi->page_waitq);
+ init_waitqueue_head(&fi->direct_io_waitq);
fi->writepages = RB_ROOT;
if (IS_ENABLED(CONFIG_FUSE_DAX))
/* Waitq for writepage completion */
wait_queue_head_t page_waitq;
+ /* waitq for direct-io completion */
+ wait_queue_head_t direct_io_waitq;
+
/* List of writepage requestst (pending or sent) */
struct rb_root writepages;
};
/* iomode.c */
int fuse_file_cached_io_start(struct inode *inode, struct fuse_file *ff);
+int fuse_file_uncached_io_start(struct inode *inode, struct fuse_file *ff);
+void fuse_file_uncached_io_end(struct inode *inode, struct fuse_file *ff);
int fuse_file_io_open(struct file *file, struct inode *inode);
void fuse_file_io_release(struct fuse_file *ff, struct inode *inode);
#include <linux/fs.h>
/*
- * Start cached io mode, where parallel dio writes are not allowed.
+ * Return true if need to wait for new opens in caching mode.
+ */
+static inline bool fuse_is_io_cache_wait(struct fuse_inode *fi)
+{
+ return READ_ONCE(fi->iocachectr) < 0;
+}
+
+/*
+ * Start cached io mode.
+ *
+ * Blocks new parallel dio writes and waits for the in-progress parallel dio
+ * writes to complete.
*/
int fuse_file_cached_io_start(struct inode *inode, struct fuse_file *ff)
{
struct fuse_inode *fi = get_fuse_inode(inode);
- int err = 0;
/* There are no io modes if server does not implement open */
if (!ff->release_args)
return 0;
spin_lock(&fi->lock);
- if (fi->iocachectr < 0) {
- err = -ETXTBSY;
- goto unlock;
+ /*
+ * Setting the bit advises new direct-io writes to use an exclusive
+ * lock - without it the wait below might be forever.
+ */
+ while (fuse_is_io_cache_wait(fi)) {
+ set_bit(FUSE_I_CACHE_IO_MODE, &fi->state);
+ spin_unlock(&fi->lock);
+ wait_event(fi->direct_io_waitq, !fuse_is_io_cache_wait(fi));
+ spin_lock(&fi->lock);
}
WARN_ON(ff->iomode == IOM_UNCACHED);
if (ff->iomode == IOM_NONE) {
set_bit(FUSE_I_CACHE_IO_MODE, &fi->state);
fi->iocachectr++;
}
-unlock:
spin_unlock(&fi->lock);
- return err;
+ return 0;
}
static void fuse_file_cached_io_end(struct inode *inode, struct fuse_file *ff)
}
/* Start strictly uncached io mode where cache access is not allowed */
-static int fuse_file_uncached_io_start(struct inode *inode, struct fuse_file *ff)
+int fuse_file_uncached_io_start(struct inode *inode, struct fuse_file *ff)
{
struct fuse_inode *fi = get_fuse_inode(inode);
int err = 0;
return err;
}
-static void fuse_file_uncached_io_end(struct inode *inode, struct fuse_file *ff)
+void fuse_file_uncached_io_end(struct inode *inode, struct fuse_file *ff)
{
struct fuse_inode *fi = get_fuse_inode(inode);
WARN_ON(ff->iomode != IOM_UNCACHED);
ff->iomode = IOM_NONE;
fi->iocachectr++;
+ if (!fi->iocachectr)
+ wake_up(&fi->direct_io_waitq);
spin_unlock(&fi->lock);
}
ff->open_flags &= ~FOPEN_PARALLEL_DIRECT_WRITES;
/*
- * First parallel dio open denies caching inode io mode.
* First caching file open enters caching inode io mode.
*
* Note that if user opens a file open with O_DIRECT, but server did
* not specify FOPEN_DIRECT_IO, a later fcntl() could remove O_DIRECT,
* so we put the inode in caching mode to prevent parallel dio.
*/
- if (ff->open_flags & FOPEN_DIRECT_IO) {
- if (ff->open_flags & FOPEN_PARALLEL_DIRECT_WRITES)
- err = fuse_file_uncached_io_start(inode, ff);
- else
- return 0;
- } else {
- err = fuse_file_cached_io_start(inode, ff);
- }
+ if (ff->open_flags & FOPEN_DIRECT_IO)
+ return 0;
+
+ err = fuse_file_cached_io_start(inode, ff);
if (err)
goto fail;