*/
#include "internal.h"
#include <linux/prefetch.h>
-
+#include <linux/iomap.h>
+#include <linux/dax.h>
#include <trace/events/erofs.h>
static void erofs_readendio(struct bio *bio)
return 0;
}
+static int erofs_iomap_begin(struct inode *inode, loff_t offset, loff_t length,
+ unsigned int flags, struct iomap *iomap, struct iomap *srcmap)
+{
+ int ret;
+ struct erofs_map_blocks map;
+
+ map.m_la = offset;
+ map.m_llen = length;
+
+ ret = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
+ if (ret < 0)
+ return ret;
+
+ iomap->bdev = inode->i_sb->s_bdev;
+ iomap->dax_dev = EROFS_I_SB(inode)->dax_dev;
+ iomap->offset = map.m_la;
+ iomap->length = map.m_llen;
+ iomap->flags = 0;
+
+ if (!(map.m_flags & EROFS_MAP_MAPPED)) {
+ iomap->type = IOMAP_HOLE;
+ iomap->addr = IOMAP_NULL_ADDR;
+ if (!iomap->length)
+ iomap->length = length;
+ return 0;
+ }
+
+ /* that shouldn't happen for now */
+ if (map.m_flags & EROFS_MAP_META) {
+ DBG_BUGON(1);
+ return -ENOTBLK;
+ }
+ iomap->type = IOMAP_MAPPED;
+ iomap->addr = map.m_pa;
+ return 0;
+}
+
+static const struct iomap_ops erofs_iomap_ops = {
+ .iomap_begin = erofs_iomap_begin,
+};
+
+static int erofs_prepare_dio(struct kiocb *iocb, struct iov_iter *to)
+{
+ struct inode *inode = file_inode(iocb->ki_filp);
+ loff_t align = iocb->ki_pos | iov_iter_count(to) |
+ iov_iter_alignment(to);
+ struct block_device *bdev = inode->i_sb->s_bdev;
+ unsigned int blksize_mask;
+
+ if (bdev)
+ blksize_mask = (1 << ilog2(bdev_logical_block_size(bdev))) - 1;
+ else
+ blksize_mask = (1 << inode->i_blkbits) - 1;
+
+ if (align & blksize_mask)
+ return -EINVAL;
+
+ /*
+ * Temporarily fall back tail-packing inline to buffered I/O instead
+ * since tail-packing inline support relies on an iomap core update.
+ */
+ if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE &&
+ iocb->ki_pos + iov_iter_count(to) >
+ rounddown(inode->i_size, EROFS_BLKSIZ))
+ return 1;
+ return 0;
+}
+
+static ssize_t erofs_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
+{
+ /* no need taking (shared) inode lock since it's a ro filesystem */
+ if (!iov_iter_count(to))
+ return 0;
+
+#ifdef CONFIG_FS_DAX
+ if (IS_DAX(iocb->ki_filp->f_mapping->host))
+ return dax_iomap_rw(iocb, to, &erofs_iomap_ops);
+#endif
+ if (iocb->ki_flags & IOCB_DIRECT) {
+ int err = erofs_prepare_dio(iocb, to);
+
+ if (!err)
+ return iomap_dio_rw(iocb, to, &erofs_iomap_ops,
+ NULL, 0);
+ if (err < 0)
+ return err;
+ }
+ return filemap_read(iocb, to, 0);
+}
+
/* for uncompressed (aligned) files and raw access for other files */
const struct address_space_operations erofs_raw_access_aops = {
.readpage = erofs_raw_access_readpage,
.readahead = erofs_raw_access_readahead,
.bmap = erofs_bmap,
+ .direct_IO = noop_direct_IO,
+};
+
+#ifdef CONFIG_FS_DAX
+static vm_fault_t erofs_dax_huge_fault(struct vm_fault *vmf,
+ enum page_entry_size pe_size)
+{
+ return dax_iomap_fault(vmf, pe_size, NULL, NULL, &erofs_iomap_ops);
+}
+
+static vm_fault_t erofs_dax_fault(struct vm_fault *vmf)
+{
+ return erofs_dax_huge_fault(vmf, PE_SIZE_PTE);
+}
+
+static const struct vm_operations_struct erofs_dax_vm_ops = {
+ .fault = erofs_dax_fault,
+ .huge_fault = erofs_dax_huge_fault,
+};
+
+static int erofs_file_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ if (!IS_DAX(file_inode(file)))
+ return generic_file_readonly_mmap(file, vma);
+
+ if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
+ return -EINVAL;
+
+ vma->vm_ops = &erofs_dax_vm_ops;
+ vma->vm_flags |= VM_HUGEPAGE;
+ return 0;
+}
+#else
+#define erofs_file_mmap generic_file_readonly_mmap
+#endif
+
+const struct file_operations erofs_file_fops = {
+ .llseek = generic_file_llseek,
+ .read_iter = erofs_file_read_iter,
+ .mmap = erofs_file_mmap,
+ .splice_read = generic_file_splice_read,
};