fs/erofs/data.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*
   3  * Copyright (C) 2017-2018 HUAWEI, Inc.
   4  *             https://www.huawei.com/
   5  */
   6 #include "internal.h"
   7 #include <linux/prefetch.h>
   8
   9 #include <trace/events/erofs.h>
  10
  11 static void erofs_readendio(struct bio *bio)
  12 {
  13         struct bio_vec *bvec;
  14         blk_status_t err = bio->bi_status;
  15         struct bvec_iter_all iter_all;
  16
  17         bio_for_each_segment_all(bvec, bio, iter_all) {
  18                 struct page *page = bvec->bv_page;
  19
  20                 /* page is already locked */
  21                 DBG_BUGON(PageUptodate(page));
  22
  23                 if (err)
  24                         SetPageError(page);
  25                 else
  26                         SetPageUptodate(page);
  27
  28                 unlock_page(page);
  29                 /* page could be reclaimed now */
  30         }
  31         bio_put(bio);
  32 }
  33
  34 struct page *erofs_get_meta_page(struct super_block *sb, erofs_blk_t blkaddr)
  35 {
  36         struct address_space *const mapping = sb->s_bdev->bd_inode->i_mapping;
  37         struct page *page;
  38
  39         page = read_cache_page_gfp(mapping, blkaddr,
  40                                    mapping_gfp_constraint(mapping, ~__GFP_FS));
  41         /* should already be PageUptodate */
  42         if (!IS_ERR(page))
  43                 lock_page(page);
  44         return page;
  45 }
  46
  47 static int erofs_map_blocks_flatmode(struct inode *inode,
  48                                      struct erofs_map_blocks *map,
  49                                      int flags)
  50 {
  51         int err = 0;
  52         erofs_blk_t nblocks, lastblk;
  53         u64 offset = map->m_la;
  54         struct erofs_inode *vi = EROFS_I(inode);
  55         bool tailendpacking = (vi->datalayout == EROFS_INODE_FLAT_INLINE);
  56
  57         trace_erofs_map_blocks_flatmode_enter(inode, map, flags);
  58
  59         nblocks = DIV_ROUND_UP(inode->i_size, PAGE_SIZE);
  60         lastblk = nblocks - tailendpacking;
  61
  62         if (offset >= inode->i_size) {
  63                 /* leave out-of-bound access unmapped */
  64                 map->m_flags = 0;
  65                 map->m_plen = 0;
  66                 goto out;
  67         }
  68
  69         /* there is no hole in flatmode */
  70         map->m_flags = EROFS_MAP_MAPPED;
  71
  72         if (offset < blknr_to_addr(lastblk)) {
  73                 map->m_pa = blknr_to_addr(vi->raw_blkaddr) + map->m_la;
  74                 map->m_plen = blknr_to_addr(lastblk) - offset;
  75         } else if (tailendpacking) {
  76                 /* 2 - inode inline B: inode, [xattrs], inline last blk... */
  77                 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
  78
  79                 map->m_pa = iloc(sbi, vi->nid) + vi->inode_isize +
  80                         vi->xattr_isize + erofs_blkoff(map->m_la);
  81                 map->m_plen = inode->i_size - offset;
  82
  83                 /* inline data should be located in one meta block */
  84                 if (erofs_blkoff(map->m_pa) + map->m_plen > PAGE_SIZE) {
  85                         erofs_err(inode->i_sb,
  86                                   "inline data cross block boundary @ nid %llu",
  87                                   vi->nid);
  88                         DBG_BUGON(1);
  89                         err = -EFSCORRUPTED;
  90                         goto err_out;
  91                 }
  92
  93                 map->m_flags |= EROFS_MAP_META;
  94         } else {
  95                 erofs_err(inode->i_sb,
  96                           "internal error @ nid: %llu (size %llu), m_la 0x%llx",
  97                           vi->nid, inode->i_size, map->m_la);
  98                 DBG_BUGON(1);
  99                 err = -EIO;
 100                 goto err_out;
 101         }
 102
 103 out:
 104         map->m_llen = map->m_plen;
 105
 106 err_out:
 107         trace_erofs_map_blocks_flatmode_exit(inode, map, flags, 0);
 108         return err;
 109 }
 110
 111 static inline struct bio *erofs_read_raw_page(struct bio *bio,
 112                                               struct address_space *mapping,
 113                                               struct page *page,
 114                                               erofs_off_t *last_block,
 115                                               unsigned int nblocks,
 116                                               unsigned int *eblks,
 117                                               bool ra)
 118 {
 119         struct inode *const inode = mapping->host;
 120         struct super_block *const sb = inode->i_sb;
 121         erofs_off_t current_block = (erofs_off_t)page->index;
 122         int err;
 123
 124         DBG_BUGON(!nblocks);
 125
 126         if (PageUptodate(page)) {
 127                 err = 0;
 128                 goto has_updated;
 129         }
 130
 131         /* note that for readpage case, bio also equals to NULL */
 132         if (bio &&
 133             (*last_block + 1 != current_block || !*eblks)) {
 134 submit_bio_retry:
 135                 submit_bio(bio);
 136                 bio = NULL;
 137         }
 138
 139         if (!bio) {
 140                 struct erofs_map_blocks map = {
 141                         .m_la = blknr_to_addr(current_block),
 142                 };
 143                 erofs_blk_t blknr;
 144                 unsigned int blkoff;
 145
 146                 err = erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW);
 147                 if (err)
 148                         goto err_out;
 149
 150                 /* zero out the holed page */
 151                 if (!(map.m_flags & EROFS_MAP_MAPPED)) {
 152                         zero_user_segment(page, 0, PAGE_SIZE);
 153                         SetPageUptodate(page);
 154
 155                         /* imply err = 0, see erofs_map_blocks */
 156                         goto has_updated;
 157                 }
 158
 159                 /* for RAW access mode, m_plen must be equal to m_llen */
 160                 DBG_BUGON(map.m_plen != map.m_llen);
 161
 162                 blknr = erofs_blknr(map.m_pa);
 163                 blkoff = erofs_blkoff(map.m_pa);
 164
 165                 /* deal with inline page */
 166                 if (map.m_flags & EROFS_MAP_META) {
 167                         void *vsrc, *vto;
 168                         struct page *ipage;
 169
 170                         DBG_BUGON(map.m_plen > PAGE_SIZE);
 171
 172                         ipage = erofs_get_meta_page(inode->i_sb, blknr);
 173
 174                         if (IS_ERR(ipage)) {
 175                                 err = PTR_ERR(ipage);
 176                                 goto err_out;
 177                         }
 178
 179                         vsrc = kmap_atomic(ipage);
 180                         vto = kmap_atomic(page);
 181                         memcpy(vto, vsrc + blkoff, map.m_plen);
 182                         memset(vto + map.m_plen, 0, PAGE_SIZE - map.m_plen);
 183                         kunmap_atomic(vto);
 184                         kunmap_atomic(vsrc);
 185                         flush_dcache_page(page);
 186
 187                         SetPageUptodate(page);
 188                         /* TODO: could we unlock the page earlier? */
 189                         unlock_page(ipage);
 190                         put_page(ipage);
 191
 192                         /* imply err = 0, see erofs_map_blocks */
 193                         goto has_updated;
 194                 }
 195
 196                 /* pa must be block-aligned for raw reading */
 197                 DBG_BUGON(erofs_blkoff(map.m_pa));
 198
 199                 /* max # of continuous pages */
 200                 if (nblocks > DIV_ROUND_UP(map.m_plen, PAGE_SIZE))
 201                         nblocks = DIV_ROUND_UP(map.m_plen, PAGE_SIZE);
 202
 203                 *eblks = bio_max_segs(nblocks);
 204                 bio = bio_alloc(GFP_NOIO, *eblks);
 205
 206                 bio->bi_end_io = erofs_readendio;
 207                 bio_set_dev(bio, sb->s_bdev);
 208                 bio->bi_iter.bi_sector = (sector_t)blknr <<
 209                         LOG_SECTORS_PER_BLOCK;
 210                 bio->bi_opf = REQ_OP_READ | (ra ? REQ_RAHEAD : 0);
 211         }
 212
 213         err = bio_add_page(bio, page, PAGE_SIZE, 0);
 214         /* out of the extent or bio is full */
 215         if (err < PAGE_SIZE)
 216                 goto submit_bio_retry;
 217         --*eblks;
 218         *last_block = current_block;
 219         return bio;
 220
 221 err_out:
 222         /* for sync reading, set page error immediately */
 223         if (!ra) {
 224                 SetPageError(page);
 225                 ClearPageUptodate(page);
 226         }
 227 has_updated:
 228         unlock_page(page);
 229
 230         /* if updated manually, continuous pages has a gap */
 231         if (bio)
 232                 submit_bio(bio);
 233         return err ? ERR_PTR(err) : NULL;
 234 }
 235
 236 /*
 237  * since we dont have write or truncate flows, so no inode
 238  * locking needs to be held at the moment.
 239  */
 240 static int erofs_raw_access_readpage(struct file *file, struct page *page)
 241 {
 242         erofs_off_t last_block;
 243         unsigned int eblks;
 244         struct bio *bio;
 245
 246         trace_erofs_readpage(page, true);
 247
 248         bio = erofs_read_raw_page(NULL, page->mapping,
 249                                   page, &last_block, 1, &eblks, false);
 250
 251         if (IS_ERR(bio))
 252                 return PTR_ERR(bio);
 253
 254         if (bio)
 255                 submit_bio(bio);
 256         return 0;
 257 }
 258
 259 static void erofs_raw_access_readahead(struct readahead_control *rac)
 260 {
 261         erofs_off_t last_block;
 262         unsigned int eblks;
 263         struct bio *bio = NULL;
 264         struct page *page;
 265
 266         trace_erofs_readpages(rac->mapping->host, readahead_index(rac),
 267                         readahead_count(rac), true);
 268
 269         while ((page = readahead_page(rac))) {
 270                 prefetchw(&page->flags);
 271
 272                 bio = erofs_read_raw_page(bio, rac->mapping, page, &last_block,
 273                                 readahead_count(rac), &eblks, true);
 274
 275                 /* all the page errors are ignored when readahead */
 276                 if (IS_ERR(bio)) {
 277                         pr_err("%s, readahead error at page %lu of nid %llu\n",
 278                                __func__, page->index,
 279                                EROFS_I(rac->mapping->host)->nid);
 280
 281                         bio = NULL;
 282                 }
 283
 284                 put_page(page);
 285         }
 286
 287         if (bio)
 288                 submit_bio(bio);
 289 }
 290
 291 static sector_t erofs_bmap(struct address_space *mapping, sector_t block)
 292 {
 293         struct inode *inode = mapping->host;
 294         struct erofs_map_blocks map = {
 295                 .m_la = blknr_to_addr(block),
 296         };
 297
 298         if (EROFS_I(inode)->datalayout == EROFS_INODE_FLAT_INLINE) {
 299                 erofs_blk_t blks = i_size_read(inode) >> LOG_BLOCK_SIZE;
 300
 301                 if (block >> LOG_SECTORS_PER_BLOCK >= blks)
 302                         return 0;
 303         }
 304
 305         if (!erofs_map_blocks_flatmode(inode, &map, EROFS_GET_BLOCKS_RAW))
 306                 return erofs_blknr(map.m_pa);
 307
 308         return 0;
 309 }
 310
 311 /* for uncompressed (aligned) files and raw access for other files */
 312 const struct address_space_operations erofs_raw_access_aops = {
 313         .readpage = erofs_raw_access_readpage,
 314         .readahead = erofs_raw_access_readahead,
 315         .bmap = erofs_bmap,
 316 };