1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2018-2019 HUAWEI, Inc.
4 * https://www.huawei.com/
5 * Created by Gao Xiang <gaoxiang25@huawei.com>
8 #include <asm/unaligned.h>
9 #include <trace/events/erofs.h>
11 int z_erofs_fill_inode(struct inode *inode)
13 struct erofs_inode *const vi = EROFS_I(inode);
14 struct erofs_sb_info *sbi = EROFS_SB(inode->i_sb);
16 if (!erofs_sb_has_big_pcluster(sbi) &&
17 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY) {
19 vi->z_algorithmtype[0] = 0;
20 vi->z_algorithmtype[1] = 0;
21 vi->z_logical_clusterbits = LOG_BLOCK_SIZE;
22 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
24 inode->i_mapping->a_ops = &z_erofs_aops;
28 static int z_erofs_fill_inode_lazy(struct inode *inode)
30 struct erofs_inode *const vi = EROFS_I(inode);
31 struct super_block *const sb = inode->i_sb;
36 struct z_erofs_map_header *h;
38 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags)) {
40 * paired with smp_mb() at the end of the function to ensure
41 * fields will only be observed after the bit is set.
47 if (wait_on_bit_lock(&vi->flags, EROFS_I_BL_Z_BIT, TASK_KILLABLE))
51 if (test_bit(EROFS_I_Z_INITED_BIT, &vi->flags))
54 DBG_BUGON(!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
55 vi->datalayout == EROFS_INODE_FLAT_COMPRESSION_LEGACY);
57 pos = ALIGN(iloc(EROFS_SB(sb), vi->nid) + vi->inode_isize +
59 page = erofs_get_meta_page(sb, erofs_blknr(pos));
65 kaddr = kmap_atomic(page);
67 h = kaddr + erofs_blkoff(pos);
68 vi->z_advise = le16_to_cpu(h->h_advise);
69 vi->z_algorithmtype[0] = h->h_algorithmtype & 15;
70 vi->z_algorithmtype[1] = h->h_algorithmtype >> 4;
72 if (vi->z_algorithmtype[0] >= Z_EROFS_COMPRESSION_MAX) {
73 erofs_err(sb, "unknown compression format %u for nid %llu, please upgrade kernel",
74 vi->z_algorithmtype[0], vi->nid);
79 vi->z_logical_clusterbits = LOG_BLOCK_SIZE + (h->h_clusterbits & 7);
80 if (!erofs_sb_has_big_pcluster(EROFS_SB(sb)) &&
81 vi->z_advise & (Z_EROFS_ADVISE_BIG_PCLUSTER_1 |
82 Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
83 erofs_err(sb, "per-inode big pcluster without sb feature for nid %llu",
88 if (vi->datalayout == EROFS_INODE_FLAT_COMPRESSION &&
89 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1) ^
90 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_2)) {
91 erofs_err(sb, "big pcluster head1/2 of compact indexes should be consistent for nid %llu",
96 /* paired with smp_mb() at the beginning of the function */
98 set_bit(EROFS_I_Z_INITED_BIT, &vi->flags);
100 kunmap_atomic(kaddr);
104 clear_and_wake_up_bit(EROFS_I_BL_Z_BIT, &vi->flags);
108 struct z_erofs_maprecorder {
110 struct erofs_map_blocks *map;
114 /* compression extent information gathered */
118 erofs_blk_t pblk, compressedlcs;
121 static int z_erofs_reload_indexes(struct z_erofs_maprecorder *m,
124 struct super_block *const sb = m->inode->i_sb;
125 struct erofs_map_blocks *const map = m->map;
126 struct page *mpage = map->mpage;
129 if (mpage->index == eblk) {
131 m->kaddr = kmap_atomic(mpage);
136 kunmap_atomic(m->kaddr);
142 mpage = erofs_get_meta_page(sb, eblk);
145 return PTR_ERR(mpage);
147 m->kaddr = kmap_atomic(mpage);
153 static int legacy_load_cluster_from_disk(struct z_erofs_maprecorder *m,
156 struct inode *const inode = m->inode;
157 struct erofs_inode *const vi = EROFS_I(inode);
158 const erofs_off_t ibase = iloc(EROFS_I_SB(inode), vi->nid);
159 const erofs_off_t pos =
160 Z_EROFS_VLE_LEGACY_INDEX_ALIGN(ibase + vi->inode_isize +
162 lcn * sizeof(struct z_erofs_vle_decompressed_index);
163 struct z_erofs_vle_decompressed_index *di;
164 unsigned int advise, type;
167 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
172 di = m->kaddr + erofs_blkoff(pos);
174 advise = le16_to_cpu(di->di_advise);
175 type = (advise >> Z_EROFS_VLE_DI_CLUSTER_TYPE_BIT) &
176 ((1 << Z_EROFS_VLE_DI_CLUSTER_TYPE_BITS) - 1);
178 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
179 m->clusterofs = 1 << vi->z_logical_clusterbits;
180 m->delta[0] = le16_to_cpu(di->di_u.delta[0]);
181 if (m->delta[0] & Z_EROFS_VLE_DI_D0_CBLKCNT) {
182 if (!(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
184 return -EFSCORRUPTED;
186 m->compressedlcs = m->delta[0] &
187 ~Z_EROFS_VLE_DI_D0_CBLKCNT;
190 m->delta[1] = le16_to_cpu(di->di_u.delta[1]);
192 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
193 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
194 m->clusterofs = le16_to_cpu(di->di_clusterofs);
195 m->pblk = le32_to_cpu(di->di_u.blkaddr);
205 static unsigned int decode_compactedbits(unsigned int lobits,
207 u8 *in, unsigned int pos, u8 *type)
209 const unsigned int v = get_unaligned_le32(in + pos / 8) >> (pos & 7);
210 const unsigned int lo = v & lomask;
212 *type = (v >> lobits) & 3;
216 static int unpack_compacted_index(struct z_erofs_maprecorder *m,
217 unsigned int amortizedshift,
220 struct erofs_inode *const vi = EROFS_I(m->inode);
221 const unsigned int lclusterbits = vi->z_logical_clusterbits;
222 const unsigned int lomask = (1 << lclusterbits) - 1;
223 unsigned int vcnt, base, lo, encodebits, nblk;
228 if (1 << amortizedshift == 4)
230 else if (1 << amortizedshift == 2 && lclusterbits == 12)
235 big_pcluster = vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1;
236 encodebits = ((vcnt << amortizedshift) - sizeof(__le32)) * 8 / vcnt;
237 base = round_down(eofs, vcnt << amortizedshift);
238 in = m->kaddr + base;
240 i = (eofs - base) >> amortizedshift;
242 lo = decode_compactedbits(lclusterbits, lomask,
243 in, encodebits * i, &type);
245 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
246 m->clusterofs = 1 << lclusterbits;
247 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
250 return -EFSCORRUPTED;
252 m->compressedlcs = lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
255 } else if (i + 1 != (int)vcnt) {
260 * since the last lcluster in the pack is special,
261 * of which lo saves delta[1] rather than delta[0].
262 * Hence, get delta[0] by the previous lcluster indirectly.
264 lo = decode_compactedbits(lclusterbits, lomask,
265 in, encodebits * (i - 1), &type);
266 if (type != Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
268 else if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT)
270 m->delta[0] = lo + 1;
275 /* figout out blkaddr (pblk) for HEAD lclusters */
280 lo = decode_compactedbits(lclusterbits, lomask,
281 in, encodebits * i, &type);
282 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD)
292 lo = decode_compactedbits(lclusterbits, lomask,
293 in, encodebits * i, &type);
294 if (type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD) {
295 if (lo & Z_EROFS_VLE_DI_D0_CBLKCNT) {
297 nblk += lo & ~Z_EROFS_VLE_DI_D0_CBLKCNT;
300 /* bigpcluster shouldn't have plain d0 == 1 */
303 return -EFSCORRUPTED;
311 in += (vcnt << amortizedshift) - sizeof(__le32);
312 m->pblk = le32_to_cpu(*(__le32 *)in) + nblk;
316 static int compacted_load_cluster_from_disk(struct z_erofs_maprecorder *m,
319 struct inode *const inode = m->inode;
320 struct erofs_inode *const vi = EROFS_I(inode);
321 const unsigned int lclusterbits = vi->z_logical_clusterbits;
322 const erofs_off_t ebase = ALIGN(iloc(EROFS_I_SB(inode), vi->nid) +
323 vi->inode_isize + vi->xattr_isize, 8) +
324 sizeof(struct z_erofs_map_header);
325 const unsigned int totalidx = DIV_ROUND_UP(inode->i_size, EROFS_BLKSIZ);
326 unsigned int compacted_4b_initial, compacted_2b;
327 unsigned int amortizedshift;
331 if (lclusterbits != 12)
338 /* used to align to 32-byte (compacted_2b) alignment */
339 compacted_4b_initial = (32 - ebase % 32) / 4;
340 if (compacted_4b_initial == 32 / 4)
341 compacted_4b_initial = 0;
343 if (vi->z_advise & Z_EROFS_ADVISE_COMPACTED_2B)
344 compacted_2b = rounddown(totalidx - compacted_4b_initial, 16);
349 if (lcn < compacted_4b_initial) {
353 pos += compacted_4b_initial * 4;
354 lcn -= compacted_4b_initial;
356 if (lcn < compacted_2b) {
360 pos += compacted_2b * 2;
364 pos += lcn * (1 << amortizedshift);
365 err = z_erofs_reload_indexes(m, erofs_blknr(pos));
368 return unpack_compacted_index(m, amortizedshift, erofs_blkoff(pos));
371 static int z_erofs_load_cluster_from_disk(struct z_erofs_maprecorder *m,
374 const unsigned int datamode = EROFS_I(m->inode)->datalayout;
376 if (datamode == EROFS_INODE_FLAT_COMPRESSION_LEGACY)
377 return legacy_load_cluster_from_disk(m, lcn);
379 if (datamode == EROFS_INODE_FLAT_COMPRESSION)
380 return compacted_load_cluster_from_disk(m, lcn);
385 static int z_erofs_extent_lookback(struct z_erofs_maprecorder *m,
386 unsigned int lookback_distance)
388 struct erofs_inode *const vi = EROFS_I(m->inode);
389 struct erofs_map_blocks *const map = m->map;
390 const unsigned int lclusterbits = vi->z_logical_clusterbits;
391 unsigned long lcn = m->lcn;
394 if (lcn < lookback_distance) {
395 erofs_err(m->inode->i_sb,
396 "bogus lookback distance @ nid %llu", vi->nid);
398 return -EFSCORRUPTED;
401 /* load extent head logical cluster if needed */
402 lcn -= lookback_distance;
403 err = z_erofs_load_cluster_from_disk(m, lcn);
408 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
410 erofs_err(m->inode->i_sb,
411 "invalid lookback distance 0 @ nid %llu",
414 return -EFSCORRUPTED;
416 return z_erofs_extent_lookback(m, m->delta[0]);
417 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
418 map->m_flags &= ~EROFS_MAP_ZIPPED;
420 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
421 map->m_la = (lcn << lclusterbits) | m->clusterofs;
424 erofs_err(m->inode->i_sb,
425 "unknown type %u @ lcn %lu of nid %llu",
426 m->type, lcn, vi->nid);
433 static int z_erofs_get_extent_compressedlen(struct z_erofs_maprecorder *m,
434 unsigned int initial_lcn)
436 struct erofs_inode *const vi = EROFS_I(m->inode);
437 struct erofs_map_blocks *const map = m->map;
438 const unsigned int lclusterbits = vi->z_logical_clusterbits;
442 DBG_BUGON(m->type != Z_EROFS_VLE_CLUSTER_TYPE_PLAIN &&
443 m->type != Z_EROFS_VLE_CLUSTER_TYPE_HEAD);
444 if (!(map->m_flags & EROFS_MAP_ZIPPED) ||
445 !(vi->z_advise & Z_EROFS_ADVISE_BIG_PCLUSTER_1)) {
446 map->m_plen = 1 << lclusterbits;
451 if (m->compressedlcs)
454 err = z_erofs_load_cluster_from_disk(m, lcn);
459 * If the 1st NONHEAD lcluster has already been handled initially w/o
460 * valid compressedlcs, which means at least it mustn't be CBLKCNT, or
461 * an internal implemenatation error is detected.
463 * The following code can also handle it properly anyway, but let's
464 * BUG_ON in the debugging mode only for developers to notice that.
466 DBG_BUGON(lcn == initial_lcn &&
467 m->type == Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD);
470 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
471 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
473 * if the 1st NONHEAD lcluster is actually PLAIN or HEAD type
474 * rather than CBLKCNT, it's a 1 lcluster-sized pcluster.
476 m->compressedlcs = 1;
478 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
479 if (m->delta[0] != 1)
480 goto err_bonus_cblkcnt;
481 if (m->compressedlcs)
485 erofs_err(m->inode->i_sb,
486 "cannot found CBLKCNT @ lcn %lu of nid %llu",
489 return -EFSCORRUPTED;
492 map->m_plen = m->compressedlcs << lclusterbits;
495 erofs_err(m->inode->i_sb,
496 "bogus CBLKCNT @ lcn %lu of nid %llu",
499 return -EFSCORRUPTED;
502 int z_erofs_map_blocks_iter(struct inode *inode,
503 struct erofs_map_blocks *map,
506 struct erofs_inode *const vi = EROFS_I(inode);
507 struct z_erofs_maprecorder m = {
512 unsigned int lclusterbits, endoff;
513 unsigned long initial_lcn;
514 unsigned long long ofs, end;
516 trace_z_erofs_map_blocks_iter_enter(inode, map, flags);
518 /* when trying to read beyond EOF, leave it unmapped */
519 if (map->m_la >= inode->i_size) {
520 map->m_llen = map->m_la + 1 - inode->i_size;
521 map->m_la = inode->i_size;
526 err = z_erofs_fill_inode_lazy(inode);
530 lclusterbits = vi->z_logical_clusterbits;
532 initial_lcn = ofs >> lclusterbits;
533 endoff = ofs & ((1 << lclusterbits) - 1);
535 err = z_erofs_load_cluster_from_disk(&m, initial_lcn);
539 map->m_flags = EROFS_MAP_ZIPPED; /* by default, compressed */
540 end = (m.lcn + 1ULL) << lclusterbits;
543 case Z_EROFS_VLE_CLUSTER_TYPE_PLAIN:
544 if (endoff >= m.clusterofs)
545 map->m_flags &= ~EROFS_MAP_ZIPPED;
547 case Z_EROFS_VLE_CLUSTER_TYPE_HEAD:
548 if (endoff >= m.clusterofs) {
549 map->m_la = (m.lcn << lclusterbits) | m.clusterofs;
552 /* m.lcn should be >= 1 if endoff < m.clusterofs */
554 erofs_err(inode->i_sb,
555 "invalid logical cluster 0 at nid %llu",
560 end = (m.lcn << lclusterbits) | m.clusterofs;
561 map->m_flags |= EROFS_MAP_FULL_MAPPED;
564 case Z_EROFS_VLE_CLUSTER_TYPE_NONHEAD:
565 /* get the corresponding first chunk */
566 err = z_erofs_extent_lookback(&m, m.delta[0]);
571 erofs_err(inode->i_sb,
572 "unknown type %u @ offset %llu of nid %llu",
573 m.type, ofs, vi->nid);
578 map->m_llen = end - map->m_la;
579 map->m_pa = blknr_to_addr(m.pblk);
580 map->m_flags |= EROFS_MAP_MAPPED;
582 err = z_erofs_get_extent_compressedlen(&m, initial_lcn);
587 kunmap_atomic(m.kaddr);
590 erofs_dbg("%s, m_la %llu m_pa %llu m_llen %llu m_plen %llu m_flags 0%o",
591 __func__, map->m_la, map->m_pa,
592 map->m_llen, map->m_plen, map->m_flags);
594 trace_z_erofs_map_blocks_iter_exit(inode, map, flags, err);
596 /* aggressively BUG_ON iff CONFIG_EROFS_FS_DEBUG is on */
597 DBG_BUGON(err < 0 && err != -ENOMEM);