1 // SPDX-License-Identifier: GPL-2.0-or-later
3 * Copyright (C) 2019 Oracle. All Rights Reserved.
4 * Author: Darrick J. Wong <darrick.wong@oracle.com>
8 #include "xfs_shared.h"
9 #include "xfs_format.h"
10 #include "xfs_log_format.h"
11 #include "xfs_trans_resv.h"
12 #include "xfs_mount.h"
13 #include "xfs_inode.h"
14 #include "xfs_btree.h"
15 #include "xfs_ialloc.h"
16 #include "xfs_ialloc_btree.h"
17 #include "xfs_iwalk.h"
18 #include "xfs_error.h"
19 #include "xfs_trace.h"
20 #include "xfs_icache.h"
21 #include "xfs_health.h"
22 #include "xfs_trans.h"
23 #include "xfs_pwork.h"
28 * Walking Inodes in the Filesystem
29 * ================================
31 * This iterator function walks a subset of filesystem inodes in increasing
32 * order from @startino until there are no more inodes. For each allocated
33 * inode it finds, it calls a walk function with the relevant inode number and
34 * a pointer to caller-provided data. The walk function can return the usual
35 * negative error code to stop the iteration; 0 to continue the iteration; or
36 * -ECANCELED to stop the iteration. This return value is returned to the
39 * Internally, we allow the walk function to do anything, which means that we
40 * cannot maintain the inobt cursor or our lock on the AGI buffer. We
41 * therefore cache the inobt records in kernel memory and only call the walk
42 * function when our memory buffer is full. @nr_recs is the number of records
43 * that we've cached, and @sz_recs is the size of our cache.
45 * It is the responsibility of the walk function to ensure it accesses
46 * allocated inodes, as the inobt records may be stale by the time they are
51 /* parallel work control data; will be null if single threaded */
52 struct xfs_pwork pwork;
56 struct xfs_perag *pag;
58 /* Where do we start the traversal? */
61 /* What was the last inode number we saw when iterating the inobt? */
64 /* Array of inobt records we cache. */
65 struct xfs_inobt_rec_incore *recs;
67 /* Number of entries allocated for the @recs array. */
70 /* Number of entries in the @recs array that are in use. */
73 /* Inode walk function and data pointer. */
74 xfs_iwalk_fn iwalk_fn;
75 xfs_inobt_walk_fn inobt_walk_fn;
79 * Make it look like the inodes up to startino are free so that
80 * bulkstat can start its inode iteration at the correct place without
81 * needing to special case everywhere.
83 unsigned int trim_start:1;
85 /* Skip empty inobt records? */
86 unsigned int skip_empty:1;
88 /* Drop the (hopefully empty) transaction when calling iwalk_fn. */
89 unsigned int drop_trans:1;
93 * Loop over all clusters in a chunk for a given incore inode allocation btree
94 * record. Do a readahead if there are any allocated inodes in that cluster.
99 struct xfs_perag *pag,
100 struct xfs_inobt_rec_incore *irec)
102 struct xfs_ino_geometry *igeo = M_IGEO(mp);
104 struct blk_plug plug;
105 int i; /* inode chunk index */
107 agbno = XFS_AGINO_TO_AGBNO(mp, irec->ir_startino);
109 blk_start_plug(&plug);
110 for (i = 0; i < XFS_INODES_PER_CHUNK; i += igeo->inodes_per_cluster) {
113 imask = xfs_inobt_maskn(i, igeo->inodes_per_cluster);
114 if (imask & ~irec->ir_free) {
115 xfs_btree_reada_bufs(mp, pag->pag_agno, agbno,
116 igeo->blocks_per_cluster,
119 agbno += igeo->blocks_per_cluster;
121 blk_finish_plug(&plug);
125 * Set the bits in @irec's free mask that correspond to the inodes before
126 * @agino so that we skip them. This is how we restart an inode walk that was
127 * interrupted in the middle of an inode record.
130 xfs_iwalk_adjust_start(
131 xfs_agino_t agino, /* starting inode of chunk */
132 struct xfs_inobt_rec_incore *irec) /* btree record */
134 int idx; /* index into inode chunk */
136 idx = agino - irec->ir_startino;
138 irec->ir_free |= xfs_inobt_maskn(0, idx);
139 irec->ir_freecount = hweight64(irec->ir_free);
142 /* Allocate memory for a walk. */
145 struct xfs_iwalk_ag *iwag)
149 ASSERT(iwag->recs == NULL);
152 /* Allocate a prefetch buffer for inobt records. */
153 size = iwag->sz_recs * sizeof(struct xfs_inobt_rec_incore);
154 iwag->recs = kmalloc(size, GFP_KERNEL | __GFP_RETRY_MAYFAIL);
155 if (iwag->recs == NULL)
161 /* Free memory we allocated for a walk. */
164 struct xfs_iwalk_ag *iwag)
170 /* For each inuse inode in each cached inobt record, call our function. */
173 struct xfs_iwalk_ag *iwag)
175 struct xfs_mount *mp = iwag->mp;
176 struct xfs_trans *tp = iwag->tp;
177 struct xfs_perag *pag = iwag->pag;
182 for (i = 0; i < iwag->nr_recs; i++) {
183 struct xfs_inobt_rec_incore *irec = &iwag->recs[i];
185 trace_xfs_iwalk_ag_rec(mp, pag->pag_agno, irec);
187 if (xfs_pwork_want_abort(&iwag->pwork))
190 if (iwag->inobt_walk_fn) {
191 error = iwag->inobt_walk_fn(mp, tp, pag->pag_agno, irec,
200 for (j = 0; j < XFS_INODES_PER_CHUNK; j++) {
201 if (xfs_pwork_want_abort(&iwag->pwork))
204 /* Skip if this inode is free */
205 if (XFS_INOBT_MASK(j) & irec->ir_free)
208 /* Otherwise call our function. */
209 ino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
210 irec->ir_startino + j);
211 error = iwag->iwalk_fn(mp, tp, ino, iwag->data);
220 /* Delete cursor and let go of AGI. */
223 struct xfs_trans *tp,
224 struct xfs_btree_cur **curpp,
225 struct xfs_buf **agi_bpp,
229 xfs_btree_del_cursor(*curpp, error);
233 xfs_trans_brelse(tp, *agi_bpp);
239 * Set ourselves up for walking inobt records starting from a given point in
242 * If caller passed in a nonzero start inode number, load the record from the
243 * inobt and make the record look like all the inodes before agino are free so
244 * that we skip them, and then move the cursor to the next inobt record. This
245 * is how we support starting an iwalk in the middle of an inode chunk.
247 * If the caller passed in a start number of zero, move the cursor to the first
250 * The caller is responsible for cleaning up the cursor and buffer pointer
251 * regardless of the error status.
255 struct xfs_iwalk_ag *iwag,
257 struct xfs_btree_cur **curpp,
258 struct xfs_buf **agi_bpp,
261 struct xfs_mount *mp = iwag->mp;
262 struct xfs_trans *tp = iwag->tp;
263 struct xfs_perag *pag = iwag->pag;
264 struct xfs_inobt_rec_incore *irec;
267 /* Set up a fresh cursor and empty the inobt cache. */
269 error = xfs_inobt_cur(pag, tp, XFS_BTNUM_INO, curpp, agi_bpp);
273 /* Starting at the beginning of the AG? That's easy! */
275 return xfs_inobt_lookup(*curpp, 0, XFS_LOOKUP_GE, has_more);
278 * Otherwise, we have to grab the inobt record where we left off, stuff
279 * the record into our cache, and then see if there are more records.
280 * We require a lookup cache of at least two elements so that the
281 * caller doesn't have to deal with tearing down the cursor to walk the
284 error = xfs_inobt_lookup(*curpp, agino, XFS_LOOKUP_LE, has_more);
289 * If the LE lookup at @agino yields no records, jump ahead to the
290 * inobt cursor increment to see if there are more records to process.
295 /* Get the record, should always work */
296 irec = &iwag->recs[iwag->nr_recs];
297 error = xfs_inobt_get_rec(*curpp, irec, has_more);
300 if (XFS_IS_CORRUPT(mp, *has_more != 1))
301 return -EFSCORRUPTED;
303 iwag->lastino = XFS_AGINO_TO_INO(mp, pag->pag_agno,
304 irec->ir_startino + XFS_INODES_PER_CHUNK - 1);
307 * If the LE lookup yielded an inobt record before the cursor position,
308 * skip it and see if there's another one after it.
310 if (irec->ir_startino + XFS_INODES_PER_CHUNK <= agino)
314 * If agino fell in the middle of the inode record, make it look like
315 * the inodes up to agino are free so that we don't return them again.
317 if (iwag->trim_start)
318 xfs_iwalk_adjust_start(agino, irec);
321 * The prefetch calculation is supposed to give us a large enough inobt
322 * record cache that grab_ichunk can stage a partial first record and
323 * the loop body can cache a record without having to check for cache
324 * space until after it reads an inobt record.
327 ASSERT(iwag->nr_recs < iwag->sz_recs);
330 return xfs_btree_increment(*curpp, 0, has_more);
334 * The inobt record cache is full, so preserve the inobt cursor state and
335 * run callbacks on the cached inobt records. When we're done, restore the
336 * cursor state to wherever the cursor would have been had the cache not been
337 * full (and therefore we could've just incremented the cursor) if *@has_more
338 * is true. On exit, *@has_more will indicate whether or not the caller should
339 * try for more inode records.
342 xfs_iwalk_run_callbacks(
343 struct xfs_iwalk_ag *iwag,
344 struct xfs_btree_cur **curpp,
345 struct xfs_buf **agi_bpp,
348 struct xfs_mount *mp = iwag->mp;
349 struct xfs_inobt_rec_incore *irec;
350 xfs_agino_t next_agino;
353 next_agino = XFS_INO_TO_AGINO(mp, iwag->lastino) + 1;
355 ASSERT(iwag->nr_recs > 0);
357 /* Delete cursor but remember the last record we cached... */
358 xfs_iwalk_del_inobt(iwag->tp, curpp, agi_bpp, 0);
359 irec = &iwag->recs[iwag->nr_recs - 1];
360 ASSERT(next_agino >= irec->ir_startino + XFS_INODES_PER_CHUNK);
362 if (iwag->drop_trans) {
363 xfs_trans_cancel(iwag->tp);
367 error = xfs_iwalk_ag_recs(iwag);
371 /* ...empty the cache... */
377 if (iwag->drop_trans) {
378 error = xfs_trans_alloc_empty(mp, &iwag->tp);
383 /* ...and recreate the cursor just past where we left off. */
384 error = xfs_inobt_cur(iwag->pag, iwag->tp, XFS_BTNUM_INO, curpp,
389 return xfs_inobt_lookup(*curpp, next_agino, XFS_LOOKUP_GE, has_more);
392 /* Walk all inodes in a single AG, from @iwag->startino to the end of the AG. */
395 struct xfs_iwalk_ag *iwag)
397 struct xfs_mount *mp = iwag->mp;
398 struct xfs_perag *pag = iwag->pag;
399 struct xfs_buf *agi_bp = NULL;
400 struct xfs_btree_cur *cur = NULL;
405 /* Set up our cursor at the right place in the inode btree. */
406 ASSERT(pag->pag_agno == XFS_INO_TO_AGNO(mp, iwag->startino));
407 agino = XFS_INO_TO_AGINO(mp, iwag->startino);
408 error = xfs_iwalk_ag_start(iwag, agino, &cur, &agi_bp, &has_more);
410 while (!error && has_more) {
411 struct xfs_inobt_rec_incore *irec;
415 if (xfs_pwork_want_abort(&iwag->pwork))
418 /* Fetch the inobt record. */
419 irec = &iwag->recs[iwag->nr_recs];
420 error = xfs_inobt_get_rec(cur, irec, &has_more);
421 if (error || !has_more)
424 /* Make sure that we always move forward. */
425 rec_fsino = XFS_AGINO_TO_INO(mp, pag->pag_agno, irec->ir_startino);
426 if (iwag->lastino != NULLFSINO &&
427 XFS_IS_CORRUPT(mp, iwag->lastino >= rec_fsino)) {
428 error = -EFSCORRUPTED;
431 iwag->lastino = rec_fsino + XFS_INODES_PER_CHUNK - 1;
433 /* No allocated inodes in this chunk; skip it. */
434 if (iwag->skip_empty && irec->ir_freecount == irec->ir_count) {
435 error = xfs_btree_increment(cur, 0, &has_more);
442 * Start readahead for this inode chunk in anticipation of
443 * walking the inodes.
446 xfs_iwalk_ichunk_ra(mp, pag, irec);
449 * If there's space in the buffer for more records, increment
450 * the btree cursor and grab more.
452 if (++iwag->nr_recs < iwag->sz_recs) {
453 error = xfs_btree_increment(cur, 0, &has_more);
454 if (error || !has_more)
460 * Otherwise, we need to save cursor state and run the callback
461 * function on the cached records. The run_callbacks function
462 * is supposed to return a cursor pointing to the record where
463 * we would be if we had been able to increment like above.
466 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
469 if (iwag->nr_recs == 0 || error)
472 /* Walk the unprocessed records in the cache. */
473 error = xfs_iwalk_run_callbacks(iwag, &cur, &agi_bp, &has_more);
476 xfs_iwalk_del_inobt(iwag->tp, &cur, &agi_bp, error);
481 * We experimentally determined that the reduction in ioctl call overhead
482 * diminishes when userspace asks for more than 2048 inodes, so we'll cap
483 * prefetch at this point.
485 #define IWALK_MAX_INODE_PREFETCH (2048U)
488 * Given the number of inodes to prefetch, set the number of inobt records that
489 * we cache in memory, which controls the number of inodes we try to read
490 * ahead. Set the maximum if @inodes == 0.
492 static inline unsigned int
496 unsigned int inobt_records;
499 * If the caller didn't tell us the number of inodes they wanted,
500 * assume the maximum prefetch possible for best performance.
501 * Otherwise, cap prefetch at that maximum so that we don't start an
502 * absurd amount of prefetch.
505 inodes = IWALK_MAX_INODE_PREFETCH;
506 inodes = min(inodes, IWALK_MAX_INODE_PREFETCH);
508 /* Round the inode count up to a full chunk. */
509 inodes = round_up(inodes, XFS_INODES_PER_CHUNK);
512 * In order to convert the number of inodes to prefetch into an
513 * estimate of the number of inobt records to cache, we require a
514 * conversion factor that reflects our expectations of the average
515 * loading factor of an inode chunk. Based on data gathered, most
516 * (but not all) filesystems manage to keep the inode chunks totally
517 * full, so we'll underestimate slightly so that our readahead will
518 * still deliver the performance we want on aging filesystems:
520 * inobt = inodes / (INODES_PER_CHUNK * (4 / 5));
522 * The funny math is to avoid integer division.
524 inobt_records = (inodes * 5) / (4 * XFS_INODES_PER_CHUNK);
527 * Allocate enough space to prefetch at least two inobt records so that
528 * we can cache both the record where the iwalk started and the next
529 * record. This simplifies the AG inode walk loop setup code.
531 return max(inobt_records, 2U);
535 * Walk all inodes in the filesystem starting from @startino. The @iwalk_fn
536 * will be called for each allocated inode, being passed the inode's number and
537 * @data. @max_prefetch controls how many inobt records' worth of inodes we
542 struct xfs_mount *mp,
543 struct xfs_trans *tp,
546 xfs_iwalk_fn iwalk_fn,
547 unsigned int inode_records,
550 struct xfs_iwalk_ag iwag = {
553 .iwalk_fn = iwalk_fn,
555 .startino = startino,
556 .sz_recs = xfs_iwalk_prefetch(inode_records),
559 .pwork = XFS_PWORK_SINGLE_THREADED,
560 .lastino = NULLFSINO,
562 struct xfs_perag *pag;
563 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
566 ASSERT(agno < mp->m_sb.sb_agcount);
567 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
569 error = xfs_iwalk_alloc(&iwag);
573 for_each_perag_from(mp, agno, pag) {
575 error = xfs_iwalk_ag(&iwag);
578 iwag.startino = XFS_AGINO_TO_INO(mp, agno + 1, 0);
579 if (flags & XFS_INOBT_WALK_SAME_AG)
586 xfs_iwalk_free(&iwag);
590 /* Run per-thread iwalk work. */
593 struct xfs_mount *mp,
594 struct xfs_pwork *pwork)
596 struct xfs_iwalk_ag *iwag;
599 iwag = container_of(pwork, struct xfs_iwalk_ag, pwork);
600 if (xfs_pwork_want_abort(pwork))
603 error = xfs_iwalk_alloc(iwag);
607 * Grab an empty transaction so that we can use its recursive buffer
608 * locking abilities to detect cycles in the inobt without deadlocking.
610 error = xfs_trans_alloc_empty(mp, &iwag->tp);
613 iwag->drop_trans = 1;
615 error = xfs_iwalk_ag(iwag);
617 xfs_trans_cancel(iwag->tp);
618 xfs_iwalk_free(iwag);
620 xfs_perag_put(iwag->pag);
626 * Walk all the inodes in the filesystem using multiple threads to process each
631 struct xfs_mount *mp,
634 xfs_iwalk_fn iwalk_fn,
635 unsigned int inode_records,
639 struct xfs_pwork_ctl pctl;
640 struct xfs_perag *pag;
641 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
644 ASSERT(agno < mp->m_sb.sb_agcount);
645 ASSERT(!(flags & ~XFS_IWALK_FLAGS_ALL));
647 error = xfs_pwork_init(mp, &pctl, xfs_iwalk_ag_work, "xfs_iwalk");
651 for_each_perag_from(mp, agno, pag) {
652 struct xfs_iwalk_ag *iwag;
654 if (xfs_pwork_ctl_want_abort(&pctl))
657 iwag = kzalloc(sizeof(struct xfs_iwalk_ag),
658 GFP_KERNEL | __GFP_NOFAIL);
662 * perag is being handed off to async work, so take a passive
663 * reference for the async work to release.
665 iwag->pag = xfs_perag_hold(pag);
666 iwag->iwalk_fn = iwalk_fn;
668 iwag->startino = startino;
669 iwag->sz_recs = xfs_iwalk_prefetch(inode_records);
670 iwag->lastino = NULLFSINO;
671 xfs_pwork_queue(&pctl, &iwag->pwork);
672 startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
673 if (flags & XFS_INOBT_WALK_SAME_AG)
679 xfs_pwork_poll(&pctl);
680 return xfs_pwork_destroy(&pctl);
684 * Allow callers to cache up to a page's worth of inobt records. This reflects
685 * the existing inumbers prefetching behavior. Since the inobt walk does not
686 * itself do anything with the inobt records, we can set a fairly high limit
689 #define MAX_INOBT_WALK_PREFETCH \
690 (PAGE_SIZE / sizeof(struct xfs_inobt_rec_incore))
693 * Given the number of records that the user wanted, set the number of inobt
694 * records that we buffer in memory. Set the maximum if @inobt_records == 0.
696 static inline unsigned int
697 xfs_inobt_walk_prefetch(
698 unsigned int inobt_records)
701 * If the caller didn't tell us the number of inobt records they
702 * wanted, assume the maximum prefetch possible for best performance.
704 if (inobt_records == 0)
705 inobt_records = MAX_INOBT_WALK_PREFETCH;
708 * Allocate enough space to prefetch at least two inobt records so that
709 * we can cache both the record where the iwalk started and the next
710 * record. This simplifies the AG inode walk loop setup code.
712 inobt_records = max(inobt_records, 2U);
715 * Cap prefetch at that maximum so that we don't use an absurd amount
718 return min_t(unsigned int, inobt_records, MAX_INOBT_WALK_PREFETCH);
722 * Walk all inode btree records in the filesystem starting from @startino. The
723 * @inobt_walk_fn will be called for each btree record, being passed the incore
724 * record and @data. @max_prefetch controls how many inobt records we try to
725 * cache ahead of time.
729 struct xfs_mount *mp,
730 struct xfs_trans *tp,
733 xfs_inobt_walk_fn inobt_walk_fn,
734 unsigned int inobt_records,
737 struct xfs_iwalk_ag iwag = {
740 .inobt_walk_fn = inobt_walk_fn,
742 .startino = startino,
743 .sz_recs = xfs_inobt_walk_prefetch(inobt_records),
744 .pwork = XFS_PWORK_SINGLE_THREADED,
745 .lastino = NULLFSINO,
747 struct xfs_perag *pag;
748 xfs_agnumber_t agno = XFS_INO_TO_AGNO(mp, startino);
751 ASSERT(agno < mp->m_sb.sb_agcount);
752 ASSERT(!(flags & ~XFS_INOBT_WALK_FLAGS_ALL));
754 error = xfs_iwalk_alloc(&iwag);
758 for_each_perag_from(mp, agno, pag) {
760 error = xfs_iwalk_ag(&iwag);
763 iwag.startino = XFS_AGINO_TO_INO(mp, pag->pag_agno + 1, 0);
764 if (flags & XFS_INOBT_WALK_SAME_AG)
771 xfs_iwalk_free(&iwag);