2 * Copyright (C) Sistina Software, Inc. 1997-2003 All rights reserved.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
5 * This copyrighted material is made available to anyone wishing to use,
6 * modify, copy, or redistribute it subject to the terms and conditions
7 * of the GNU General Public License version 2.
10 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12 #include <linux/slab.h>
13 #include <linux/spinlock.h>
14 #include <linux/completion.h>
15 #include <linux/buffer_head.h>
17 #include <linux/gfs2_ondisk.h>
18 #include <linux/prefetch.h>
19 #include <linux/blkdev.h>
20 #include <linux/rbtree.h>
21 #include <linux/random.h>
36 #include "trace_gfs2.h"
39 #define BFITNOENT ((u32)~0)
40 #define NO_BLOCK ((u64)~0)
42 #if BITS_PER_LONG == 32
43 #define LBITMASK (0x55555555UL)
44 #define LBITSKIP55 (0x55555555UL)
45 #define LBITSKIP00 (0x00000000UL)
47 #define LBITMASK (0x5555555555555555UL)
48 #define LBITSKIP55 (0x5555555555555555UL)
49 #define LBITSKIP00 (0x0000000000000000UL)
53 * These routines are used by the resource group routines (rgrp.c)
54 * to keep track of block allocation. Each block is represented by two
55 * bits. So, each byte represents GFS2_NBBY (i.e. 4) blocks.
58 * 1 = Used (not metadata)
59 * 2 = Unlinked (still in use) inode
68 static const char valid_change[16] = {
76 static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
77 const struct gfs2_inode *ip, bool nowrap);
81 * gfs2_setbit - Set a bit in the bitmaps
82 * @rbm: The position of the bit to set
83 * @do_clone: Also set the clone bitmap, if it exists
84 * @new_state: the new state of the block
88 static inline void gfs2_setbit(const struct gfs2_rbm *rbm, bool do_clone,
89 unsigned char new_state)
91 unsigned char *byte1, *byte2, *end, cur_state;
92 struct gfs2_bitmap *bi = rbm_bi(rbm);
93 unsigned int buflen = bi->bi_len;
94 const unsigned int bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE;
96 byte1 = bi->bi_bh->b_data + bi->bi_offset + (rbm->offset / GFS2_NBBY);
97 end = bi->bi_bh->b_data + bi->bi_offset + buflen;
101 cur_state = (*byte1 >> bit) & GFS2_BIT_MASK;
103 if (unlikely(!valid_change[new_state * 4 + cur_state])) {
104 pr_warn("buf_blk = 0x%x old_state=%d, new_state=%d\n",
105 rbm->offset, cur_state, new_state);
106 pr_warn("rgrp=0x%llx bi_start=0x%x\n",
107 (unsigned long long)rbm->rgd->rd_addr, bi->bi_start);
108 pr_warn("bi_offset=0x%x bi_len=0x%x\n",
109 bi->bi_offset, bi->bi_len);
111 gfs2_consist_rgrpd(rbm->rgd);
114 *byte1 ^= (cur_state ^ new_state) << bit;
116 if (do_clone && bi->bi_clone) {
117 byte2 = bi->bi_clone + bi->bi_offset + (rbm->offset / GFS2_NBBY);
118 cur_state = (*byte2 >> bit) & GFS2_BIT_MASK;
119 *byte2 ^= (cur_state ^ new_state) << bit;
124 * gfs2_testbit - test a bit in the bitmaps
125 * @rbm: The bit to test
127 * Returns: The two bit block state of the requested bit
130 static inline u8 gfs2_testbit(const struct gfs2_rbm *rbm)
132 struct gfs2_bitmap *bi = rbm_bi(rbm);
133 const u8 *buffer = bi->bi_bh->b_data + bi->bi_offset;
137 byte = buffer + (rbm->offset / GFS2_NBBY);
138 bit = (rbm->offset % GFS2_NBBY) * GFS2_BIT_SIZE;
140 return (*byte >> bit) & GFS2_BIT_MASK;
145 * @ptr: Pointer to bitmap data
146 * @mask: Mask to use (normally 0x55555.... but adjusted for search start)
147 * @state: The state we are searching for
149 * We xor the bitmap data with a patter which is the bitwise opposite
150 * of what we are looking for, this gives rise to a pattern of ones
151 * wherever there is a match. Since we have two bits per entry, we
152 * take this pattern, shift it down by one place and then and it with
153 * the original. All the even bit positions (0,2,4, etc) then represent
154 * successful matches, so we mask with 0x55555..... to remove the unwanted
157 * This allows searching of a whole u64 at once (32 blocks) with a
158 * single test (on 64 bit arches).
161 static inline u64 gfs2_bit_search(const __le64 *ptr, u64 mask, u8 state)
164 static const u64 search[] = {
165 [0] = 0xffffffffffffffffULL,
166 [1] = 0xaaaaaaaaaaaaaaaaULL,
167 [2] = 0x5555555555555555ULL,
168 [3] = 0x0000000000000000ULL,
170 tmp = le64_to_cpu(*ptr) ^ search[state];
177 * rs_cmp - multi-block reservation range compare
178 * @blk: absolute file system block number of the new reservation
179 * @len: number of blocks in the new reservation
180 * @rs: existing reservation to compare against
182 * returns: 1 if the block range is beyond the reach of the reservation
183 * -1 if the block range is before the start of the reservation
184 * 0 if the block range overlaps with the reservation
186 static inline int rs_cmp(u64 blk, u32 len, struct gfs2_blkreserv *rs)
188 u64 startblk = gfs2_rbm_to_block(&rs->rs_rbm);
190 if (blk >= startblk + rs->rs_free)
192 if (blk + len - 1 < startblk)
198 * gfs2_bitfit - Search an rgrp's bitmap buffer to find a bit-pair representing
199 * a block in a given allocation state.
200 * @buf: the buffer that holds the bitmaps
201 * @len: the length (in bytes) of the buffer
202 * @goal: start search at this block's bit-pair (within @buffer)
203 * @state: GFS2_BLKST_XXX the state of the block we're looking for.
205 * Scope of @goal and returned block number is only within this bitmap buffer,
206 * not entire rgrp or filesystem. @buffer will be offset from the actual
207 * beginning of a bitmap block buffer, skipping any header structures, but
208 * headers are always a multiple of 64 bits long so that the buffer is
209 * always aligned to a 64 bit boundary.
211 * The size of the buffer is in bytes, but is it assumed that it is
212 * always ok to read a complete multiple of 64 bits at the end
213 * of the block in case the end is no aligned to a natural boundary.
215 * Return: the block number (bitmap buffer scope) that was found
218 static u32 gfs2_bitfit(const u8 *buf, const unsigned int len,
221 u32 spoint = (goal << 1) & ((8*sizeof(u64)) - 1);
222 const __le64 *ptr = ((__le64 *)buf) + (goal >> 5);
223 const __le64 *end = (__le64 *)(buf + ALIGN(len, sizeof(u64)));
225 u64 mask = 0x5555555555555555ULL;
228 /* Mask off bits we don't care about at the start of the search */
230 tmp = gfs2_bit_search(ptr, mask, state);
232 while(tmp == 0 && ptr < end) {
233 tmp = gfs2_bit_search(ptr, 0x5555555555555555ULL, state);
236 /* Mask off any bits which are more than len bytes from the start */
237 if (ptr == end && (len & (sizeof(u64) - 1)))
238 tmp &= (((u64)~0) >> (64 - 8*(len & (sizeof(u64) - 1))));
239 /* Didn't find anything, so return */
244 bit /= 2; /* two bits per entry in the bitmap */
245 return (((const unsigned char *)ptr - buf) * GFS2_NBBY) + bit;
249 * gfs2_rbm_from_block - Set the rbm based upon rgd and block number
250 * @rbm: The rbm with rgd already set correctly
251 * @block: The block number (filesystem relative)
253 * This sets the bi and offset members of an rbm based on a
254 * resource group and a filesystem relative block number. The
255 * resource group must be set in the rbm on entry, the bi and
256 * offset members will be set by this function.
258 * Returns: 0 on success, or an error code
261 static int gfs2_rbm_from_block(struct gfs2_rbm *rbm, u64 block)
263 u64 rblock = block - rbm->rgd->rd_data0;
265 if (WARN_ON_ONCE(rblock > UINT_MAX))
267 if (block >= rbm->rgd->rd_data0 + rbm->rgd->rd_data)
271 rbm->offset = (u32)(rblock);
272 /* Check if the block is within the first block */
273 if (rbm->offset < rbm_bi(rbm)->bi_blocks)
276 /* Adjust for the size diff between gfs2_meta_header and gfs2_rgrp */
277 rbm->offset += (sizeof(struct gfs2_rgrp) -
278 sizeof(struct gfs2_meta_header)) * GFS2_NBBY;
279 rbm->bii = rbm->offset / rbm->rgd->rd_sbd->sd_blocks_per_bitmap;
280 rbm->offset -= rbm->bii * rbm->rgd->rd_sbd->sd_blocks_per_bitmap;
285 * gfs2_rbm_incr - increment an rbm structure
286 * @rbm: The rbm with rgd already set correctly
288 * This function takes an existing rbm structure and increments it to the next
289 * viable block offset.
291 * Returns: If incrementing the offset would cause the rbm to go past the
292 * end of the rgrp, true is returned, otherwise false.
296 static bool gfs2_rbm_incr(struct gfs2_rbm *rbm)
298 if (rbm->offset + 1 < rbm_bi(rbm)->bi_blocks) { /* in the same bitmap */
302 if (rbm->bii == rbm->rgd->rd_length - 1) /* at the last bitmap */
311 * gfs2_unaligned_extlen - Look for free blocks which are not byte aligned
312 * @rbm: Position to search (value/result)
313 * @n_unaligned: Number of unaligned blocks to check
314 * @len: Decremented for each block found (terminate on zero)
316 * Returns: true if a non-free block is encountered
319 static bool gfs2_unaligned_extlen(struct gfs2_rbm *rbm, u32 n_unaligned, u32 *len)
324 for (n = 0; n < n_unaligned; n++) {
325 res = gfs2_testbit(rbm);
326 if (res != GFS2_BLKST_FREE)
331 if (gfs2_rbm_incr(rbm))
339 * gfs2_free_extlen - Return extent length of free blocks
340 * @rrbm: Starting position
341 * @len: Max length to check
343 * Starting at the block specified by the rbm, see how many free blocks
344 * there are, not reading more than len blocks ahead. This can be done
345 * using memchr_inv when the blocks are byte aligned, but has to be done
346 * on a block by block basis in case of unaligned blocks. Also this
347 * function can cope with bitmap boundaries (although it must stop on
348 * a resource group boundary)
350 * Returns: Number of free blocks in the extent
353 static u32 gfs2_free_extlen(const struct gfs2_rbm *rrbm, u32 len)
355 struct gfs2_rbm rbm = *rrbm;
356 u32 n_unaligned = rbm.offset & 3;
360 u8 *ptr, *start, *end;
362 struct gfs2_bitmap *bi;
365 gfs2_unaligned_extlen(&rbm, 4 - n_unaligned, &len))
368 n_unaligned = len & 3;
369 /* Start is now byte aligned */
372 start = bi->bi_bh->b_data;
374 start = bi->bi_clone;
375 start += bi->bi_offset;
376 end = start + bi->bi_len;
377 BUG_ON(rbm.offset & 3);
378 start += (rbm.offset / GFS2_NBBY);
379 bytes = min_t(u32, len / GFS2_NBBY, (end - start));
380 ptr = memchr_inv(start, 0, bytes);
381 chunk_size = ((ptr == NULL) ? bytes : (ptr - start));
382 chunk_size *= GFS2_NBBY;
383 BUG_ON(len < chunk_size);
385 block = gfs2_rbm_to_block(&rbm);
386 if (gfs2_rbm_from_block(&rbm, block + chunk_size)) {
394 n_unaligned = len & 3;
397 /* Deal with any bits left over at the end */
399 gfs2_unaligned_extlen(&rbm, n_unaligned, &len);
405 * gfs2_bitcount - count the number of bits in a certain state
406 * @rgd: the resource group descriptor
407 * @buffer: the buffer that holds the bitmaps
408 * @buflen: the length (in bytes) of the buffer
409 * @state: the state of the block we're looking for
411 * Returns: The number of bits
414 static u32 gfs2_bitcount(struct gfs2_rgrpd *rgd, const u8 *buffer,
415 unsigned int buflen, u8 state)
417 const u8 *byte = buffer;
418 const u8 *end = buffer + buflen;
419 const u8 state1 = state << 2;
420 const u8 state2 = state << 4;
421 const u8 state3 = state << 6;
424 for (; byte < end; byte++) {
425 if (((*byte) & 0x03) == state)
427 if (((*byte) & 0x0C) == state1)
429 if (((*byte) & 0x30) == state2)
431 if (((*byte) & 0xC0) == state3)
439 * gfs2_rgrp_verify - Verify that a resource group is consistent
444 void gfs2_rgrp_verify(struct gfs2_rgrpd *rgd)
446 struct gfs2_sbd *sdp = rgd->rd_sbd;
447 struct gfs2_bitmap *bi = NULL;
448 u32 length = rgd->rd_length;
452 memset(count, 0, 4 * sizeof(u32));
454 /* Count # blocks in each of 4 possible allocation states */
455 for (buf = 0; buf < length; buf++) {
456 bi = rgd->rd_bits + buf;
457 for (x = 0; x < 4; x++)
458 count[x] += gfs2_bitcount(rgd,
464 if (count[0] != rgd->rd_free) {
465 if (gfs2_consist_rgrpd(rgd))
466 fs_err(sdp, "free data mismatch: %u != %u\n",
467 count[0], rgd->rd_free);
471 tmp = rgd->rd_data - rgd->rd_free - rgd->rd_dinodes;
472 if (count[1] != tmp) {
473 if (gfs2_consist_rgrpd(rgd))
474 fs_err(sdp, "used data mismatch: %u != %u\n",
479 if (count[2] + count[3] != rgd->rd_dinodes) {
480 if (gfs2_consist_rgrpd(rgd))
481 fs_err(sdp, "used metadata mismatch: %u != %u\n",
482 count[2] + count[3], rgd->rd_dinodes);
488 * gfs2_blk2rgrpd - Find resource group for a given data/meta block number
489 * @sdp: The GFS2 superblock
490 * @blk: The data block number
491 * @exact: True if this needs to be an exact match
493 * The @exact argument should be set to true by most callers. The exception
494 * is when we need to match blocks which are not represented by the rgrp
495 * bitmap, but which are part of the rgrp (i.e. padding blocks) which are
496 * there for alignment purposes. Another way of looking at it is that @exact
497 * matches only valid data/metadata blocks, but with @exact false, it will
498 * match any block within the extent of the rgrp.
500 * Returns: The resource group, or NULL if not found
503 struct gfs2_rgrpd *gfs2_blk2rgrpd(struct gfs2_sbd *sdp, u64 blk, bool exact)
505 struct rb_node *n, *next;
506 struct gfs2_rgrpd *cur;
508 spin_lock(&sdp->sd_rindex_spin);
509 n = sdp->sd_rindex_tree.rb_node;
511 cur = rb_entry(n, struct gfs2_rgrpd, rd_node);
513 if (blk < cur->rd_addr)
515 else if (blk >= cur->rd_data0 + cur->rd_data)
518 spin_unlock(&sdp->sd_rindex_spin);
520 if (blk < cur->rd_addr)
522 if (blk >= cur->rd_data0 + cur->rd_data)
529 spin_unlock(&sdp->sd_rindex_spin);
535 * gfs2_rgrpd_get_first - get the first Resource Group in the filesystem
536 * @sdp: The GFS2 superblock
538 * Returns: The first rgrp in the filesystem
541 struct gfs2_rgrpd *gfs2_rgrpd_get_first(struct gfs2_sbd *sdp)
543 const struct rb_node *n;
544 struct gfs2_rgrpd *rgd;
546 spin_lock(&sdp->sd_rindex_spin);
547 n = rb_first(&sdp->sd_rindex_tree);
548 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
549 spin_unlock(&sdp->sd_rindex_spin);
555 * gfs2_rgrpd_get_next - get the next RG
556 * @rgd: the resource group descriptor
558 * Returns: The next rgrp
561 struct gfs2_rgrpd *gfs2_rgrpd_get_next(struct gfs2_rgrpd *rgd)
563 struct gfs2_sbd *sdp = rgd->rd_sbd;
564 const struct rb_node *n;
566 spin_lock(&sdp->sd_rindex_spin);
567 n = rb_next(&rgd->rd_node);
569 n = rb_first(&sdp->sd_rindex_tree);
571 if (unlikely(&rgd->rd_node == n)) {
572 spin_unlock(&sdp->sd_rindex_spin);
575 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
576 spin_unlock(&sdp->sd_rindex_spin);
580 void check_and_update_goal(struct gfs2_inode *ip)
582 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
583 if (!ip->i_goal || gfs2_blk2rgrpd(sdp, ip->i_goal, 1) == NULL)
584 ip->i_goal = ip->i_no_addr;
587 void gfs2_free_clones(struct gfs2_rgrpd *rgd)
591 for (x = 0; x < rgd->rd_length; x++) {
592 struct gfs2_bitmap *bi = rgd->rd_bits + x;
599 * gfs2_rsqa_alloc - make sure we have a reservation assigned to the inode
600 * plus a quota allocations data structure, if necessary
601 * @ip: the inode for this reservation
603 int gfs2_rsqa_alloc(struct gfs2_inode *ip)
605 return gfs2_qa_alloc(ip);
608 static void dump_rs(struct seq_file *seq, const struct gfs2_blkreserv *rs)
610 struct gfs2_inode *ip = container_of(rs, struct gfs2_inode, i_res);
612 gfs2_print_dbg(seq, " B: n:%llu s:%llu b:%u f:%u\n",
613 (unsigned long long)ip->i_no_addr,
614 (unsigned long long)gfs2_rbm_to_block(&rs->rs_rbm),
615 rs->rs_rbm.offset, rs->rs_free);
619 * __rs_deltree - remove a multi-block reservation from the rgd tree
620 * @rs: The reservation to remove
623 static void __rs_deltree(struct gfs2_blkreserv *rs)
625 struct gfs2_rgrpd *rgd;
627 if (!gfs2_rs_active(rs))
630 rgd = rs->rs_rbm.rgd;
631 trace_gfs2_rs(rs, TRACE_RS_TREEDEL);
632 rb_erase(&rs->rs_node, &rgd->rd_rstree);
633 RB_CLEAR_NODE(&rs->rs_node);
636 struct gfs2_bitmap *bi = rbm_bi(&rs->rs_rbm);
638 /* return reserved blocks to the rgrp */
639 BUG_ON(rs->rs_rbm.rgd->rd_reserved < rs->rs_free);
640 rs->rs_rbm.rgd->rd_reserved -= rs->rs_free;
641 /* The rgrp extent failure point is likely not to increase;
642 it will only do so if the freed blocks are somehow
643 contiguous with a span of free blocks that follows. Still,
644 it will force the number to be recalculated later. */
645 rgd->rd_extfail_pt += rs->rs_free;
647 clear_bit(GBF_FULL, &bi->bi_flags);
652 * gfs2_rs_deltree - remove a multi-block reservation from the rgd tree
653 * @rs: The reservation to remove
656 void gfs2_rs_deltree(struct gfs2_blkreserv *rs)
658 struct gfs2_rgrpd *rgd;
660 rgd = rs->rs_rbm.rgd;
662 spin_lock(&rgd->rd_rsspin);
665 spin_unlock(&rgd->rd_rsspin);
670 * gfs2_rsqa_delete - delete a multi-block reservation and quota allocation
671 * @ip: The inode for this reservation
672 * @wcount: The inode's write count, or NULL
675 void gfs2_rsqa_delete(struct gfs2_inode *ip, atomic_t *wcount)
677 down_write(&ip->i_rw_mutex);
678 if ((wcount == NULL) || (atomic_read(wcount) <= 1))
679 gfs2_rs_deltree(&ip->i_res);
680 up_write(&ip->i_rw_mutex);
681 gfs2_qa_delete(ip, wcount);
685 * return_all_reservations - return all reserved blocks back to the rgrp.
686 * @rgd: the rgrp that needs its space back
688 * We previously reserved a bunch of blocks for allocation. Now we need to
689 * give them back. This leave the reservation structures in tact, but removes
690 * all of their corresponding "no-fly zones".
692 static void return_all_reservations(struct gfs2_rgrpd *rgd)
695 struct gfs2_blkreserv *rs;
697 spin_lock(&rgd->rd_rsspin);
698 while ((n = rb_first(&rgd->rd_rstree))) {
699 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
702 spin_unlock(&rgd->rd_rsspin);
705 void gfs2_clear_rgrpd(struct gfs2_sbd *sdp)
708 struct gfs2_rgrpd *rgd;
709 struct gfs2_glock *gl;
711 while ((n = rb_first(&sdp->sd_rindex_tree))) {
712 rgd = rb_entry(n, struct gfs2_rgrpd, rd_node);
715 rb_erase(n, &sdp->sd_rindex_tree);
718 glock_clear_object(gl, rgd);
722 gfs2_free_clones(rgd);
725 return_all_reservations(rgd);
726 kmem_cache_free(gfs2_rgrpd_cachep, rgd);
730 static void gfs2_rindex_print(const struct gfs2_rgrpd *rgd)
732 pr_info("ri_addr = %llu\n", (unsigned long long)rgd->rd_addr);
733 pr_info("ri_length = %u\n", rgd->rd_length);
734 pr_info("ri_data0 = %llu\n", (unsigned long long)rgd->rd_data0);
735 pr_info("ri_data = %u\n", rgd->rd_data);
736 pr_info("ri_bitbytes = %u\n", rgd->rd_bitbytes);
740 * gfs2_compute_bitstructs - Compute the bitmap sizes
741 * @rgd: The resource group descriptor
743 * Calculates bitmap descriptors, one for each block that contains bitmap data
748 static int compute_bitstructs(struct gfs2_rgrpd *rgd)
750 struct gfs2_sbd *sdp = rgd->rd_sbd;
751 struct gfs2_bitmap *bi;
752 u32 length = rgd->rd_length; /* # blocks in hdr & bitmap */
753 u32 bytes_left, bytes;
759 rgd->rd_bits = kcalloc(length, sizeof(struct gfs2_bitmap), GFP_NOFS);
763 bytes_left = rgd->rd_bitbytes;
765 for (x = 0; x < length; x++) {
766 bi = rgd->rd_bits + x;
769 /* small rgrp; bitmap stored completely in header block */
772 bi->bi_offset = sizeof(struct gfs2_rgrp);
775 bi->bi_blocks = bytes * GFS2_NBBY;
778 bytes = sdp->sd_sb.sb_bsize - sizeof(struct gfs2_rgrp);
779 bi->bi_offset = sizeof(struct gfs2_rgrp);
782 bi->bi_blocks = bytes * GFS2_NBBY;
784 } else if (x + 1 == length) {
786 bi->bi_offset = sizeof(struct gfs2_meta_header);
787 bi->bi_start = rgd->rd_bitbytes - bytes_left;
789 bi->bi_blocks = bytes * GFS2_NBBY;
792 bytes = sdp->sd_sb.sb_bsize -
793 sizeof(struct gfs2_meta_header);
794 bi->bi_offset = sizeof(struct gfs2_meta_header);
795 bi->bi_start = rgd->rd_bitbytes - bytes_left;
797 bi->bi_blocks = bytes * GFS2_NBBY;
804 gfs2_consist_rgrpd(rgd);
807 bi = rgd->rd_bits + (length - 1);
808 if ((bi->bi_start + bi->bi_len) * GFS2_NBBY != rgd->rd_data) {
809 if (gfs2_consist_rgrpd(rgd)) {
810 gfs2_rindex_print(rgd);
811 fs_err(sdp, "start=%u len=%u offset=%u\n",
812 bi->bi_start, bi->bi_len, bi->bi_offset);
821 * gfs2_ri_total - Total up the file system space, according to the rindex.
822 * @sdp: the filesystem
825 u64 gfs2_ri_total(struct gfs2_sbd *sdp)
828 struct inode *inode = sdp->sd_rindex;
829 struct gfs2_inode *ip = GFS2_I(inode);
830 char buf[sizeof(struct gfs2_rindex)];
833 for (rgrps = 0;; rgrps++) {
834 loff_t pos = rgrps * sizeof(struct gfs2_rindex);
836 if (pos + sizeof(struct gfs2_rindex) > i_size_read(inode))
838 error = gfs2_internal_read(ip, buf, &pos,
839 sizeof(struct gfs2_rindex));
840 if (error != sizeof(struct gfs2_rindex))
842 total_data += be32_to_cpu(((struct gfs2_rindex *)buf)->ri_data);
847 static int rgd_insert(struct gfs2_rgrpd *rgd)
849 struct gfs2_sbd *sdp = rgd->rd_sbd;
850 struct rb_node **newn = &sdp->sd_rindex_tree.rb_node, *parent = NULL;
852 /* Figure out where to put new node */
854 struct gfs2_rgrpd *cur = rb_entry(*newn, struct gfs2_rgrpd,
858 if (rgd->rd_addr < cur->rd_addr)
859 newn = &((*newn)->rb_left);
860 else if (rgd->rd_addr > cur->rd_addr)
861 newn = &((*newn)->rb_right);
866 rb_link_node(&rgd->rd_node, parent, newn);
867 rb_insert_color(&rgd->rd_node, &sdp->sd_rindex_tree);
873 * read_rindex_entry - Pull in a new resource index entry from the disk
874 * @ip: Pointer to the rindex inode
876 * Returns: 0 on success, > 0 on EOF, error code otherwise
879 static int read_rindex_entry(struct gfs2_inode *ip)
881 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
882 const unsigned bsize = sdp->sd_sb.sb_bsize;
883 loff_t pos = sdp->sd_rgrps * sizeof(struct gfs2_rindex);
884 struct gfs2_rindex buf;
886 struct gfs2_rgrpd *rgd;
888 if (pos >= i_size_read(&ip->i_inode))
891 error = gfs2_internal_read(ip, (char *)&buf, &pos,
892 sizeof(struct gfs2_rindex));
894 if (error != sizeof(struct gfs2_rindex))
895 return (error == 0) ? 1 : error;
897 rgd = kmem_cache_zalloc(gfs2_rgrpd_cachep, GFP_NOFS);
903 rgd->rd_addr = be64_to_cpu(buf.ri_addr);
904 rgd->rd_length = be32_to_cpu(buf.ri_length);
905 rgd->rd_data0 = be64_to_cpu(buf.ri_data0);
906 rgd->rd_data = be32_to_cpu(buf.ri_data);
907 rgd->rd_bitbytes = be32_to_cpu(buf.ri_bitbytes);
908 spin_lock_init(&rgd->rd_rsspin);
910 error = compute_bitstructs(rgd);
914 error = gfs2_glock_get(sdp, rgd->rd_addr,
915 &gfs2_rgrp_glops, CREATE, &rgd->rd_gl);
919 rgd->rd_rgl = (struct gfs2_rgrp_lvb *)rgd->rd_gl->gl_lksb.sb_lvbptr;
920 rgd->rd_flags &= ~(GFS2_RDF_UPTODATE | GFS2_RDF_PREFERRED);
921 if (rgd->rd_data > sdp->sd_max_rg_data)
922 sdp->sd_max_rg_data = rgd->rd_data;
923 spin_lock(&sdp->sd_rindex_spin);
924 error = rgd_insert(rgd);
925 spin_unlock(&sdp->sd_rindex_spin);
927 glock_set_object(rgd->rd_gl, rgd);
928 rgd->rd_gl->gl_vm.start = (rgd->rd_addr * bsize) & PAGE_MASK;
929 rgd->rd_gl->gl_vm.end = PAGE_ALIGN((rgd->rd_addr +
930 rgd->rd_length) * bsize) - 1;
934 error = 0; /* someone else read in the rgrp; free it and ignore it */
935 gfs2_glock_put(rgd->rd_gl);
940 kmem_cache_free(gfs2_rgrpd_cachep, rgd);
945 * set_rgrp_preferences - Run all the rgrps, selecting some we prefer to use
946 * @sdp: the GFS2 superblock
948 * The purpose of this function is to select a subset of the resource groups
949 * and mark them as PREFERRED. We do it in such a way that each node prefers
950 * to use a unique set of rgrps to minimize glock contention.
952 static void set_rgrp_preferences(struct gfs2_sbd *sdp)
954 struct gfs2_rgrpd *rgd, *first;
957 /* Skip an initial number of rgrps, based on this node's journal ID.
958 That should start each node out on its own set. */
959 rgd = gfs2_rgrpd_get_first(sdp);
960 for (i = 0; i < sdp->sd_lockstruct.ls_jid; i++)
961 rgd = gfs2_rgrpd_get_next(rgd);
965 rgd->rd_flags |= GFS2_RDF_PREFERRED;
966 for (i = 0; i < sdp->sd_journals; i++) {
967 rgd = gfs2_rgrpd_get_next(rgd);
968 if (!rgd || rgd == first)
971 } while (rgd && rgd != first);
975 * gfs2_ri_update - Pull in a new resource index from the disk
976 * @ip: pointer to the rindex inode
978 * Returns: 0 on successful update, error code otherwise
981 static int gfs2_ri_update(struct gfs2_inode *ip)
983 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
987 error = read_rindex_entry(ip);
988 } while (error == 0);
993 set_rgrp_preferences(sdp);
995 sdp->sd_rindex_uptodate = 1;
1000 * gfs2_rindex_update - Update the rindex if required
1001 * @sdp: The GFS2 superblock
1003 * We grab a lock on the rindex inode to make sure that it doesn't
1004 * change whilst we are performing an operation. We keep this lock
1005 * for quite long periods of time compared to other locks. This
1006 * doesn't matter, since it is shared and it is very, very rarely
1007 * accessed in the exclusive mode (i.e. only when expanding the filesystem).
1009 * This makes sure that we're using the latest copy of the resource index
1010 * special file, which might have been updated if someone expanded the
1011 * filesystem (via gfs2_grow utility), which adds new resource groups.
1013 * Returns: 0 on succeess, error code otherwise
1016 int gfs2_rindex_update(struct gfs2_sbd *sdp)
1018 struct gfs2_inode *ip = GFS2_I(sdp->sd_rindex);
1019 struct gfs2_glock *gl = ip->i_gl;
1020 struct gfs2_holder ri_gh;
1022 int unlock_required = 0;
1024 /* Read new copy from disk if we don't have the latest */
1025 if (!sdp->sd_rindex_uptodate) {
1026 if (!gfs2_glock_is_locked_by_me(gl)) {
1027 error = gfs2_glock_nq_init(gl, LM_ST_SHARED, 0, &ri_gh);
1030 unlock_required = 1;
1032 if (!sdp->sd_rindex_uptodate)
1033 error = gfs2_ri_update(ip);
1034 if (unlock_required)
1035 gfs2_glock_dq_uninit(&ri_gh);
1041 static void gfs2_rgrp_in(struct gfs2_rgrpd *rgd, const void *buf)
1043 const struct gfs2_rgrp *str = buf;
1046 rg_flags = be32_to_cpu(str->rg_flags);
1047 rg_flags &= ~GFS2_RDF_MASK;
1048 rgd->rd_flags &= GFS2_RDF_MASK;
1049 rgd->rd_flags |= rg_flags;
1050 rgd->rd_free = be32_to_cpu(str->rg_free);
1051 rgd->rd_dinodes = be32_to_cpu(str->rg_dinodes);
1052 rgd->rd_igeneration = be64_to_cpu(str->rg_igeneration);
1053 /* rd_data0, rd_data and rd_bitbytes already set from rindex */
1056 static void gfs2_rgrp_out(struct gfs2_rgrpd *rgd, void *buf)
1058 struct gfs2_rgrpd *next = gfs2_rgrpd_get_next(rgd);
1059 struct gfs2_rgrp *str = buf;
1062 str->rg_flags = cpu_to_be32(rgd->rd_flags & ~GFS2_RDF_MASK);
1063 str->rg_free = cpu_to_be32(rgd->rd_free);
1064 str->rg_dinodes = cpu_to_be32(rgd->rd_dinodes);
1067 else if (next->rd_addr > rgd->rd_addr)
1068 str->rg_skip = cpu_to_be32(next->rd_addr - rgd->rd_addr);
1069 str->rg_igeneration = cpu_to_be64(rgd->rd_igeneration);
1070 str->rg_data0 = cpu_to_be64(rgd->rd_data0);
1071 str->rg_data = cpu_to_be32(rgd->rd_data);
1072 str->rg_bitbytes = cpu_to_be32(rgd->rd_bitbytes);
1074 crc = gfs2_disk_hash(buf, sizeof(struct gfs2_rgrp));
1075 str->rg_crc = cpu_to_be32(crc);
1077 memset(&str->rg_reserved, 0, sizeof(str->rg_reserved));
1080 static int gfs2_rgrp_lvb_valid(struct gfs2_rgrpd *rgd)
1082 struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl;
1083 struct gfs2_rgrp *str = (struct gfs2_rgrp *)rgd->rd_bits[0].bi_bh->b_data;
1085 if (rgl->rl_flags != str->rg_flags || rgl->rl_free != str->rg_free ||
1086 rgl->rl_dinodes != str->rg_dinodes ||
1087 rgl->rl_igeneration != str->rg_igeneration)
1092 static void gfs2_rgrp_ondisk2lvb(struct gfs2_rgrp_lvb *rgl, const void *buf)
1094 const struct gfs2_rgrp *str = buf;
1096 rgl->rl_magic = cpu_to_be32(GFS2_MAGIC);
1097 rgl->rl_flags = str->rg_flags;
1098 rgl->rl_free = str->rg_free;
1099 rgl->rl_dinodes = str->rg_dinodes;
1100 rgl->rl_igeneration = str->rg_igeneration;
1104 static void update_rgrp_lvb_unlinked(struct gfs2_rgrpd *rgd, u32 change)
1106 struct gfs2_rgrp_lvb *rgl = rgd->rd_rgl;
1107 u32 unlinked = be32_to_cpu(rgl->rl_unlinked) + change;
1108 rgl->rl_unlinked = cpu_to_be32(unlinked);
1111 static u32 count_unlinked(struct gfs2_rgrpd *rgd)
1113 struct gfs2_bitmap *bi;
1114 const u32 length = rgd->rd_length;
1115 const u8 *buffer = NULL;
1116 u32 i, goal, count = 0;
1118 for (i = 0, bi = rgd->rd_bits; i < length; i++, bi++) {
1120 buffer = bi->bi_bh->b_data + bi->bi_offset;
1121 WARN_ON(!buffer_uptodate(bi->bi_bh));
1122 while (goal < bi->bi_len * GFS2_NBBY) {
1123 goal = gfs2_bitfit(buffer, bi->bi_len, goal,
1124 GFS2_BLKST_UNLINKED);
1125 if (goal == BFITNOENT)
1137 * gfs2_rgrp_bh_get - Read in a RG's header and bitmaps
1138 * @rgd: the struct gfs2_rgrpd describing the RG to read in
1140 * Read in all of a Resource Group's header and bitmap blocks.
1141 * Caller must eventually call gfs2_rgrp_relse() to free the bitmaps.
1146 static int gfs2_rgrp_bh_get(struct gfs2_rgrpd *rgd)
1148 struct gfs2_sbd *sdp = rgd->rd_sbd;
1149 struct gfs2_glock *gl = rgd->rd_gl;
1150 unsigned int length = rgd->rd_length;
1151 struct gfs2_bitmap *bi;
1155 if (rgd->rd_bits[0].bi_bh != NULL)
1158 for (x = 0; x < length; x++) {
1159 bi = rgd->rd_bits + x;
1160 error = gfs2_meta_read(gl, rgd->rd_addr + x, 0, 0, &bi->bi_bh);
1165 for (y = length; y--;) {
1166 bi = rgd->rd_bits + y;
1167 error = gfs2_meta_wait(sdp, bi->bi_bh);
1170 if (gfs2_metatype_check(sdp, bi->bi_bh, y ? GFS2_METATYPE_RB :
1171 GFS2_METATYPE_RG)) {
1177 if (!(rgd->rd_flags & GFS2_RDF_UPTODATE)) {
1178 for (x = 0; x < length; x++)
1179 clear_bit(GBF_FULL, &rgd->rd_bits[x].bi_flags);
1180 gfs2_rgrp_in(rgd, (rgd->rd_bits[0].bi_bh)->b_data);
1181 rgd->rd_flags |= (GFS2_RDF_UPTODATE | GFS2_RDF_CHECK);
1182 rgd->rd_free_clone = rgd->rd_free;
1183 /* max out the rgrp allocation failure point */
1184 rgd->rd_extfail_pt = rgd->rd_free;
1186 if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic) {
1187 rgd->rd_rgl->rl_unlinked = cpu_to_be32(count_unlinked(rgd));
1188 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl,
1189 rgd->rd_bits[0].bi_bh->b_data);
1191 else if (sdp->sd_args.ar_rgrplvb) {
1192 if (!gfs2_rgrp_lvb_valid(rgd)){
1193 gfs2_consist_rgrpd(rgd);
1197 if (rgd->rd_rgl->rl_unlinked == 0)
1198 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1204 bi = rgd->rd_bits + x;
1207 gfs2_assert_warn(sdp, !bi->bi_clone);
1213 static int update_rgrp_lvb(struct gfs2_rgrpd *rgd)
1217 if (rgd->rd_flags & GFS2_RDF_UPTODATE)
1220 if (cpu_to_be32(GFS2_MAGIC) != rgd->rd_rgl->rl_magic)
1221 return gfs2_rgrp_bh_get(rgd);
1223 rl_flags = be32_to_cpu(rgd->rd_rgl->rl_flags);
1224 rl_flags &= ~GFS2_RDF_MASK;
1225 rgd->rd_flags &= GFS2_RDF_MASK;
1226 rgd->rd_flags |= (rl_flags | GFS2_RDF_UPTODATE | GFS2_RDF_CHECK);
1227 if (rgd->rd_rgl->rl_unlinked == 0)
1228 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1229 rgd->rd_free = be32_to_cpu(rgd->rd_rgl->rl_free);
1230 rgd->rd_free_clone = rgd->rd_free;
1231 rgd->rd_dinodes = be32_to_cpu(rgd->rd_rgl->rl_dinodes);
1232 rgd->rd_igeneration = be64_to_cpu(rgd->rd_rgl->rl_igeneration);
1236 int gfs2_rgrp_go_lock(struct gfs2_holder *gh)
1238 struct gfs2_rgrpd *rgd = gh->gh_gl->gl_object;
1239 struct gfs2_sbd *sdp = rgd->rd_sbd;
1241 if (gh->gh_flags & GL_SKIP && sdp->sd_args.ar_rgrplvb)
1243 return gfs2_rgrp_bh_get(rgd);
1247 * gfs2_rgrp_brelse - Release RG bitmaps read in with gfs2_rgrp_bh_get()
1248 * @rgd: The resource group
1252 void gfs2_rgrp_brelse(struct gfs2_rgrpd *rgd)
1254 int x, length = rgd->rd_length;
1256 for (x = 0; x < length; x++) {
1257 struct gfs2_bitmap *bi = rgd->rd_bits + x;
1267 * gfs2_rgrp_go_unlock - Unlock a rgrp glock
1268 * @gh: The glock holder for the resource group
1272 void gfs2_rgrp_go_unlock(struct gfs2_holder *gh)
1274 struct gfs2_rgrpd *rgd = gh->gh_gl->gl_object;
1275 int demote_requested = test_bit(GLF_DEMOTE, &gh->gh_gl->gl_flags) |
1276 test_bit(GLF_PENDING_DEMOTE, &gh->gh_gl->gl_flags);
1278 if (rgd && demote_requested)
1279 gfs2_rgrp_brelse(rgd);
1282 int gfs2_rgrp_send_discards(struct gfs2_sbd *sdp, u64 offset,
1283 struct buffer_head *bh,
1284 const struct gfs2_bitmap *bi, unsigned minlen, u64 *ptrimmed)
1286 struct super_block *sb = sdp->sd_vfs;
1289 sector_t nr_blks = 0;
1295 for (x = 0; x < bi->bi_len; x++) {
1296 const u8 *clone = bi->bi_clone ? bi->bi_clone : bi->bi_bh->b_data;
1297 clone += bi->bi_offset;
1300 const u8 *orig = bh->b_data + bi->bi_offset + x;
1301 diff = ~(*orig | (*orig >> 1)) & (*clone | (*clone >> 1));
1303 diff = ~(*clone | (*clone >> 1));
1308 blk = offset + ((bi->bi_start + x) * GFS2_NBBY);
1312 goto start_new_extent;
1313 if ((start + nr_blks) != blk) {
1314 if (nr_blks >= minlen) {
1315 rv = sb_issue_discard(sb,
1332 if (nr_blks >= minlen) {
1333 rv = sb_issue_discard(sb, start, nr_blks, GFP_NOFS, 0);
1339 *ptrimmed = trimmed;
1343 if (sdp->sd_args.ar_discard)
1344 fs_warn(sdp, "error %d on discard request, turning discards off for this filesystem\n", rv);
1345 sdp->sd_args.ar_discard = 0;
1350 * gfs2_fitrim - Generate discard requests for unused bits of the filesystem
1351 * @filp: Any file on the filesystem
1352 * @argp: Pointer to the arguments (also used to pass result)
1354 * Returns: 0 on success, otherwise error code
1357 int gfs2_fitrim(struct file *filp, void __user *argp)
1359 struct inode *inode = file_inode(filp);
1360 struct gfs2_sbd *sdp = GFS2_SB(inode);
1361 struct request_queue *q = bdev_get_queue(sdp->sd_vfs->s_bdev);
1362 struct buffer_head *bh;
1363 struct gfs2_rgrpd *rgd;
1364 struct gfs2_rgrpd *rgd_end;
1365 struct gfs2_holder gh;
1366 struct fstrim_range r;
1370 u64 start, end, minlen;
1372 unsigned bs_shift = sdp->sd_sb.sb_bsize_shift;
1374 if (!capable(CAP_SYS_ADMIN))
1377 if (!blk_queue_discard(q))
1380 if (copy_from_user(&r, argp, sizeof(r)))
1383 ret = gfs2_rindex_update(sdp);
1387 start = r.start >> bs_shift;
1388 end = start + (r.len >> bs_shift);
1389 minlen = max_t(u64, r.minlen,
1390 q->limits.discard_granularity) >> bs_shift;
1392 if (end <= start || minlen > sdp->sd_max_rg_data)
1395 rgd = gfs2_blk2rgrpd(sdp, start, 0);
1396 rgd_end = gfs2_blk2rgrpd(sdp, end, 0);
1398 if ((gfs2_rgrpd_get_first(sdp) == gfs2_rgrpd_get_next(rgd_end))
1399 && (start > rgd_end->rd_data0 + rgd_end->rd_data))
1400 return -EINVAL; /* start is beyond the end of the fs */
1404 ret = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_EXCLUSIVE, 0, &gh);
1408 if (!(rgd->rd_flags & GFS2_RGF_TRIMMED)) {
1409 /* Trim each bitmap in the rgrp */
1410 for (x = 0; x < rgd->rd_length; x++) {
1411 struct gfs2_bitmap *bi = rgd->rd_bits + x;
1412 ret = gfs2_rgrp_send_discards(sdp,
1413 rgd->rd_data0, NULL, bi, minlen,
1416 gfs2_glock_dq_uninit(&gh);
1422 /* Mark rgrp as having been trimmed */
1423 ret = gfs2_trans_begin(sdp, RES_RG_HDR, 0);
1425 bh = rgd->rd_bits[0].bi_bh;
1426 rgd->rd_flags |= GFS2_RGF_TRIMMED;
1427 gfs2_trans_add_meta(rgd->rd_gl, bh);
1428 gfs2_rgrp_out(rgd, bh->b_data);
1429 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, bh->b_data);
1430 gfs2_trans_end(sdp);
1433 gfs2_glock_dq_uninit(&gh);
1438 rgd = gfs2_rgrpd_get_next(rgd);
1442 r.len = trimmed << bs_shift;
1443 if (copy_to_user(argp, &r, sizeof(r)))
1450 * rs_insert - insert a new multi-block reservation into the rgrp's rb_tree
1451 * @ip: the inode structure
1454 static void rs_insert(struct gfs2_inode *ip)
1456 struct rb_node **newn, *parent = NULL;
1458 struct gfs2_blkreserv *rs = &ip->i_res;
1459 struct gfs2_rgrpd *rgd = rs->rs_rbm.rgd;
1460 u64 fsblock = gfs2_rbm_to_block(&rs->rs_rbm);
1462 BUG_ON(gfs2_rs_active(rs));
1464 spin_lock(&rgd->rd_rsspin);
1465 newn = &rgd->rd_rstree.rb_node;
1467 struct gfs2_blkreserv *cur =
1468 rb_entry(*newn, struct gfs2_blkreserv, rs_node);
1471 rc = rs_cmp(fsblock, rs->rs_free, cur);
1473 newn = &((*newn)->rb_right);
1475 newn = &((*newn)->rb_left);
1477 spin_unlock(&rgd->rd_rsspin);
1483 rb_link_node(&rs->rs_node, parent, newn);
1484 rb_insert_color(&rs->rs_node, &rgd->rd_rstree);
1486 /* Do our rgrp accounting for the reservation */
1487 rgd->rd_reserved += rs->rs_free; /* blocks reserved */
1488 spin_unlock(&rgd->rd_rsspin);
1489 trace_gfs2_rs(rs, TRACE_RS_INSERT);
1493 * rg_mblk_search - find a group of multiple free blocks to form a reservation
1494 * @rgd: the resource group descriptor
1495 * @ip: pointer to the inode for which we're reserving blocks
1496 * @ap: the allocation parameters
1500 static void rg_mblk_search(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip,
1501 const struct gfs2_alloc_parms *ap)
1503 struct gfs2_rbm rbm = { .rgd = rgd, };
1505 struct gfs2_blkreserv *rs = &ip->i_res;
1507 u32 free_blocks = rgd->rd_free_clone - rgd->rd_reserved;
1509 struct inode *inode = &ip->i_inode;
1511 if (S_ISDIR(inode->i_mode))
1514 extlen = max_t(u32, atomic_read(&rs->rs_sizehint), ap->target);
1515 extlen = clamp(extlen, RGRP_RSRV_MINBLKS, free_blocks);
1517 if ((rgd->rd_free_clone < rgd->rd_reserved) || (free_blocks < extlen))
1520 /* Find bitmap block that contains bits for goal block */
1521 if (rgrp_contains_block(rgd, ip->i_goal))
1524 goal = rgd->rd_last_alloc + rgd->rd_data0;
1526 if (WARN_ON(gfs2_rbm_from_block(&rbm, goal)))
1529 ret = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, &extlen, ip, true);
1532 rs->rs_free = extlen;
1535 if (goal == rgd->rd_last_alloc + rgd->rd_data0)
1536 rgd->rd_last_alloc = 0;
1541 * gfs2_next_unreserved_block - Return next block that is not reserved
1542 * @rgd: The resource group
1543 * @block: The starting block
1544 * @length: The required length
1545 * @ip: Ignore any reservations for this inode
1547 * If the block does not appear in any reservation, then return the
1548 * block number unchanged. If it does appear in the reservation, then
1549 * keep looking through the tree of reservations in order to find the
1550 * first block number which is not reserved.
1553 static u64 gfs2_next_unreserved_block(struct gfs2_rgrpd *rgd, u64 block,
1555 const struct gfs2_inode *ip)
1557 struct gfs2_blkreserv *rs;
1561 spin_lock(&rgd->rd_rsspin);
1562 n = rgd->rd_rstree.rb_node;
1564 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
1565 rc = rs_cmp(block, length, rs);
1575 while ((rs_cmp(block, length, rs) == 0) && (&ip->i_res != rs)) {
1576 block = gfs2_rbm_to_block(&rs->rs_rbm) + rs->rs_free;
1580 rs = rb_entry(n, struct gfs2_blkreserv, rs_node);
1584 spin_unlock(&rgd->rd_rsspin);
1589 * gfs2_reservation_check_and_update - Check for reservations during block alloc
1590 * @rbm: The current position in the resource group
1591 * @ip: The inode for which we are searching for blocks
1592 * @minext: The minimum extent length
1593 * @maxext: A pointer to the maximum extent structure
1595 * This checks the current position in the rgrp to see whether there is
1596 * a reservation covering this block. If not then this function is a
1597 * no-op. If there is, then the position is moved to the end of the
1598 * contiguous reservation(s) so that we are pointing at the first
1599 * non-reserved block.
1601 * Returns: 0 if no reservation, 1 if @rbm has changed, otherwise an error
1604 static int gfs2_reservation_check_and_update(struct gfs2_rbm *rbm,
1605 const struct gfs2_inode *ip,
1607 struct gfs2_extent *maxext)
1609 u64 block = gfs2_rbm_to_block(rbm);
1615 * If we have a minimum extent length, then skip over any extent
1616 * which is less than the min extent length in size.
1619 extlen = gfs2_free_extlen(rbm, minext);
1620 if (extlen <= maxext->len)
1625 * Check the extent which has been found against the reservations
1626 * and skip if parts of it are already reserved
1628 nblock = gfs2_next_unreserved_block(rbm->rgd, block, extlen, ip);
1629 if (nblock == block) {
1630 if (!minext || extlen >= minext)
1633 if (extlen > maxext->len) {
1634 maxext->len = extlen;
1638 nblock = block + extlen;
1640 ret = gfs2_rbm_from_block(rbm, nblock);
1647 * gfs2_rbm_find - Look for blocks of a particular state
1648 * @rbm: Value/result starting position and final position
1649 * @state: The state which we want to find
1650 * @minext: Pointer to the requested extent length (NULL for a single block)
1651 * This is updated to be the actual reservation size.
1652 * @ip: If set, check for reservations
1653 * @nowrap: Stop looking at the end of the rgrp, rather than wrapping
1654 * around until we've reached the starting point.
1657 * - If looking for free blocks, we set GBF_FULL on each bitmap which
1658 * has no free blocks in it.
1659 * - If looking for free blocks, we set rd_extfail_pt on each rgrp which
1660 * has come up short on a free block search.
1662 * Returns: 0 on success, -ENOSPC if there is no block of the requested state
1665 static int gfs2_rbm_find(struct gfs2_rbm *rbm, u8 state, u32 *minext,
1666 const struct gfs2_inode *ip, bool nowrap)
1668 struct buffer_head *bh;
1671 int first_bii = rbm->bii;
1672 u32 first_offset = rbm->offset;
1676 int iters = rbm->rgd->rd_length;
1678 struct gfs2_bitmap *bi;
1679 struct gfs2_extent maxext = { .rbm.rgd = rbm->rgd, };
1681 /* If we are not starting at the beginning of a bitmap, then we
1682 * need to add one to the bitmap count to ensure that we search
1683 * the starting bitmap twice.
1685 if (rbm->offset != 0)
1690 if ((ip == NULL || !gfs2_rs_active(&ip->i_res)) &&
1691 test_bit(GBF_FULL, &bi->bi_flags) &&
1692 (state == GFS2_BLKST_FREE))
1696 buffer = bh->b_data + bi->bi_offset;
1697 WARN_ON(!buffer_uptodate(bh));
1698 if (state != GFS2_BLKST_UNLINKED && bi->bi_clone)
1699 buffer = bi->bi_clone + bi->bi_offset;
1700 initial_offset = rbm->offset;
1701 offset = gfs2_bitfit(buffer, bi->bi_len, rbm->offset, state);
1702 if (offset == BFITNOENT)
1704 rbm->offset = offset;
1708 initial_bii = rbm->bii;
1709 ret = gfs2_reservation_check_and_update(rbm, ip,
1710 minext ? *minext : 0,
1715 n += (rbm->bii - initial_bii);
1718 if (ret == -E2BIG) {
1721 n += (rbm->bii - initial_bii);
1722 goto res_covered_end_of_rgrp;
1726 bitmap_full: /* Mark bitmap as full and fall through */
1727 if ((state == GFS2_BLKST_FREE) && initial_offset == 0)
1728 set_bit(GBF_FULL, &bi->bi_flags);
1730 next_bitmap: /* Find next bitmap in the rgrp */
1733 if (rbm->bii == rbm->rgd->rd_length)
1735 res_covered_end_of_rgrp:
1736 if ((rbm->bii == 0) && nowrap)
1744 if (minext == NULL || state != GFS2_BLKST_FREE)
1747 /* If the extent was too small, and it's smaller than the smallest
1748 to have failed before, remember for future reference that it's
1749 useless to search this rgrp again for this amount or more. */
1750 if ((first_offset == 0) && (first_bii == 0) &&
1751 (*minext < rbm->rgd->rd_extfail_pt))
1752 rbm->rgd->rd_extfail_pt = *minext;
1754 /* If the maximum extent we found is big enough to fulfill the
1755 minimum requirements, use it anyway. */
1758 *minext = maxext.len;
1766 * try_rgrp_unlink - Look for any unlinked, allocated, but unused inodes
1768 * @last_unlinked: block address of the last dinode we unlinked
1769 * @skip: block address we should explicitly not unlink
1771 * Returns: 0 if no error
1772 * The inode, if one has been found, in inode.
1775 static void try_rgrp_unlink(struct gfs2_rgrpd *rgd, u64 *last_unlinked, u64 skip)
1778 struct gfs2_sbd *sdp = rgd->rd_sbd;
1779 struct gfs2_glock *gl;
1780 struct gfs2_inode *ip;
1783 struct gfs2_rbm rbm = { .rgd = rgd, .bii = 0, .offset = 0 };
1786 down_write(&sdp->sd_log_flush_lock);
1787 error = gfs2_rbm_find(&rbm, GFS2_BLKST_UNLINKED, NULL, NULL,
1789 up_write(&sdp->sd_log_flush_lock);
1790 if (error == -ENOSPC)
1792 if (WARN_ON_ONCE(error))
1795 block = gfs2_rbm_to_block(&rbm);
1796 if (gfs2_rbm_from_block(&rbm, block + 1))
1798 if (*last_unlinked != NO_BLOCK && block <= *last_unlinked)
1802 *last_unlinked = block;
1804 error = gfs2_glock_get(sdp, block, &gfs2_iopen_glops, CREATE, &gl);
1808 /* If the inode is already in cache, we can ignore it here
1809 * because the existing inode disposal code will deal with
1810 * it when all refs have gone away. Accessing gl_object like
1811 * this is not safe in general. Here it is ok because we do
1812 * not dereference the pointer, and we only need an approx
1813 * answer to whether it is NULL or not.
1817 if (ip || queue_work(gfs2_delete_workqueue, &gl->gl_delete) == 0)
1822 /* Limit reclaim to sensible number of tasks */
1823 if (found > NR_CPUS)
1827 rgd->rd_flags &= ~GFS2_RDF_CHECK;
1832 * gfs2_rgrp_congested - Use stats to figure out whether an rgrp is congested
1833 * @rgd: The rgrp in question
1834 * @loops: An indication of how picky we can be (0=very, 1=less so)
1836 * This function uses the recently added glock statistics in order to
1837 * figure out whether a parciular resource group is suffering from
1838 * contention from multiple nodes. This is done purely on the basis
1839 * of timings, since this is the only data we have to work with and
1840 * our aim here is to reject a resource group which is highly contended
1841 * but (very important) not to do this too often in order to ensure that
1842 * we do not land up introducing fragmentation by changing resource
1843 * groups when not actually required.
1845 * The calculation is fairly simple, we want to know whether the SRTTB
1846 * (i.e. smoothed round trip time for blocking operations) to acquire
1847 * the lock for this rgrp's glock is significantly greater than the
1848 * time taken for resource groups on average. We introduce a margin in
1849 * the form of the variable @var which is computed as the sum of the two
1850 * respective variences, and multiplied by a factor depending on @loops
1851 * and whether we have a lot of data to base the decision on. This is
1852 * then tested against the square difference of the means in order to
1853 * decide whether the result is statistically significant or not.
1855 * Returns: A boolean verdict on the congestion status
1858 static bool gfs2_rgrp_congested(const struct gfs2_rgrpd *rgd, int loops)
1860 const struct gfs2_glock *gl = rgd->rd_gl;
1861 const struct gfs2_sbd *sdp = gl->gl_name.ln_sbd;
1862 struct gfs2_lkstats *st;
1863 u64 r_dcount, l_dcount;
1864 u64 l_srttb, a_srttb = 0;
1868 int cpu, nonzero = 0;
1871 for_each_present_cpu(cpu) {
1872 st = &per_cpu_ptr(sdp->sd_lkstats, cpu)->lkstats[LM_TYPE_RGRP];
1873 if (st->stats[GFS2_LKS_SRTTB]) {
1874 a_srttb += st->stats[GFS2_LKS_SRTTB];
1878 st = &this_cpu_ptr(sdp->sd_lkstats)->lkstats[LM_TYPE_RGRP];
1880 do_div(a_srttb, nonzero);
1881 r_dcount = st->stats[GFS2_LKS_DCOUNT];
1882 var = st->stats[GFS2_LKS_SRTTVARB] +
1883 gl->gl_stats.stats[GFS2_LKS_SRTTVARB];
1886 l_srttb = gl->gl_stats.stats[GFS2_LKS_SRTTB];
1887 l_dcount = gl->gl_stats.stats[GFS2_LKS_DCOUNT];
1889 if ((l_dcount < 1) || (r_dcount < 1) || (a_srttb == 0))
1892 srttb_diff = a_srttb - l_srttb;
1893 sqr_diff = srttb_diff * srttb_diff;
1896 if (l_dcount < 8 || r_dcount < 8)
1901 return ((srttb_diff < 0) && (sqr_diff > var));
1905 * gfs2_rgrp_used_recently
1906 * @rs: The block reservation with the rgrp to test
1907 * @msecs: The time limit in milliseconds
1909 * Returns: True if the rgrp glock has been used within the time limit
1911 static bool gfs2_rgrp_used_recently(const struct gfs2_blkreserv *rs,
1916 tdiff = ktime_to_ns(ktime_sub(ktime_get_real(),
1917 rs->rs_rbm.rgd->rd_gl->gl_dstamp));
1919 return tdiff > (msecs * 1000 * 1000);
1922 static u32 gfs2_orlov_skip(const struct gfs2_inode *ip)
1924 const struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1927 get_random_bytes(&skip, sizeof(skip));
1928 return skip % sdp->sd_rgrps;
1931 static bool gfs2_select_rgrp(struct gfs2_rgrpd **pos, const struct gfs2_rgrpd *begin)
1933 struct gfs2_rgrpd *rgd = *pos;
1934 struct gfs2_sbd *sdp = rgd->rd_sbd;
1936 rgd = gfs2_rgrpd_get_next(rgd);
1938 rgd = gfs2_rgrpd_get_first(sdp);
1940 if (rgd != begin) /* If we didn't wrap */
1946 * fast_to_acquire - determine if a resource group will be fast to acquire
1948 * If this is one of our preferred rgrps, it should be quicker to acquire,
1949 * because we tried to set ourselves up as dlm lock master.
1951 static inline int fast_to_acquire(struct gfs2_rgrpd *rgd)
1953 struct gfs2_glock *gl = rgd->rd_gl;
1955 if (gl->gl_state != LM_ST_UNLOCKED && list_empty(&gl->gl_holders) &&
1956 !test_bit(GLF_DEMOTE_IN_PROGRESS, &gl->gl_flags) &&
1957 !test_bit(GLF_DEMOTE, &gl->gl_flags))
1959 if (rgd->rd_flags & GFS2_RDF_PREFERRED)
1965 * gfs2_inplace_reserve - Reserve space in the filesystem
1966 * @ip: the inode to reserve space for
1967 * @ap: the allocation parameters
1969 * We try our best to find an rgrp that has at least ap->target blocks
1970 * available. After a couple of passes (loops == 2), the prospects of finding
1971 * such an rgrp diminish. At this stage, we return the first rgrp that has
1972 * atleast ap->min_target blocks available. Either way, we set ap->allowed to
1973 * the number of blocks available in the chosen rgrp.
1975 * Returns: 0 on success,
1976 * -ENOMEM if a suitable rgrp can't be found
1980 int gfs2_inplace_reserve(struct gfs2_inode *ip, struct gfs2_alloc_parms *ap)
1982 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
1983 struct gfs2_rgrpd *begin = NULL;
1984 struct gfs2_blkreserv *rs = &ip->i_res;
1985 int error = 0, rg_locked, flags = 0;
1986 u64 last_unlinked = NO_BLOCK;
1990 if (sdp->sd_args.ar_rgrplvb)
1992 if (gfs2_assert_warn(sdp, ap->target))
1994 if (gfs2_rs_active(rs)) {
1995 begin = rs->rs_rbm.rgd;
1996 } else if (rs->rs_rbm.rgd &&
1997 rgrp_contains_block(rs->rs_rbm.rgd, ip->i_goal)) {
1998 begin = rs->rs_rbm.rgd;
2000 check_and_update_goal(ip);
2001 rs->rs_rbm.rgd = begin = gfs2_blk2rgrpd(sdp, ip->i_goal, 1);
2003 if (S_ISDIR(ip->i_inode.i_mode) && (ap->aflags & GFS2_AF_ORLOV))
2004 skip = gfs2_orlov_skip(ip);
2005 if (rs->rs_rbm.rgd == NULL)
2011 if (!gfs2_glock_is_locked_by_me(rs->rs_rbm.rgd->rd_gl)) {
2015 if (!gfs2_rs_active(rs)) {
2017 !fast_to_acquire(rs->rs_rbm.rgd))
2020 gfs2_rgrp_used_recently(rs, 1000) &&
2021 gfs2_rgrp_congested(rs->rs_rbm.rgd, loops))
2024 error = gfs2_glock_nq_init(rs->rs_rbm.rgd->rd_gl,
2025 LM_ST_EXCLUSIVE, flags,
2027 if (unlikely(error))
2029 if (!gfs2_rs_active(rs) && (loops < 2) &&
2030 gfs2_rgrp_congested(rs->rs_rbm.rgd, loops))
2032 if (sdp->sd_args.ar_rgrplvb) {
2033 error = update_rgrp_lvb(rs->rs_rbm.rgd);
2034 if (unlikely(error)) {
2035 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2041 /* Skip unuseable resource groups */
2042 if ((rs->rs_rbm.rgd->rd_flags & (GFS2_RGF_NOALLOC |
2044 (loops == 0 && ap->target > rs->rs_rbm.rgd->rd_extfail_pt))
2047 if (sdp->sd_args.ar_rgrplvb)
2048 gfs2_rgrp_bh_get(rs->rs_rbm.rgd);
2050 /* Get a reservation if we don't already have one */
2051 if (!gfs2_rs_active(rs))
2052 rg_mblk_search(rs->rs_rbm.rgd, ip, ap);
2054 /* Skip rgrps when we can't get a reservation on first pass */
2055 if (!gfs2_rs_active(rs) && (loops < 1))
2058 /* If rgrp has enough free space, use it */
2059 if (rs->rs_rbm.rgd->rd_free_clone >= ap->target ||
2060 (loops == 2 && ap->min_target &&
2061 rs->rs_rbm.rgd->rd_free_clone >= ap->min_target)) {
2062 ap->allowed = rs->rs_rbm.rgd->rd_free_clone;
2066 /* Check for unlinked inodes which can be reclaimed */
2067 if (rs->rs_rbm.rgd->rd_flags & GFS2_RDF_CHECK)
2068 try_rgrp_unlink(rs->rs_rbm.rgd, &last_unlinked,
2071 /* Drop reservation, if we couldn't use reserved rgrp */
2072 if (gfs2_rs_active(rs))
2073 gfs2_rs_deltree(rs);
2075 /* Unlock rgrp if required */
2077 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2079 /* Find the next rgrp, and continue looking */
2080 if (gfs2_select_rgrp(&rs->rs_rbm.rgd, begin))
2085 /* If we've scanned all the rgrps, but found no free blocks
2086 * then this checks for some less likely conditions before
2090 /* Check that fs hasn't grown if writing to rindex */
2091 if (ip == GFS2_I(sdp->sd_rindex) && !sdp->sd_rindex_uptodate) {
2092 error = gfs2_ri_update(ip);
2096 /* Flushing the log may release space */
2098 gfs2_log_flush(sdp, NULL, GFS2_LOG_HEAD_FLUSH_NORMAL |
2099 GFS2_LFC_INPLACE_RESERVE);
2106 * gfs2_inplace_release - release an inplace reservation
2107 * @ip: the inode the reservation was taken out on
2109 * Release a reservation made by gfs2_inplace_reserve().
2112 void gfs2_inplace_release(struct gfs2_inode *ip)
2114 struct gfs2_blkreserv *rs = &ip->i_res;
2116 if (gfs2_holder_initialized(&rs->rs_rgd_gh))
2117 gfs2_glock_dq_uninit(&rs->rs_rgd_gh);
2121 * gfs2_get_block_type - Check a block in a RG is of given type
2122 * @rgd: the resource group holding the block
2123 * @block: the block number
2125 * Returns: The block type (GFS2_BLKST_*)
2128 static unsigned char gfs2_get_block_type(struct gfs2_rgrpd *rgd, u64 block)
2130 struct gfs2_rbm rbm = { .rgd = rgd, };
2133 ret = gfs2_rbm_from_block(&rbm, block);
2134 WARN_ON_ONCE(ret != 0);
2136 return gfs2_testbit(&rbm);
2141 * gfs2_alloc_extent - allocate an extent from a given bitmap
2142 * @rbm: the resource group information
2143 * @dinode: TRUE if the first block we allocate is for a dinode
2144 * @n: The extent length (value/result)
2146 * Add the bitmap buffer to the transaction.
2147 * Set the found bits to @new_state to change block's allocation state.
2149 static void gfs2_alloc_extent(const struct gfs2_rbm *rbm, bool dinode,
2152 struct gfs2_rbm pos = { .rgd = rbm->rgd, };
2153 const unsigned int elen = *n;
2158 block = gfs2_rbm_to_block(rbm);
2159 gfs2_trans_add_meta(rbm->rgd->rd_gl, rbm_bi(rbm)->bi_bh);
2160 gfs2_setbit(rbm, true, dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED);
2163 ret = gfs2_rbm_from_block(&pos, block);
2164 if (ret || gfs2_testbit(&pos) != GFS2_BLKST_FREE)
2166 gfs2_trans_add_meta(pos.rgd->rd_gl, rbm_bi(&pos)->bi_bh);
2167 gfs2_setbit(&pos, true, GFS2_BLKST_USED);
2174 * rgblk_free - Change alloc state of given block(s)
2175 * @sdp: the filesystem
2176 * @bstart: the start of a run of blocks to free
2177 * @blen: the length of the block run (all must lie within ONE RG!)
2178 * @new_state: GFS2_BLKST_XXX the after-allocation block state
2180 * Returns: Resource group containing the block(s)
2183 static struct gfs2_rgrpd *rgblk_free(struct gfs2_sbd *sdp, u64 bstart,
2184 u32 blen, unsigned char new_state)
2186 struct gfs2_rbm rbm;
2187 struct gfs2_bitmap *bi, *bi_prev = NULL;
2189 rbm.rgd = gfs2_blk2rgrpd(sdp, bstart, 1);
2191 if (gfs2_consist(sdp))
2192 fs_err(sdp, "block = %llu\n", (unsigned long long)bstart);
2196 gfs2_rbm_from_block(&rbm, bstart);
2199 if (bi != bi_prev) {
2200 if (!bi->bi_clone) {
2201 bi->bi_clone = kmalloc(bi->bi_bh->b_size,
2202 GFP_NOFS | __GFP_NOFAIL);
2203 memcpy(bi->bi_clone + bi->bi_offset,
2204 bi->bi_bh->b_data + bi->bi_offset,
2207 gfs2_trans_add_meta(rbm.rgd->rd_gl, bi->bi_bh);
2210 gfs2_setbit(&rbm, false, new_state);
2211 gfs2_rbm_incr(&rbm);
2218 * gfs2_rgrp_dump - print out an rgrp
2219 * @seq: The iterator
2220 * @gl: The glock in question
2224 void gfs2_rgrp_dump(struct seq_file *seq, const struct gfs2_glock *gl)
2226 struct gfs2_rgrpd *rgd = gl->gl_object;
2227 struct gfs2_blkreserv *trs;
2228 const struct rb_node *n;
2232 gfs2_print_dbg(seq, " R: n:%llu f:%02x b:%u/%u i:%u r:%u e:%u\n",
2233 (unsigned long long)rgd->rd_addr, rgd->rd_flags,
2234 rgd->rd_free, rgd->rd_free_clone, rgd->rd_dinodes,
2235 rgd->rd_reserved, rgd->rd_extfail_pt);
2236 spin_lock(&rgd->rd_rsspin);
2237 for (n = rb_first(&rgd->rd_rstree); n; n = rb_next(&trs->rs_node)) {
2238 trs = rb_entry(n, struct gfs2_blkreserv, rs_node);
2241 spin_unlock(&rgd->rd_rsspin);
2244 static void gfs2_rgrp_error(struct gfs2_rgrpd *rgd)
2246 struct gfs2_sbd *sdp = rgd->rd_sbd;
2247 fs_warn(sdp, "rgrp %llu has an error, marking it readonly until umount\n",
2248 (unsigned long long)rgd->rd_addr);
2249 fs_warn(sdp, "umount on all nodes and run fsck.gfs2 to fix the error\n");
2250 gfs2_rgrp_dump(NULL, rgd->rd_gl);
2251 rgd->rd_flags |= GFS2_RDF_ERROR;
2255 * gfs2_adjust_reservation - Adjust (or remove) a reservation after allocation
2256 * @ip: The inode we have just allocated blocks for
2257 * @rbm: The start of the allocated blocks
2258 * @len: The extent length
2260 * Adjusts a reservation after an allocation has taken place. If the
2261 * reservation does not match the allocation, or if it is now empty
2262 * then it is removed.
2265 static void gfs2_adjust_reservation(struct gfs2_inode *ip,
2266 const struct gfs2_rbm *rbm, unsigned len)
2268 struct gfs2_blkreserv *rs = &ip->i_res;
2269 struct gfs2_rgrpd *rgd = rbm->rgd;
2274 spin_lock(&rgd->rd_rsspin);
2275 if (gfs2_rs_active(rs)) {
2276 if (gfs2_rbm_eq(&rs->rs_rbm, rbm)) {
2277 block = gfs2_rbm_to_block(rbm);
2278 ret = gfs2_rbm_from_block(&rs->rs_rbm, block + len);
2279 rlen = min(rs->rs_free, len);
2280 rs->rs_free -= rlen;
2281 rgd->rd_reserved -= rlen;
2282 trace_gfs2_rs(rs, TRACE_RS_CLAIM);
2283 if (rs->rs_free && !ret)
2285 /* We used up our block reservation, so we should
2286 reserve more blocks next time. */
2287 atomic_add(RGRP_RSRV_ADDBLKS, &rs->rs_sizehint);
2292 spin_unlock(&rgd->rd_rsspin);
2296 * gfs2_set_alloc_start - Set starting point for block allocation
2297 * @rbm: The rbm which will be set to the required location
2298 * @ip: The gfs2 inode
2299 * @dinode: Flag to say if allocation includes a new inode
2301 * This sets the starting point from the reservation if one is active
2302 * otherwise it falls back to guessing a start point based on the
2303 * inode's goal block or the last allocation point in the rgrp.
2306 static void gfs2_set_alloc_start(struct gfs2_rbm *rbm,
2307 const struct gfs2_inode *ip, bool dinode)
2311 if (gfs2_rs_active(&ip->i_res)) {
2312 *rbm = ip->i_res.rs_rbm;
2316 if (!dinode && rgrp_contains_block(rbm->rgd, ip->i_goal))
2319 goal = rbm->rgd->rd_last_alloc + rbm->rgd->rd_data0;
2321 gfs2_rbm_from_block(rbm, goal);
2325 * gfs2_alloc_blocks - Allocate one or more blocks of data and/or a dinode
2326 * @ip: the inode to allocate the block for
2327 * @bn: Used to return the starting block number
2328 * @nblocks: requested number of blocks/extent length (value/result)
2329 * @dinode: 1 if we're allocating a dinode block, else 0
2330 * @generation: the generation number of the inode
2332 * Returns: 0 or error
2335 int gfs2_alloc_blocks(struct gfs2_inode *ip, u64 *bn, unsigned int *nblocks,
2336 bool dinode, u64 *generation)
2338 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2339 struct buffer_head *dibh;
2340 struct gfs2_rbm rbm = { .rgd = ip->i_res.rs_rbm.rgd, };
2342 u64 block; /* block, within the file system scope */
2345 gfs2_set_alloc_start(&rbm, ip, dinode);
2346 error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, ip, false);
2348 if (error == -ENOSPC) {
2349 gfs2_set_alloc_start(&rbm, ip, dinode);
2350 error = gfs2_rbm_find(&rbm, GFS2_BLKST_FREE, NULL, NULL, false);
2353 /* Since all blocks are reserved in advance, this shouldn't happen */
2355 fs_warn(sdp, "inum=%llu error=%d, nblocks=%u, full=%d fail_pt=%d\n",
2356 (unsigned long long)ip->i_no_addr, error, *nblocks,
2357 test_bit(GBF_FULL, &rbm.rgd->rd_bits->bi_flags),
2358 rbm.rgd->rd_extfail_pt);
2362 gfs2_alloc_extent(&rbm, dinode, nblocks);
2363 block = gfs2_rbm_to_block(&rbm);
2364 rbm.rgd->rd_last_alloc = block - rbm.rgd->rd_data0;
2365 if (gfs2_rs_active(&ip->i_res))
2366 gfs2_adjust_reservation(ip, &rbm, *nblocks);
2372 ip->i_goal = block + ndata - 1;
2373 error = gfs2_meta_inode_buffer(ip, &dibh);
2375 struct gfs2_dinode *di =
2376 (struct gfs2_dinode *)dibh->b_data;
2377 gfs2_trans_add_meta(ip->i_gl, dibh);
2378 di->di_goal_meta = di->di_goal_data =
2379 cpu_to_be64(ip->i_goal);
2383 if (rbm.rgd->rd_free < *nblocks) {
2384 pr_warn("nblocks=%u\n", *nblocks);
2388 rbm.rgd->rd_free -= *nblocks;
2390 rbm.rgd->rd_dinodes++;
2391 *generation = rbm.rgd->rd_igeneration++;
2392 if (*generation == 0)
2393 *generation = rbm.rgd->rd_igeneration++;
2396 gfs2_trans_add_meta(rbm.rgd->rd_gl, rbm.rgd->rd_bits[0].bi_bh);
2397 gfs2_rgrp_out(rbm.rgd, rbm.rgd->rd_bits[0].bi_bh->b_data);
2398 gfs2_rgrp_ondisk2lvb(rbm.rgd->rd_rgl, rbm.rgd->rd_bits[0].bi_bh->b_data);
2400 gfs2_statfs_change(sdp, 0, -(s64)*nblocks, dinode ? 1 : 0);
2402 gfs2_trans_add_unrevoke(sdp, block, *nblocks);
2404 gfs2_quota_change(ip, *nblocks, ip->i_inode.i_uid, ip->i_inode.i_gid);
2406 rbm.rgd->rd_free_clone -= *nblocks;
2407 trace_gfs2_block_alloc(ip, rbm.rgd, block, *nblocks,
2408 dinode ? GFS2_BLKST_DINODE : GFS2_BLKST_USED);
2413 gfs2_rgrp_error(rbm.rgd);
2418 * __gfs2_free_blocks - free a contiguous run of block(s)
2419 * @ip: the inode these blocks are being freed from
2420 * @bstart: first block of a run of contiguous blocks
2421 * @blen: the length of the block run
2422 * @meta: 1 if the blocks represent metadata
2426 void __gfs2_free_blocks(struct gfs2_inode *ip, u64 bstart, u32 blen, int meta)
2428 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2429 struct gfs2_rgrpd *rgd;
2431 rgd = rgblk_free(sdp, bstart, blen, GFS2_BLKST_FREE);
2434 trace_gfs2_block_alloc(ip, rgd, bstart, blen, GFS2_BLKST_FREE);
2435 rgd->rd_free += blen;
2436 rgd->rd_flags &= ~GFS2_RGF_TRIMMED;
2437 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2438 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2439 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2441 /* Directories keep their data in the metadata address space */
2442 if (meta || ip->i_depth)
2443 gfs2_meta_wipe(ip, bstart, blen);
2447 * gfs2_free_meta - free a contiguous run of data block(s)
2448 * @ip: the inode these blocks are being freed from
2449 * @bstart: first block of a run of contiguous blocks
2450 * @blen: the length of the block run
2454 void gfs2_free_meta(struct gfs2_inode *ip, u64 bstart, u32 blen)
2456 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2458 __gfs2_free_blocks(ip, bstart, blen, 1);
2459 gfs2_statfs_change(sdp, 0, +blen, 0);
2460 gfs2_quota_change(ip, -(s64)blen, ip->i_inode.i_uid, ip->i_inode.i_gid);
2463 void gfs2_unlink_di(struct inode *inode)
2465 struct gfs2_inode *ip = GFS2_I(inode);
2466 struct gfs2_sbd *sdp = GFS2_SB(inode);
2467 struct gfs2_rgrpd *rgd;
2468 u64 blkno = ip->i_no_addr;
2470 rgd = rgblk_free(sdp, blkno, 1, GFS2_BLKST_UNLINKED);
2473 trace_gfs2_block_alloc(ip, rgd, blkno, 1, GFS2_BLKST_UNLINKED);
2474 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2475 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2476 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2477 update_rgrp_lvb_unlinked(rgd, 1);
2480 void gfs2_free_di(struct gfs2_rgrpd *rgd, struct gfs2_inode *ip)
2482 struct gfs2_sbd *sdp = rgd->rd_sbd;
2483 struct gfs2_rgrpd *tmp_rgd;
2485 tmp_rgd = rgblk_free(sdp, ip->i_no_addr, 1, GFS2_BLKST_FREE);
2488 gfs2_assert_withdraw(sdp, rgd == tmp_rgd);
2490 if (!rgd->rd_dinodes)
2491 gfs2_consist_rgrpd(rgd);
2495 gfs2_trans_add_meta(rgd->rd_gl, rgd->rd_bits[0].bi_bh);
2496 gfs2_rgrp_out(rgd, rgd->rd_bits[0].bi_bh->b_data);
2497 gfs2_rgrp_ondisk2lvb(rgd->rd_rgl, rgd->rd_bits[0].bi_bh->b_data);
2498 update_rgrp_lvb_unlinked(rgd, -1);
2500 gfs2_statfs_change(sdp, 0, +1, -1);
2501 trace_gfs2_block_alloc(ip, rgd, ip->i_no_addr, 1, GFS2_BLKST_FREE);
2502 gfs2_quota_change(ip, -1, ip->i_inode.i_uid, ip->i_inode.i_gid);
2503 gfs2_meta_wipe(ip, ip->i_no_addr, 1);
2507 * gfs2_check_blk_type - Check the type of a block
2508 * @sdp: The superblock
2509 * @no_addr: The block number to check
2510 * @type: The block type we are looking for
2512 * Returns: 0 if the block type matches the expected type
2513 * -ESTALE if it doesn't match
2514 * or -ve errno if something went wrong while checking
2517 int gfs2_check_blk_type(struct gfs2_sbd *sdp, u64 no_addr, unsigned int type)
2519 struct gfs2_rgrpd *rgd;
2520 struct gfs2_holder rgd_gh;
2521 int error = -EINVAL;
2523 rgd = gfs2_blk2rgrpd(sdp, no_addr, 1);
2527 error = gfs2_glock_nq_init(rgd->rd_gl, LM_ST_SHARED, 0, &rgd_gh);
2531 if (gfs2_get_block_type(rgd, no_addr) != type)
2534 gfs2_glock_dq_uninit(&rgd_gh);
2540 * gfs2_rlist_add - add a RG to a list of RGs
2542 * @rlist: the list of resource groups
2545 * Figure out what RG a block belongs to and add that RG to the list
2547 * FIXME: Don't use NOFAIL
2551 void gfs2_rlist_add(struct gfs2_inode *ip, struct gfs2_rgrp_list *rlist,
2554 struct gfs2_sbd *sdp = GFS2_SB(&ip->i_inode);
2555 struct gfs2_rgrpd *rgd;
2556 struct gfs2_rgrpd **tmp;
2557 unsigned int new_space;
2560 if (gfs2_assert_warn(sdp, !rlist->rl_ghs))
2564 * The resource group last accessed is kept in the last position.
2567 if (rlist->rl_rgrps) {
2568 rgd = rlist->rl_rgd[rlist->rl_rgrps - 1];
2569 if (rgrp_contains_block(rgd, block))
2571 rgd = gfs2_blk2rgrpd(sdp, block, 1);
2573 rgd = ip->i_res.rs_rbm.rgd;
2574 if (!rgd || !rgrp_contains_block(rgd, block))
2575 rgd = gfs2_blk2rgrpd(sdp, block, 1);
2579 fs_err(sdp, "rlist_add: no rgrp for block %llu\n",
2580 (unsigned long long)block);
2584 for (x = 0; x < rlist->rl_rgrps; x++) {
2585 if (rlist->rl_rgd[x] == rgd) {
2586 swap(rlist->rl_rgd[x],
2587 rlist->rl_rgd[rlist->rl_rgrps - 1]);
2592 if (rlist->rl_rgrps == rlist->rl_space) {
2593 new_space = rlist->rl_space + 10;
2595 tmp = kcalloc(new_space, sizeof(struct gfs2_rgrpd *),
2596 GFP_NOFS | __GFP_NOFAIL);
2598 if (rlist->rl_rgd) {
2599 memcpy(tmp, rlist->rl_rgd,
2600 rlist->rl_space * sizeof(struct gfs2_rgrpd *));
2601 kfree(rlist->rl_rgd);
2604 rlist->rl_space = new_space;
2605 rlist->rl_rgd = tmp;
2608 rlist->rl_rgd[rlist->rl_rgrps++] = rgd;
2612 * gfs2_rlist_alloc - all RGs have been added to the rlist, now allocate
2613 * and initialize an array of glock holders for them
2614 * @rlist: the list of resource groups
2615 * @state: the lock state to acquire the RG lock in
2617 * FIXME: Don't use NOFAIL
2621 void gfs2_rlist_alloc(struct gfs2_rgrp_list *rlist, unsigned int state)
2625 rlist->rl_ghs = kmalloc_array(rlist->rl_rgrps,
2626 sizeof(struct gfs2_holder),
2627 GFP_NOFS | __GFP_NOFAIL);
2628 for (x = 0; x < rlist->rl_rgrps; x++)
2629 gfs2_holder_init(rlist->rl_rgd[x]->rd_gl,
2635 * gfs2_rlist_free - free a resource group list
2636 * @rlist: the list of resource groups
2640 void gfs2_rlist_free(struct gfs2_rgrp_list *rlist)
2644 kfree(rlist->rl_rgd);
2646 if (rlist->rl_ghs) {
2647 for (x = 0; x < rlist->rl_rgrps; x++)
2648 gfs2_holder_uninit(&rlist->rl_ghs[x]);
2649 kfree(rlist->rl_ghs);
2650 rlist->rl_ghs = NULL;