1 // SPDX-License-Identifier: GPL-2.0
3 #include <linux/slab.h>
7 int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
8 struct page *page, enum btrfs_subpage_type type)
10 struct btrfs_subpage *subpage = NULL;
14 * We have cases like a dummy extent buffer page, which is not mappped
15 * and doesn't need to be locked.
18 ASSERT(PageLocked(page));
19 /* Either not subpage, or the page already has private attached */
20 if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page))
23 ret = btrfs_alloc_subpage(fs_info, &subpage, type);
26 attach_page_private(page, subpage);
30 void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info,
33 struct btrfs_subpage *subpage;
35 /* Either not subpage, or already detached */
36 if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page))
39 subpage = (struct btrfs_subpage *)detach_page_private(page);
41 btrfs_free_subpage(subpage);
44 int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
45 struct btrfs_subpage **ret,
46 enum btrfs_subpage_type type)
48 if (fs_info->sectorsize == PAGE_SIZE)
51 *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS);
54 spin_lock_init(&(*ret)->lock);
55 if (type == BTRFS_SUBPAGE_METADATA)
56 atomic_set(&(*ret)->eb_refs, 0);
58 atomic_set(&(*ret)->readers, 0);
62 void btrfs_free_subpage(struct btrfs_subpage *subpage)
68 * Increase the eb_refs of current subpage.
70 * This is important for eb allocation, to prevent race with last eb freeing
72 * With the eb_refs increased before the eb inserted into radix tree,
73 * detach_extent_buffer_page() won't detach the page private while we're still
74 * allocating the extent buffer.
76 void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
79 struct btrfs_subpage *subpage;
81 if (fs_info->sectorsize == PAGE_SIZE)
84 ASSERT(PagePrivate(page) && page->mapping);
85 lockdep_assert_held(&page->mapping->private_lock);
87 subpage = (struct btrfs_subpage *)page->private;
88 atomic_inc(&subpage->eb_refs);
91 void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
94 struct btrfs_subpage *subpage;
96 if (fs_info->sectorsize == PAGE_SIZE)
99 ASSERT(PagePrivate(page) && page->mapping);
100 lockdep_assert_held(&page->mapping->private_lock);
102 subpage = (struct btrfs_subpage *)page->private;
103 ASSERT(atomic_read(&subpage->eb_refs));
104 atomic_dec(&subpage->eb_refs);
107 static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
108 struct page *page, u64 start, u32 len)
111 ASSERT(PagePrivate(page) && page->private);
112 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
113 IS_ALIGNED(len, fs_info->sectorsize));
115 * The range check only works for mapped page, we can still have
116 * unmapped page like dummy extent buffer pages.
119 ASSERT(page_offset(page) <= start &&
120 start + len <= page_offset(page) + PAGE_SIZE);
123 void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
124 struct page *page, u64 start, u32 len)
126 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
127 const int nbits = len >> fs_info->sectorsize_bits;
130 btrfs_subpage_assert(fs_info, page, start, len);
132 ret = atomic_add_return(nbits, &subpage->readers);
133 ASSERT(ret == nbits);
136 void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
137 struct page *page, u64 start, u32 len)
139 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
140 const int nbits = len >> fs_info->sectorsize_bits;
142 btrfs_subpage_assert(fs_info, page, start, len);
143 ASSERT(atomic_read(&subpage->readers) >= nbits);
144 if (atomic_sub_and_test(nbits, &subpage->readers))
149 * Convert the [start, start + len) range into a u16 bitmap
151 * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0.
153 static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info,
154 struct page *page, u64 start, u32 len)
156 const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits;
157 const int nbits = len >> fs_info->sectorsize_bits;
159 btrfs_subpage_assert(fs_info, page, start, len);
162 * Here nbits can be 16, thus can go beyond u16 range. We make the
163 * first left shift to be calculate in unsigned long (at least u32),
164 * then truncate the result to u16.
166 return (u16)(((1UL << nbits) - 1) << bit_start);
169 void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
170 struct page *page, u64 start, u32 len)
172 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
173 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
176 spin_lock_irqsave(&subpage->lock, flags);
177 subpage->uptodate_bitmap |= tmp;
178 if (subpage->uptodate_bitmap == U16_MAX)
179 SetPageUptodate(page);
180 spin_unlock_irqrestore(&subpage->lock, flags);
183 void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
184 struct page *page, u64 start, u32 len)
186 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
187 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
190 spin_lock_irqsave(&subpage->lock, flags);
191 subpage->uptodate_bitmap &= ~tmp;
192 ClearPageUptodate(page);
193 spin_unlock_irqrestore(&subpage->lock, flags);
196 void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
197 struct page *page, u64 start, u32 len)
199 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
200 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
203 spin_lock_irqsave(&subpage->lock, flags);
204 subpage->error_bitmap |= tmp;
206 spin_unlock_irqrestore(&subpage->lock, flags);
209 void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
210 struct page *page, u64 start, u32 len)
212 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
213 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
216 spin_lock_irqsave(&subpage->lock, flags);
217 subpage->error_bitmap &= ~tmp;
218 if (subpage->error_bitmap == 0)
219 ClearPageError(page);
220 spin_unlock_irqrestore(&subpage->lock, flags);
224 * Unlike set/clear which is dependent on each page status, for test all bits
225 * are tested in the same way.
227 #define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \
228 bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \
229 struct page *page, u64 start, u32 len) \
231 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
232 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \
233 unsigned long flags; \
236 spin_lock_irqsave(&subpage->lock, flags); \
237 ret = ((subpage->name##_bitmap & tmp) == tmp); \
238 spin_unlock_irqrestore(&subpage->lock, flags); \
241 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
242 IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error);
245 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
246 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
247 * back to regular sectorsize branch.
249 #define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \
251 void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \
252 struct page *page, u64 start, u32 len) \
254 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
255 set_page_func(page); \
258 btrfs_subpage_set_##name(fs_info, page, start, len); \
260 void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \
261 struct page *page, u64 start, u32 len) \
263 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
264 clear_page_func(page); \
267 btrfs_subpage_clear_##name(fs_info, page, start, len); \
269 bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \
270 struct page *page, u64 start, u32 len) \
272 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \
273 return test_page_func(page); \
274 return btrfs_subpage_test_##name(fs_info, page, start, len); \
276 IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate,
278 IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError);