1 // SPDX-License-Identifier: MIT
3 * Copyright © 2019 Intel Corporation
6 #include <linux/kmemleak.h>
7 #include <linux/slab.h>
9 #include "i915_buddy.h"
12 #include "i915_globals.h"
13 #include "i915_utils.h"
15 static struct i915_global_block {
16 struct i915_global base;
17 struct kmem_cache *slab_blocks;
20 static void i915_global_buddy_shrink(void)
22 kmem_cache_shrink(global.slab_blocks);
25 static void i915_global_buddy_exit(void)
27 kmem_cache_destroy(global.slab_blocks);
30 static struct i915_global_block global = { {
31 .shrink = i915_global_buddy_shrink,
32 .exit = i915_global_buddy_exit,
35 int __init i915_global_buddy_init(void)
37 global.slab_blocks = KMEM_CACHE(i915_buddy_block, SLAB_HWCACHE_ALIGN);
38 if (!global.slab_blocks)
41 i915_global_register(&global.base);
45 static struct i915_buddy_block *i915_block_alloc(struct i915_buddy_block *parent,
49 struct i915_buddy_block *block;
51 GEM_BUG_ON(order > I915_BUDDY_MAX_ORDER);
53 block = kmem_cache_zalloc(global.slab_blocks, GFP_KERNEL);
57 block->header = offset;
58 block->header |= order;
59 block->parent = parent;
61 GEM_BUG_ON(block->header & I915_BUDDY_HEADER_UNUSED);
65 static void i915_block_free(struct i915_buddy_block *block)
67 kmem_cache_free(global.slab_blocks, block);
70 static void mark_allocated(struct i915_buddy_block *block)
72 block->header &= ~I915_BUDDY_HEADER_STATE;
73 block->header |= I915_BUDDY_ALLOCATED;
75 list_del(&block->link);
78 static void mark_free(struct i915_buddy_mm *mm,
79 struct i915_buddy_block *block)
81 block->header &= ~I915_BUDDY_HEADER_STATE;
82 block->header |= I915_BUDDY_FREE;
84 list_add(&block->link,
85 &mm->free_list[i915_buddy_block_order(block)]);
88 static void mark_split(struct i915_buddy_block *block)
90 block->header &= ~I915_BUDDY_HEADER_STATE;
91 block->header |= I915_BUDDY_SPLIT;
93 list_del(&block->link);
96 int i915_buddy_init(struct i915_buddy_mm *mm, u64 size, u64 chunk_size)
101 if (size < chunk_size)
104 if (chunk_size < PAGE_SIZE)
107 if (!is_power_of_2(chunk_size))
110 size = round_down(size, chunk_size);
113 mm->chunk_size = chunk_size;
114 mm->max_order = ilog2(size) - ilog2(chunk_size);
116 GEM_BUG_ON(mm->max_order > I915_BUDDY_MAX_ORDER);
118 mm->free_list = kmalloc_array(mm->max_order + 1,
119 sizeof(struct list_head),
124 for (i = 0; i <= mm->max_order; ++i)
125 INIT_LIST_HEAD(&mm->free_list[i]);
127 mm->n_roots = hweight64(size);
129 mm->roots = kmalloc_array(mm->n_roots,
130 sizeof(struct i915_buddy_block *),
139 * Split into power-of-two blocks, in case we are given a size that is
140 * not itself a power-of-two.
143 struct i915_buddy_block *root;
147 root_size = rounddown_pow_of_two(size);
148 order = ilog2(root_size) - ilog2(chunk_size);
150 root = i915_block_alloc(NULL, order, offset);
156 GEM_BUG_ON(i > mm->max_order);
157 GEM_BUG_ON(i915_buddy_block_size(mm, root) < chunk_size);
170 i915_block_free(mm->roots[i]);
173 kfree(mm->free_list);
177 void i915_buddy_fini(struct i915_buddy_mm *mm)
181 for (i = 0; i < mm->n_roots; ++i) {
182 GEM_WARN_ON(!i915_buddy_block_is_free(mm->roots[i]));
183 i915_block_free(mm->roots[i]);
187 kfree(mm->free_list);
190 static int split_block(struct i915_buddy_mm *mm,
191 struct i915_buddy_block *block)
193 unsigned int block_order = i915_buddy_block_order(block) - 1;
194 u64 offset = i915_buddy_block_offset(block);
196 GEM_BUG_ON(!i915_buddy_block_is_free(block));
197 GEM_BUG_ON(!i915_buddy_block_order(block));
199 block->left = i915_block_alloc(block, block_order, offset);
203 block->right = i915_block_alloc(block, block_order,
204 offset + (mm->chunk_size << block_order));
206 i915_block_free(block->left);
210 mark_free(mm, block->left);
211 mark_free(mm, block->right);
218 static struct i915_buddy_block *
219 get_buddy(struct i915_buddy_block *block)
221 struct i915_buddy_block *parent;
223 parent = block->parent;
227 if (parent->left == block)
228 return parent->right;
233 static void __i915_buddy_free(struct i915_buddy_mm *mm,
234 struct i915_buddy_block *block)
236 struct i915_buddy_block *parent;
238 while ((parent = block->parent)) {
239 struct i915_buddy_block *buddy;
241 buddy = get_buddy(block);
243 if (!i915_buddy_block_is_free(buddy))
246 list_del(&buddy->link);
248 i915_block_free(block);
249 i915_block_free(buddy);
254 mark_free(mm, block);
257 void i915_buddy_free(struct i915_buddy_mm *mm,
258 struct i915_buddy_block *block)
260 GEM_BUG_ON(!i915_buddy_block_is_allocated(block));
261 __i915_buddy_free(mm, block);
264 void i915_buddy_free_list(struct i915_buddy_mm *mm, struct list_head *objects)
266 struct i915_buddy_block *block, *on;
268 list_for_each_entry_safe(block, on, objects, link) {
269 i915_buddy_free(mm, block);
272 INIT_LIST_HEAD(objects);
276 * Allocate power-of-two block. The order value here translates to:
278 * 0 = 2^0 * mm->chunk_size
279 * 1 = 2^1 * mm->chunk_size
280 * 2 = 2^2 * mm->chunk_size
283 struct i915_buddy_block *
284 i915_buddy_alloc(struct i915_buddy_mm *mm, unsigned int order)
286 struct i915_buddy_block *block = NULL;
290 for (i = order; i <= mm->max_order; ++i) {
291 block = list_first_entry_or_null(&mm->free_list[i],
292 struct i915_buddy_block,
299 return ERR_PTR(-ENOSPC);
301 GEM_BUG_ON(!i915_buddy_block_is_free(block));
304 err = split_block(mm, block);
313 mark_allocated(block);
314 kmemleak_update_trace(block);
319 __i915_buddy_free(mm, block);
323 static inline bool overlaps(u64 s1, u64 e1, u64 s2, u64 e2)
325 return s1 <= e2 && e1 >= s2;
328 static inline bool contains(u64 s1, u64 e1, u64 s2, u64 e2)
330 return s1 <= s2 && e1 >= e2;
334 * Allocate range. Note that it's safe to chain together multiple alloc_ranges
335 * with the same blocks list.
337 * Intended for pre-allocating portions of the address space, for example to
338 * reserve a block for the initial framebuffer or similar, hence the expectation
339 * here is that i915_buddy_alloc() is still the main vehicle for
340 * allocations, so if that's not the case then the drm_mm range allocator is
341 * probably a much better fit, and so you should probably go use that instead.
343 int i915_buddy_alloc_range(struct i915_buddy_mm *mm,
344 struct list_head *blocks,
347 struct i915_buddy_block *block;
348 struct i915_buddy_block *buddy;
349 LIST_HEAD(allocated);
355 if (size < mm->chunk_size)
358 if (!IS_ALIGNED(size | start, mm->chunk_size))
361 if (range_overflows(start, size, mm->size))
364 for (i = 0; i < mm->n_roots; ++i)
365 list_add_tail(&mm->roots[i]->tmp_link, &dfs);
367 end = start + size - 1;
373 block = list_first_entry_or_null(&dfs,
374 struct i915_buddy_block,
379 list_del(&block->tmp_link);
381 block_start = i915_buddy_block_offset(block);
382 block_end = block_start + i915_buddy_block_size(mm, block) - 1;
384 if (!overlaps(start, end, block_start, block_end))
387 if (i915_buddy_block_is_allocated(block)) {
392 if (contains(start, end, block_start, block_end)) {
393 if (!i915_buddy_block_is_free(block)) {
398 mark_allocated(block);
399 list_add_tail(&block->link, &allocated);
403 if (!i915_buddy_block_is_split(block)) {
404 err = split_block(mm, block);
409 list_add(&block->right->tmp_link, &dfs);
410 list_add(&block->left->tmp_link, &dfs);
413 list_splice_tail(&allocated, blocks);
418 * We really don't want to leave around a bunch of split blocks, since
419 * bigger is better, so make sure we merge everything back before we
420 * free the allocated blocks.
422 buddy = get_buddy(block);
424 (i915_buddy_block_is_free(block) &&
425 i915_buddy_block_is_free(buddy)))
426 __i915_buddy_free(mm, block);
429 i915_buddy_free_list(mm, &allocated);
433 #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
434 #include "selftests/i915_buddy.c"