1 /* SPDX-License-Identifier: GPL-2.0 OR MIT */
2 /**************************************************************************
4 * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA
7 * Permission is hereby granted, free of charge, to any person obtaining a
8 * copy of this software and associated documentation files (the
9 * "Software"), to deal in the Software without restriction, including
10 * without limitation the rights to use, copy, modify, merge, publish,
11 * distribute, sub license, and/or sell copies of the Software, and to
12 * permit persons to whom the Software is furnished to do so, subject to
13 * the following conditions:
15 * The above copyright notice and this permission notice (including the
16 * next paragraph) shall be included in all copies or substantial portions
19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
27 **************************************************************************/
29 #define pr_fmt(fmt) "[TTM] " fmt
31 #include <drm/ttm/ttm_memory.h>
32 #include <linux/spinlock.h>
33 #include <linux/sched.h>
34 #include <linux/wait.h>
36 #include <linux/module.h>
37 #include <linux/slab.h>
38 #include <linux/swap.h>
39 #include <drm/ttm/ttm_pool.h>
40 #include <drm/ttm/ttm_tt.h>
42 #include "ttm_module.h"
44 #define TTM_MEMORY_ALLOC_RETRIES 4
46 struct ttm_mem_global ttm_mem_glob;
47 EXPORT_SYMBOL(ttm_mem_glob);
51 struct ttm_mem_global *glob;
60 static struct attribute ttm_mem_sys = {
61 .name = "zone_memory",
64 static struct attribute ttm_mem_emer = {
65 .name = "emergency_memory",
66 .mode = S_IRUGO | S_IWUSR
68 static struct attribute ttm_mem_max = {
69 .name = "available_memory",
70 .mode = S_IRUGO | S_IWUSR
72 static struct attribute ttm_mem_swap = {
74 .mode = S_IRUGO | S_IWUSR
76 static struct attribute ttm_mem_used = {
77 .name = "used_memory",
81 static void ttm_mem_zone_kobj_release(struct kobject *kobj)
83 struct ttm_mem_zone *zone =
84 container_of(kobj, struct ttm_mem_zone, kobj);
86 pr_info("Zone %7s: Used memory at exit: %llu KiB\n",
87 zone->name, (unsigned long long)zone->used_mem >> 10);
91 static ssize_t ttm_mem_zone_show(struct kobject *kobj,
92 struct attribute *attr,
95 struct ttm_mem_zone *zone =
96 container_of(kobj, struct ttm_mem_zone, kobj);
99 spin_lock(&zone->glob->lock);
100 if (attr == &ttm_mem_sys)
101 val = zone->zone_mem;
102 else if (attr == &ttm_mem_emer)
103 val = zone->emer_mem;
104 else if (attr == &ttm_mem_max)
106 else if (attr == &ttm_mem_swap)
107 val = zone->swap_limit;
108 else if (attr == &ttm_mem_used)
109 val = zone->used_mem;
110 spin_unlock(&zone->glob->lock);
112 return snprintf(buffer, PAGE_SIZE, "%llu\n",
113 (unsigned long long) val >> 10);
116 static void ttm_check_swapping(struct ttm_mem_global *glob);
118 static ssize_t ttm_mem_zone_store(struct kobject *kobj,
119 struct attribute *attr,
123 struct ttm_mem_zone *zone =
124 container_of(kobj, struct ttm_mem_zone, kobj);
129 chars = sscanf(buffer, "%lu", &val);
136 spin_lock(&zone->glob->lock);
137 if (val64 > zone->zone_mem)
138 val64 = zone->zone_mem;
139 if (attr == &ttm_mem_emer) {
140 zone->emer_mem = val64;
141 if (zone->max_mem > val64)
142 zone->max_mem = val64;
143 } else if (attr == &ttm_mem_max) {
144 zone->max_mem = val64;
145 if (zone->emer_mem < val64)
146 zone->emer_mem = val64;
147 } else if (attr == &ttm_mem_swap)
148 zone->swap_limit = val64;
149 spin_unlock(&zone->glob->lock);
151 ttm_check_swapping(zone->glob);
156 static struct attribute *ttm_mem_zone_attrs[] = {
165 static const struct sysfs_ops ttm_mem_zone_ops = {
166 .show = &ttm_mem_zone_show,
167 .store = &ttm_mem_zone_store
170 static struct kobj_type ttm_mem_zone_kobj_type = {
171 .release = &ttm_mem_zone_kobj_release,
172 .sysfs_ops = &ttm_mem_zone_ops,
173 .default_attrs = ttm_mem_zone_attrs,
176 static struct attribute ttm_mem_global_lower_mem_limit = {
177 .name = "lower_mem_limit",
178 .mode = S_IRUGO | S_IWUSR
181 static ssize_t ttm_mem_global_show(struct kobject *kobj,
182 struct attribute *attr,
185 struct ttm_mem_global *glob =
186 container_of(kobj, struct ttm_mem_global, kobj);
189 spin_lock(&glob->lock);
190 val = glob->lower_mem_limit;
191 spin_unlock(&glob->lock);
192 /* convert from number of pages to KB */
193 val <<= (PAGE_SHIFT - 10);
194 return snprintf(buffer, PAGE_SIZE, "%llu\n",
195 (unsigned long long) val);
198 static ssize_t ttm_mem_global_store(struct kobject *kobj,
199 struct attribute *attr,
206 struct ttm_mem_global *glob =
207 container_of(kobj, struct ttm_mem_global, kobj);
209 chars = sscanf(buffer, "%lu", &val);
214 /* convert from KB to number of pages */
215 val64 >>= (PAGE_SHIFT - 10);
217 spin_lock(&glob->lock);
218 glob->lower_mem_limit = val64;
219 spin_unlock(&glob->lock);
224 static struct attribute *ttm_mem_global_attrs[] = {
225 &ttm_mem_global_lower_mem_limit,
229 static const struct sysfs_ops ttm_mem_global_ops = {
230 .show = &ttm_mem_global_show,
231 .store = &ttm_mem_global_store,
234 static struct kobj_type ttm_mem_glob_kobj_type = {
235 .sysfs_ops = &ttm_mem_global_ops,
236 .default_attrs = ttm_mem_global_attrs,
239 static bool ttm_zones_above_swap_target(struct ttm_mem_global *glob,
240 bool from_wq, uint64_t extra)
243 struct ttm_mem_zone *zone;
246 for (i = 0; i < glob->num_zones; ++i) {
247 zone = glob->zones[i];
250 target = zone->swap_limit;
251 else if (capable(CAP_SYS_ADMIN))
252 target = zone->emer_mem;
254 target = zone->max_mem;
256 target = (extra > target) ? 0ULL : target;
258 if (zone->used_mem > target)
265 * At this point we only support a single shrink callback.
266 * Extend this if needed, perhaps using a linked list of callbacks.
267 * Note that this function is reentrant:
268 * many threads may try to swap out at any given time.
271 static void ttm_shrink(struct ttm_mem_global *glob, bool from_wq,
272 uint64_t extra, struct ttm_operation_ctx *ctx)
276 spin_lock(&glob->lock);
278 while (ttm_zones_above_swap_target(glob, from_wq, extra)) {
279 spin_unlock(&glob->lock);
280 ret = ttm_bo_swapout(ctx, GFP_KERNEL);
281 spin_lock(&glob->lock);
282 if (unlikely(ret < 0))
286 spin_unlock(&glob->lock);
289 static void ttm_shrink_work(struct work_struct *work)
291 struct ttm_operation_ctx ctx = {
292 .interruptible = false,
295 struct ttm_mem_global *glob =
296 container_of(work, struct ttm_mem_global, work);
298 ttm_shrink(glob, true, 0ULL, &ctx);
301 static int ttm_mem_init_kernel_zone(struct ttm_mem_global *glob,
302 const struct sysinfo *si)
304 struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
311 mem = si->totalram - si->totalhigh;
314 zone->name = "kernel";
315 zone->zone_mem = mem;
316 zone->max_mem = mem >> 1;
317 zone->emer_mem = (mem >> 1) + (mem >> 2);
318 zone->swap_limit = zone->max_mem - (mem >> 3);
321 glob->zone_kernel = zone;
322 ret = kobject_init_and_add(
323 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
324 if (unlikely(ret != 0)) {
325 kobject_put(&zone->kobj);
328 glob->zones[glob->num_zones++] = zone;
332 #ifdef CONFIG_HIGHMEM
333 static int ttm_mem_init_highmem_zone(struct ttm_mem_global *glob,
334 const struct sysinfo *si)
336 struct ttm_mem_zone *zone;
340 if (si->totalhigh == 0)
343 zone = kzalloc(sizeof(*zone), GFP_KERNEL);
350 zone->name = "highmem";
351 zone->zone_mem = mem;
352 zone->max_mem = mem >> 1;
353 zone->emer_mem = (mem >> 1) + (mem >> 2);
354 zone->swap_limit = zone->max_mem - (mem >> 3);
357 glob->zone_highmem = zone;
358 ret = kobject_init_and_add(
359 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, "%s",
361 if (unlikely(ret != 0)) {
362 kobject_put(&zone->kobj);
365 glob->zones[glob->num_zones++] = zone;
369 static int ttm_mem_init_dma32_zone(struct ttm_mem_global *glob,
370 const struct sysinfo *si)
372 struct ttm_mem_zone *zone = kzalloc(sizeof(*zone), GFP_KERNEL);
383 * No special dma32 zone needed.
386 if (mem <= ((uint64_t) 1ULL << 32)) {
392 * Limit max dma32 memory to 4GB for now
393 * until we can figure out how big this
397 mem = ((uint64_t) 1ULL << 32);
398 zone->name = "dma32";
399 zone->zone_mem = mem;
400 zone->max_mem = mem >> 1;
401 zone->emer_mem = (mem >> 1) + (mem >> 2);
402 zone->swap_limit = zone->max_mem - (mem >> 3);
405 glob->zone_dma32 = zone;
406 ret = kobject_init_and_add(
407 &zone->kobj, &ttm_mem_zone_kobj_type, &glob->kobj, zone->name);
408 if (unlikely(ret != 0)) {
409 kobject_put(&zone->kobj);
412 glob->zones[glob->num_zones++] = zone;
417 int ttm_mem_global_init(struct ttm_mem_global *glob)
422 struct ttm_mem_zone *zone;
424 spin_lock_init(&glob->lock);
425 glob->swap_queue = create_singlethread_workqueue("ttm_swap");
426 INIT_WORK(&glob->work, ttm_shrink_work);
427 ret = kobject_init_and_add(
428 &glob->kobj, &ttm_mem_glob_kobj_type, ttm_get_kobj(), "memory_accounting");
429 if (unlikely(ret != 0)) {
430 kobject_put(&glob->kobj);
436 /* set it as 0 by default to keep original behavior of OOM */
437 glob->lower_mem_limit = 0;
439 ret = ttm_mem_init_kernel_zone(glob, &si);
440 if (unlikely(ret != 0))
442 #ifdef CONFIG_HIGHMEM
443 ret = ttm_mem_init_highmem_zone(glob, &si);
444 if (unlikely(ret != 0))
447 ret = ttm_mem_init_dma32_zone(glob, &si);
448 if (unlikely(ret != 0))
451 for (i = 0; i < glob->num_zones; ++i) {
452 zone = glob->zones[i];
453 pr_info("Zone %7s: Available graphics memory: %llu KiB\n",
454 zone->name, (unsigned long long)zone->max_mem >> 10);
456 ttm_pool_mgr_init(glob->zone_kernel->max_mem/(2*PAGE_SIZE));
460 ttm_mem_global_release(glob);
464 void ttm_mem_global_release(struct ttm_mem_global *glob)
466 struct ttm_mem_zone *zone;
469 /* let the page allocator first stop the shrink work. */
473 flush_workqueue(glob->swap_queue);
474 destroy_workqueue(glob->swap_queue);
475 glob->swap_queue = NULL;
476 for (i = 0; i < glob->num_zones; ++i) {
477 zone = glob->zones[i];
478 kobject_del(&zone->kobj);
479 kobject_put(&zone->kobj);
481 kobject_del(&glob->kobj);
482 kobject_put(&glob->kobj);
483 memset(glob, 0, sizeof(*glob));
486 static void ttm_check_swapping(struct ttm_mem_global *glob)
488 bool needs_swapping = false;
490 struct ttm_mem_zone *zone;
492 spin_lock(&glob->lock);
493 for (i = 0; i < glob->num_zones; ++i) {
494 zone = glob->zones[i];
495 if (zone->used_mem > zone->swap_limit) {
496 needs_swapping = true;
501 spin_unlock(&glob->lock);
503 if (unlikely(needs_swapping))
504 (void)queue_work(glob->swap_queue, &glob->work);
508 static void ttm_mem_global_free_zone(struct ttm_mem_global *glob,
509 struct ttm_mem_zone *single_zone,
513 struct ttm_mem_zone *zone;
515 spin_lock(&glob->lock);
516 for (i = 0; i < glob->num_zones; ++i) {
517 zone = glob->zones[i];
518 if (single_zone && zone != single_zone)
520 zone->used_mem -= amount;
522 spin_unlock(&glob->lock);
525 void ttm_mem_global_free(struct ttm_mem_global *glob,
528 return ttm_mem_global_free_zone(glob, glob->zone_kernel, amount);
530 EXPORT_SYMBOL(ttm_mem_global_free);
533 * check if the available mem is under lower memory limit
535 * a. if no swap disk at all or free swap space is under swap_mem_limit
536 * but available system mem is bigger than sys_mem_limit, allow TTM
539 * b. if the available system mem is less than sys_mem_limit but free
540 * swap disk is bigger than swap_mem_limit, allow TTM allocation.
543 ttm_check_under_lowerlimit(struct ttm_mem_global *glob,
545 struct ttm_operation_ctx *ctx)
549 /* We allow over commit during suspend */
550 if (ctx->force_alloc)
553 available = get_nr_swap_pages() + si_mem_available();
554 available -= num_pages;
555 if (available < glob->lower_mem_limit)
561 static int ttm_mem_global_reserve(struct ttm_mem_global *glob,
562 struct ttm_mem_zone *single_zone,
563 uint64_t amount, bool reserve)
568 struct ttm_mem_zone *zone;
570 spin_lock(&glob->lock);
571 for (i = 0; i < glob->num_zones; ++i) {
572 zone = glob->zones[i];
573 if (single_zone && zone != single_zone)
576 limit = (capable(CAP_SYS_ADMIN)) ?
577 zone->emer_mem : zone->max_mem;
579 if (zone->used_mem > limit)
584 for (i = 0; i < glob->num_zones; ++i) {
585 zone = glob->zones[i];
586 if (single_zone && zone != single_zone)
588 zone->used_mem += amount;
594 spin_unlock(&glob->lock);
595 ttm_check_swapping(glob);
601 static int ttm_mem_global_alloc_zone(struct ttm_mem_global *glob,
602 struct ttm_mem_zone *single_zone,
604 struct ttm_operation_ctx *ctx)
606 int count = TTM_MEMORY_ALLOC_RETRIES;
608 while (unlikely(ttm_mem_global_reserve(glob,
612 if (ctx->no_wait_gpu)
614 if (unlikely(count-- == 0))
616 ttm_shrink(glob, false, memory + (memory >> 2) + 16, ctx);
622 int ttm_mem_global_alloc(struct ttm_mem_global *glob, uint64_t memory,
623 struct ttm_operation_ctx *ctx)
626 * Normal allocations of kernel memory are registered in
630 return ttm_mem_global_alloc_zone(glob, glob->zone_kernel, memory, ctx);
632 EXPORT_SYMBOL(ttm_mem_global_alloc);
634 int ttm_mem_global_alloc_page(struct ttm_mem_global *glob,
635 struct page *page, uint64_t size,
636 struct ttm_operation_ctx *ctx)
638 struct ttm_mem_zone *zone = NULL;
641 * Page allocations may be registed in a single zone
642 * only if highmem or !dma32.
645 #ifdef CONFIG_HIGHMEM
646 if (PageHighMem(page) && glob->zone_highmem != NULL)
647 zone = glob->zone_highmem;
649 if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
650 zone = glob->zone_kernel;
652 return ttm_mem_global_alloc_zone(glob, zone, size, ctx);
655 void ttm_mem_global_free_page(struct ttm_mem_global *glob, struct page *page,
658 struct ttm_mem_zone *zone = NULL;
660 #ifdef CONFIG_HIGHMEM
661 if (PageHighMem(page) && glob->zone_highmem != NULL)
662 zone = glob->zone_highmem;
664 if (glob->zone_dma32 && page_to_pfn(page) > 0x00100000UL)
665 zone = glob->zone_kernel;
667 ttm_mem_global_free_zone(glob, zone, size);
670 size_t ttm_round_pot(size_t size)
672 if ((size & (size - 1)) == 0)
674 else if (size > PAGE_SIZE)
675 return PAGE_ALIGN(size);
679 while (tmp_size < size)
686 EXPORT_SYMBOL(ttm_round_pot);