1 // SPDX-License-Identifier: GPL-2.0
2 #include <linux/debugfs.h>
4 #include <linux/slab.h>
5 #include <linux/uaccess.h>
6 #include <linux/memblock.h>
7 #include <linux/stacktrace.h>
8 #include <linux/page_owner.h>
9 #include <linux/jump_label.h>
10 #include <linux/migrate.h>
11 #include <linux/stackdepot.h>
12 #include <linux/seq_file.h>
13 #include <linux/memcontrol.h>
14 #include <linux/sched/clock.h>
19 * TODO: teach PAGE_OWNER_STACK_DEPTH (__dump_page_owner and save_stack)
20 * to use off stack temporal storage
22 #define PAGE_OWNER_STACK_DEPTH (16)
26 short last_migrate_reason;
28 depot_stack_handle_t handle;
29 depot_stack_handle_t free_handle;
32 char comm[TASK_COMM_LEN];
37 static bool page_owner_enabled __initdata;
38 DEFINE_STATIC_KEY_FALSE(page_owner_inited);
40 static depot_stack_handle_t dummy_handle;
41 static depot_stack_handle_t failure_handle;
42 static depot_stack_handle_t early_handle;
44 static void init_early_allocated_pages(void);
46 static int __init early_page_owner_param(char *buf)
48 int ret = kstrtobool(buf, &page_owner_enabled);
50 if (page_owner_enabled)
51 stack_depot_want_early_init();
55 early_param("page_owner", early_page_owner_param);
57 static __init bool need_page_owner(void)
59 return page_owner_enabled;
62 static __always_inline depot_stack_handle_t create_dummy_stack(void)
64 unsigned long entries[4];
65 unsigned int nr_entries;
67 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
68 return stack_depot_save(entries, nr_entries, GFP_KERNEL);
71 static noinline void register_dummy_stack(void)
73 dummy_handle = create_dummy_stack();
76 static noinline void register_failure_stack(void)
78 failure_handle = create_dummy_stack();
81 static noinline void register_early_stack(void)
83 early_handle = create_dummy_stack();
86 static __init void init_page_owner(void)
88 if (!page_owner_enabled)
91 register_dummy_stack();
92 register_failure_stack();
93 register_early_stack();
94 static_branch_enable(&page_owner_inited);
95 init_early_allocated_pages();
98 struct page_ext_operations page_owner_ops = {
99 .size = sizeof(struct page_owner),
100 .need = need_page_owner,
101 .init = init_page_owner,
104 static inline struct page_owner *get_page_owner(struct page_ext *page_ext)
106 return (void *)page_ext + page_owner_ops.offset;
109 static noinline depot_stack_handle_t save_stack(gfp_t flags)
111 unsigned long entries[PAGE_OWNER_STACK_DEPTH];
112 depot_stack_handle_t handle;
113 unsigned int nr_entries;
118 * Sometimes page metadata allocation tracking requires more
119 * memory to be allocated:
120 * - when new stack trace is saved to stack depot
121 * - when backtrace itself is calculated (ia64)
123 if (current->in_page_owner)
125 current->in_page_owner = 1;
127 nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
128 handle = stack_depot_save(entries, nr_entries, flags);
130 handle = failure_handle;
132 current->in_page_owner = 0;
136 void __reset_page_owner(struct page *page, unsigned short order)
139 struct page_ext *page_ext;
140 depot_stack_handle_t handle;
141 struct page_owner *page_owner;
142 u64 free_ts_nsec = local_clock();
144 page_ext = lookup_page_ext(page);
145 if (unlikely(!page_ext))
148 handle = save_stack(GFP_NOWAIT | __GFP_NOWARN);
149 for (i = 0; i < (1 << order); i++) {
150 __clear_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
151 page_owner = get_page_owner(page_ext);
152 page_owner->free_handle = handle;
153 page_owner->free_ts_nsec = free_ts_nsec;
154 page_ext = page_ext_next(page_ext);
158 static inline void __set_page_owner_handle(struct page_ext *page_ext,
159 depot_stack_handle_t handle,
160 unsigned short order, gfp_t gfp_mask)
162 struct page_owner *page_owner;
165 for (i = 0; i < (1 << order); i++) {
166 page_owner = get_page_owner(page_ext);
167 page_owner->handle = handle;
168 page_owner->order = order;
169 page_owner->gfp_mask = gfp_mask;
170 page_owner->last_migrate_reason = -1;
171 page_owner->pid = current->pid;
172 page_owner->tgid = current->tgid;
173 page_owner->ts_nsec = local_clock();
174 strscpy(page_owner->comm, current->comm,
175 sizeof(page_owner->comm));
176 __set_bit(PAGE_EXT_OWNER, &page_ext->flags);
177 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags);
179 page_ext = page_ext_next(page_ext);
183 noinline void __set_page_owner(struct page *page, unsigned short order,
186 struct page_ext *page_ext = lookup_page_ext(page);
187 depot_stack_handle_t handle;
189 if (unlikely(!page_ext))
192 handle = save_stack(gfp_mask);
193 __set_page_owner_handle(page_ext, handle, order, gfp_mask);
196 void __set_page_owner_migrate_reason(struct page *page, int reason)
198 struct page_ext *page_ext = lookup_page_ext(page);
199 struct page_owner *page_owner;
201 if (unlikely(!page_ext))
204 page_owner = get_page_owner(page_ext);
205 page_owner->last_migrate_reason = reason;
208 void __split_page_owner(struct page *page, unsigned int nr)
211 struct page_ext *page_ext = lookup_page_ext(page);
212 struct page_owner *page_owner;
214 if (unlikely(!page_ext))
217 for (i = 0; i < nr; i++) {
218 page_owner = get_page_owner(page_ext);
219 page_owner->order = 0;
220 page_ext = page_ext_next(page_ext);
224 void __folio_copy_owner(struct folio *newfolio, struct folio *old)
226 struct page_ext *old_ext = lookup_page_ext(&old->page);
227 struct page_ext *new_ext = lookup_page_ext(&newfolio->page);
228 struct page_owner *old_page_owner, *new_page_owner;
230 if (unlikely(!old_ext || !new_ext))
233 old_page_owner = get_page_owner(old_ext);
234 new_page_owner = get_page_owner(new_ext);
235 new_page_owner->order = old_page_owner->order;
236 new_page_owner->gfp_mask = old_page_owner->gfp_mask;
237 new_page_owner->last_migrate_reason =
238 old_page_owner->last_migrate_reason;
239 new_page_owner->handle = old_page_owner->handle;
240 new_page_owner->pid = old_page_owner->pid;
241 new_page_owner->tgid = old_page_owner->tgid;
242 new_page_owner->ts_nsec = old_page_owner->ts_nsec;
243 new_page_owner->free_ts_nsec = old_page_owner->ts_nsec;
244 strcpy(new_page_owner->comm, old_page_owner->comm);
247 * We don't clear the bit on the old folio as it's going to be freed
248 * after migration. Until then, the info can be useful in case of
249 * a bug, and the overall stats will be off a bit only temporarily.
250 * Also, migrate_misplaced_transhuge_page() can still fail the
251 * migration and then we want the old folio to retain the info. But
252 * in that case we also don't need to explicitly clear the info from
253 * the new page, which will be freed.
255 __set_bit(PAGE_EXT_OWNER, &new_ext->flags);
256 __set_bit(PAGE_EXT_OWNER_ALLOCATED, &new_ext->flags);
259 void pagetypeinfo_showmixedcount_print(struct seq_file *m,
260 pg_data_t *pgdat, struct zone *zone)
263 struct page_ext *page_ext;
264 struct page_owner *page_owner;
265 unsigned long pfn, block_end_pfn;
266 unsigned long end_pfn = zone_end_pfn(zone);
267 unsigned long count[MIGRATE_TYPES] = { 0, };
268 int pageblock_mt, page_mt;
271 /* Scan block by block. First and last block may be incomplete */
272 pfn = zone->zone_start_pfn;
275 * Walk the zone in pageblock_nr_pages steps. If a page block spans
276 * a zone boundary, it will be double counted between zones. This does
277 * not matter as the mixed block count will still be correct
279 for (; pfn < end_pfn; ) {
280 page = pfn_to_online_page(pfn);
282 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
286 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
287 block_end_pfn = min(block_end_pfn, end_pfn);
289 pageblock_mt = get_pageblock_migratetype(page);
291 for (; pfn < block_end_pfn; pfn++) {
292 /* The pageblock is online, no need to recheck. */
293 page = pfn_to_page(pfn);
295 if (page_zone(page) != zone)
298 if (PageBuddy(page)) {
299 unsigned long freepage_order;
301 freepage_order = buddy_order_unsafe(page);
302 if (freepage_order < MAX_ORDER)
303 pfn += (1UL << freepage_order) - 1;
307 if (PageReserved(page))
310 page_ext = lookup_page_ext(page);
311 if (unlikely(!page_ext))
314 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
317 page_owner = get_page_owner(page_ext);
318 page_mt = gfp_migratetype(page_owner->gfp_mask);
319 if (pageblock_mt != page_mt) {
320 if (is_migrate_cma(pageblock_mt))
321 count[MIGRATE_MOVABLE]++;
323 count[pageblock_mt]++;
328 pfn += (1UL << page_owner->order) - 1;
333 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
334 for (i = 0; i < MIGRATE_TYPES; i++)
335 seq_printf(m, "%12lu ", count[i]);
340 * Looking for memcg information and print it out
342 static inline int print_page_owner_memcg(char *kbuf, size_t count, int ret,
346 unsigned long memcg_data;
347 struct mem_cgroup *memcg;
352 memcg_data = READ_ONCE(page->memcg_data);
356 if (memcg_data & MEMCG_DATA_OBJCGS)
357 ret += scnprintf(kbuf + ret, count - ret,
358 "Slab cache page\n");
360 memcg = page_memcg_check(page);
364 online = (memcg->css.flags & CSS_ONLINE);
365 cgroup_name(memcg->css.cgroup, name, sizeof(name));
366 ret += scnprintf(kbuf + ret, count - ret,
367 "Charged %sto %smemcg %s\n",
368 PageMemcgKmem(page) ? "(via objcg) " : "",
369 online ? "" : "offline ",
373 #endif /* CONFIG_MEMCG */
379 print_page_owner(char __user *buf, size_t count, unsigned long pfn,
380 struct page *page, struct page_owner *page_owner,
381 depot_stack_handle_t handle)
383 int ret, pageblock_mt, page_mt;
386 count = min_t(size_t, count, PAGE_SIZE);
387 kbuf = kmalloc(count, GFP_KERNEL);
391 ret = scnprintf(kbuf, count,
392 "Page allocated via order %u, mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu ns, free_ts %llu ns\n",
393 page_owner->order, page_owner->gfp_mask,
394 &page_owner->gfp_mask, page_owner->pid,
395 page_owner->tgid, page_owner->comm,
396 page_owner->ts_nsec, page_owner->free_ts_nsec);
398 /* Print information relevant to grouping pages by mobility */
399 pageblock_mt = get_pageblock_migratetype(page);
400 page_mt = gfp_migratetype(page_owner->gfp_mask);
401 ret += scnprintf(kbuf + ret, count - ret,
402 "PFN %lu type %s Block %lu type %s Flags %pGp\n",
404 migratetype_names[page_mt],
405 pfn >> pageblock_order,
406 migratetype_names[pageblock_mt],
409 ret += stack_depot_snprint(handle, kbuf + ret, count - ret, 0);
413 if (page_owner->last_migrate_reason != -1) {
414 ret += scnprintf(kbuf + ret, count - ret,
415 "Page has been migrated, last migrate reason: %s\n",
416 migrate_reason_names[page_owner->last_migrate_reason]);
419 ret = print_page_owner_memcg(kbuf, count, ret, page);
421 ret += snprintf(kbuf + ret, count - ret, "\n");
425 if (copy_to_user(buf, kbuf, ret))
436 void __dump_page_owner(const struct page *page)
438 struct page_ext *page_ext = lookup_page_ext(page);
439 struct page_owner *page_owner;
440 depot_stack_handle_t handle;
444 if (unlikely(!page_ext)) {
445 pr_alert("There is not page extension available.\n");
449 page_owner = get_page_owner(page_ext);
450 gfp_mask = page_owner->gfp_mask;
451 mt = gfp_migratetype(gfp_mask);
453 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags)) {
454 pr_alert("page_owner info is not present (never set?)\n");
458 if (test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
459 pr_alert("page_owner tracks the page as allocated\n");
461 pr_alert("page_owner tracks the page as freed\n");
463 pr_alert("page last allocated via order %u, migratetype %s, gfp_mask %#x(%pGg), pid %d, tgid %d (%s), ts %llu, free_ts %llu\n",
464 page_owner->order, migratetype_names[mt], gfp_mask, &gfp_mask,
465 page_owner->pid, page_owner->tgid, page_owner->comm,
466 page_owner->ts_nsec, page_owner->free_ts_nsec);
468 handle = READ_ONCE(page_owner->handle);
470 pr_alert("page_owner allocation stack trace missing\n");
472 stack_depot_print(handle);
474 handle = READ_ONCE(page_owner->free_handle);
476 pr_alert("page_owner free stack trace missing\n");
478 pr_alert("page last free stack trace:\n");
479 stack_depot_print(handle);
482 if (page_owner->last_migrate_reason != -1)
483 pr_alert("page has been migrated, last migrate reason: %s\n",
484 migrate_reason_names[page_owner->last_migrate_reason]);
488 read_page_owner(struct file *file, char __user *buf, size_t count, loff_t *ppos)
492 struct page_ext *page_ext;
493 struct page_owner *page_owner;
494 depot_stack_handle_t handle;
496 if (!static_branch_unlikely(&page_owner_inited))
500 pfn = min_low_pfn + *ppos;
502 /* Find a valid PFN or the start of a MAX_ORDER_NR_PAGES area */
503 while (!pfn_valid(pfn) && (pfn & (MAX_ORDER_NR_PAGES - 1)) != 0)
506 drain_all_pages(NULL);
508 /* Find an allocated page */
509 for (; pfn < max_pfn; pfn++) {
511 * If the new page is in a new MAX_ORDER_NR_PAGES area,
512 * validate the area as existing, skip it if not
514 if ((pfn & (MAX_ORDER_NR_PAGES - 1)) == 0 && !pfn_valid(pfn)) {
515 pfn += MAX_ORDER_NR_PAGES - 1;
519 page = pfn_to_page(pfn);
520 if (PageBuddy(page)) {
521 unsigned long freepage_order = buddy_order_unsafe(page);
523 if (freepage_order < MAX_ORDER)
524 pfn += (1UL << freepage_order) - 1;
528 page_ext = lookup_page_ext(page);
529 if (unlikely(!page_ext))
533 * Some pages could be missed by concurrent allocation or free,
534 * because we don't hold the zone lock.
536 if (!test_bit(PAGE_EXT_OWNER, &page_ext->flags))
540 * Although we do have the info about past allocation of free
541 * pages, it's not relevant for current memory usage.
543 if (!test_bit(PAGE_EXT_OWNER_ALLOCATED, &page_ext->flags))
546 page_owner = get_page_owner(page_ext);
549 * Don't print "tail" pages of high-order allocations as that
550 * would inflate the stats.
552 if (!IS_ALIGNED(pfn, 1 << page_owner->order))
556 * Access to page_ext->handle isn't synchronous so we should
557 * be careful to access it.
559 handle = READ_ONCE(page_owner->handle);
563 /* Record the next PFN to read in the file offset */
564 *ppos = (pfn - min_low_pfn) + 1;
566 return print_page_owner(buf, count, pfn, page,
573 static void init_pages_in_zone(pg_data_t *pgdat, struct zone *zone)
575 unsigned long pfn = zone->zone_start_pfn;
576 unsigned long end_pfn = zone_end_pfn(zone);
577 unsigned long count = 0;
580 * Walk the zone in pageblock_nr_pages steps. If a page block spans
581 * a zone boundary, it will be double counted between zones. This does
582 * not matter as the mixed block count will still be correct
584 for (; pfn < end_pfn; ) {
585 unsigned long block_end_pfn;
587 if (!pfn_valid(pfn)) {
588 pfn = ALIGN(pfn + 1, MAX_ORDER_NR_PAGES);
592 block_end_pfn = ALIGN(pfn + 1, pageblock_nr_pages);
593 block_end_pfn = min(block_end_pfn, end_pfn);
595 for (; pfn < block_end_pfn; pfn++) {
596 struct page *page = pfn_to_page(pfn);
597 struct page_ext *page_ext;
599 if (page_zone(page) != zone)
603 * To avoid having to grab zone->lock, be a little
604 * careful when reading buddy page order. The only
605 * danger is that we skip too much and potentially miss
606 * some early allocated pages, which is better than
607 * heavy lock contention.
609 if (PageBuddy(page)) {
610 unsigned long order = buddy_order_unsafe(page);
612 if (order > 0 && order < MAX_ORDER)
613 pfn += (1UL << order) - 1;
617 if (PageReserved(page))
620 page_ext = lookup_page_ext(page);
621 if (unlikely(!page_ext))
624 /* Maybe overlapping zone */
625 if (test_bit(PAGE_EXT_OWNER, &page_ext->flags))
628 /* Found early allocated page */
629 __set_page_owner_handle(page_ext, early_handle,
636 pr_info("Node %d, zone %8s: page owner found early allocated %lu pages\n",
637 pgdat->node_id, zone->name, count);
640 static void init_zones_in_node(pg_data_t *pgdat)
643 struct zone *node_zones = pgdat->node_zones;
645 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
646 if (!populated_zone(zone))
649 init_pages_in_zone(pgdat, zone);
653 static void init_early_allocated_pages(void)
657 for_each_online_pgdat(pgdat)
658 init_zones_in_node(pgdat);
661 static const struct file_operations proc_page_owner_operations = {
662 .read = read_page_owner,
665 static int __init pageowner_init(void)
667 if (!static_branch_unlikely(&page_owner_inited)) {
668 pr_info("page_owner is disabled\n");
672 debugfs_create_file("page_owner", 0400, NULL, NULL,
673 &proc_page_owner_operations);
677 late_initcall(pageowner_init)