1 // SPDX-License-Identifier: GPL-2.0-only
5 * Manages VM statistics
6 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
9 * Copyright (C) 2006 Silicon Graphics, Inc.,
10 * Christoph Lameter <christoph@lameter.com>
11 * Copyright (C) 2008-2014 Christoph Lameter
15 #include <linux/err.h>
16 #include <linux/module.h>
17 #include <linux/slab.h>
18 #include <linux/cpu.h>
19 #include <linux/cpumask.h>
20 #include <linux/vmstat.h>
21 #include <linux/proc_fs.h>
22 #include <linux/seq_file.h>
23 #include <linux/debugfs.h>
24 #include <linux/sched.h>
25 #include <linux/math64.h>
26 #include <linux/writeback.h>
27 #include <linux/compaction.h>
28 #include <linux/mm_inline.h>
29 #include <linux/page_ext.h>
30 #include <linux/page_owner.h>
34 #define NUMA_STATS_THRESHOLD (U16_MAX - 2)
37 int sysctl_vm_numa_stat = ENABLE_NUMA_STAT;
39 /* zero numa counters within a zone */
40 static void zero_zone_numa_counters(struct zone *zone)
44 for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++) {
45 atomic_long_set(&zone->vm_numa_stat[item], 0);
46 for_each_online_cpu(cpu)
47 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->vm_numa_stat_diff[item]
52 /* zero numa counters of all the populated zones */
53 static void zero_zones_numa_counters(void)
57 for_each_populated_zone(zone)
58 zero_zone_numa_counters(zone);
61 /* zero global numa counters */
62 static void zero_global_numa_counters(void)
66 for (item = 0; item < NR_VM_NUMA_STAT_ITEMS; item++)
67 atomic_long_set(&vm_numa_stat[item], 0);
70 static void invalid_numa_statistics(void)
72 zero_zones_numa_counters();
73 zero_global_numa_counters();
76 static DEFINE_MUTEX(vm_numa_stat_lock);
78 int sysctl_vm_numa_stat_handler(struct ctl_table *table, int write,
79 void *buffer, size_t *length, loff_t *ppos)
83 mutex_lock(&vm_numa_stat_lock);
85 oldval = sysctl_vm_numa_stat;
86 ret = proc_dointvec_minmax(table, write, buffer, length, ppos);
90 if (oldval == sysctl_vm_numa_stat)
92 else if (sysctl_vm_numa_stat == ENABLE_NUMA_STAT) {
93 static_branch_enable(&vm_numa_stat_key);
94 pr_info("enable numa statistics\n");
96 static_branch_disable(&vm_numa_stat_key);
97 invalid_numa_statistics();
98 pr_info("disable numa statistics, and clear numa counters\n");
102 mutex_unlock(&vm_numa_stat_lock);
107 #ifdef CONFIG_VM_EVENT_COUNTERS
108 DEFINE_PER_CPU(struct vm_event_state, vm_event_states) = {{0}};
109 EXPORT_PER_CPU_SYMBOL(vm_event_states);
111 static void sum_vm_events(unsigned long *ret)
116 memset(ret, 0, NR_VM_EVENT_ITEMS * sizeof(unsigned long));
118 for_each_online_cpu(cpu) {
119 struct vm_event_state *this = &per_cpu(vm_event_states, cpu);
121 for (i = 0; i < NR_VM_EVENT_ITEMS; i++)
122 ret[i] += this->event[i];
127 * Accumulate the vm event counters across all CPUs.
128 * The result is unavoidably approximate - it can change
129 * during and after execution of this function.
131 void all_vm_events(unsigned long *ret)
137 EXPORT_SYMBOL_GPL(all_vm_events);
140 * Fold the foreign cpu events into our own.
142 * This is adding to the events on one processor
143 * but keeps the global counts constant.
145 void vm_events_fold_cpu(int cpu)
147 struct vm_event_state *fold_state = &per_cpu(vm_event_states, cpu);
150 for (i = 0; i < NR_VM_EVENT_ITEMS; i++) {
151 count_vm_events(i, fold_state->event[i]);
152 fold_state->event[i] = 0;
156 #endif /* CONFIG_VM_EVENT_COUNTERS */
159 * Manage combined zone based / global counters
161 * vm_stat contains the global counters
163 atomic_long_t vm_zone_stat[NR_VM_ZONE_STAT_ITEMS] __cacheline_aligned_in_smp;
164 atomic_long_t vm_numa_stat[NR_VM_NUMA_STAT_ITEMS] __cacheline_aligned_in_smp;
165 atomic_long_t vm_node_stat[NR_VM_NODE_STAT_ITEMS] __cacheline_aligned_in_smp;
166 EXPORT_SYMBOL(vm_zone_stat);
167 EXPORT_SYMBOL(vm_numa_stat);
168 EXPORT_SYMBOL(vm_node_stat);
172 int calculate_pressure_threshold(struct zone *zone)
175 int watermark_distance;
178 * As vmstats are not up to date, there is drift between the estimated
179 * and real values. For high thresholds and a high number of CPUs, it
180 * is possible for the min watermark to be breached while the estimated
181 * value looks fine. The pressure threshold is a reduced value such
182 * that even the maximum amount of drift will not accidentally breach
185 watermark_distance = low_wmark_pages(zone) - min_wmark_pages(zone);
186 threshold = max(1, (int)(watermark_distance / num_online_cpus()));
189 * Maximum threshold is 125
191 threshold = min(125, threshold);
196 int calculate_normal_threshold(struct zone *zone)
199 int mem; /* memory in 128 MB units */
202 * The threshold scales with the number of processors and the amount
203 * of memory per zone. More memory means that we can defer updates for
204 * longer, more processors could lead to more contention.
205 * fls() is used to have a cheap way of logarithmic scaling.
207 * Some sample thresholds:
209 * Threshold Processors (fls) Zonesize fls(mem+1)
210 * ------------------------------------------------------------------
227 * 125 1024 10 8-16 GB 8
228 * 125 1024 10 16-32 GB 9
231 mem = zone_managed_pages(zone) >> (27 - PAGE_SHIFT);
233 threshold = 2 * fls(num_online_cpus()) * (1 + fls(mem));
236 * Maximum threshold is 125
238 threshold = min(125, threshold);
244 * Refresh the thresholds for each zone.
246 void refresh_zone_stat_thresholds(void)
248 struct pglist_data *pgdat;
253 /* Zero current pgdat thresholds */
254 for_each_online_pgdat(pgdat) {
255 for_each_online_cpu(cpu) {
256 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold = 0;
260 for_each_populated_zone(zone) {
261 struct pglist_data *pgdat = zone->zone_pgdat;
262 unsigned long max_drift, tolerate_drift;
264 threshold = calculate_normal_threshold(zone);
266 for_each_online_cpu(cpu) {
269 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
272 /* Base nodestat threshold on the largest populated zone. */
273 pgdat_threshold = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold;
274 per_cpu_ptr(pgdat->per_cpu_nodestats, cpu)->stat_threshold
275 = max(threshold, pgdat_threshold);
279 * Only set percpu_drift_mark if there is a danger that
280 * NR_FREE_PAGES reports the low watermark is ok when in fact
281 * the min watermark could be breached by an allocation
283 tolerate_drift = low_wmark_pages(zone) - min_wmark_pages(zone);
284 max_drift = num_online_cpus() * threshold;
285 if (max_drift > tolerate_drift)
286 zone->percpu_drift_mark = high_wmark_pages(zone) +
291 void set_pgdat_percpu_threshold(pg_data_t *pgdat,
292 int (*calculate_pressure)(struct zone *))
299 for (i = 0; i < pgdat->nr_zones; i++) {
300 zone = &pgdat->node_zones[i];
301 if (!zone->percpu_drift_mark)
304 threshold = (*calculate_pressure)(zone);
305 for_each_online_cpu(cpu)
306 per_cpu_ptr(zone->per_cpu_zonestats, cpu)->stat_threshold
312 * For use when we know that interrupts are disabled,
313 * or when we know that preemption is disabled and that
314 * particular counter cannot be updated from interrupt context.
316 void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
319 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
320 s8 __percpu *p = pcp->vm_stat_diff + item;
324 x = delta + __this_cpu_read(*p);
326 t = __this_cpu_read(pcp->stat_threshold);
328 if (unlikely(abs(x) > t)) {
329 zone_page_state_add(x, zone, item);
332 __this_cpu_write(*p, x);
334 EXPORT_SYMBOL(__mod_zone_page_state);
336 void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
339 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
340 s8 __percpu *p = pcp->vm_node_stat_diff + item;
344 if (vmstat_item_in_bytes(item)) {
346 * Only cgroups use subpage accounting right now; at
347 * the global level, these items still change in
348 * multiples of whole pages. Store them as pages
349 * internally to keep the per-cpu counters compact.
351 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
352 delta >>= PAGE_SHIFT;
355 x = delta + __this_cpu_read(*p);
357 t = __this_cpu_read(pcp->stat_threshold);
359 if (unlikely(abs(x) > t)) {
360 node_page_state_add(x, pgdat, item);
363 __this_cpu_write(*p, x);
365 EXPORT_SYMBOL(__mod_node_page_state);
368 * Optimized increment and decrement functions.
370 * These are only for a single page and therefore can take a struct page *
371 * argument instead of struct zone *. This allows the inclusion of the code
372 * generated for page_zone(page) into the optimized functions.
374 * No overflow check is necessary and therefore the differential can be
375 * incremented or decremented in place which may allow the compilers to
376 * generate better code.
377 * The increment or decrement is known and therefore one boundary check can
380 * NOTE: These functions are very performance sensitive. Change only
383 * Some processors have inc/dec instructions that are atomic vs an interrupt.
384 * However, the code must first determine the differential location in a zone
385 * based on the processor number and then inc/dec the counter. There is no
386 * guarantee without disabling preemption that the processor will not change
387 * in between and therefore the atomicity vs. interrupt cannot be exploited
388 * in a useful way here.
390 void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
392 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
393 s8 __percpu *p = pcp->vm_stat_diff + item;
396 v = __this_cpu_inc_return(*p);
397 t = __this_cpu_read(pcp->stat_threshold);
398 if (unlikely(v > t)) {
399 s8 overstep = t >> 1;
401 zone_page_state_add(v + overstep, zone, item);
402 __this_cpu_write(*p, -overstep);
406 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
408 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
409 s8 __percpu *p = pcp->vm_node_stat_diff + item;
412 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
414 v = __this_cpu_inc_return(*p);
415 t = __this_cpu_read(pcp->stat_threshold);
416 if (unlikely(v > t)) {
417 s8 overstep = t >> 1;
419 node_page_state_add(v + overstep, pgdat, item);
420 __this_cpu_write(*p, -overstep);
424 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
426 __inc_zone_state(page_zone(page), item);
428 EXPORT_SYMBOL(__inc_zone_page_state);
430 void __inc_node_page_state(struct page *page, enum node_stat_item item)
432 __inc_node_state(page_pgdat(page), item);
434 EXPORT_SYMBOL(__inc_node_page_state);
436 void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
438 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
439 s8 __percpu *p = pcp->vm_stat_diff + item;
442 v = __this_cpu_dec_return(*p);
443 t = __this_cpu_read(pcp->stat_threshold);
444 if (unlikely(v < - t)) {
445 s8 overstep = t >> 1;
447 zone_page_state_add(v - overstep, zone, item);
448 __this_cpu_write(*p, overstep);
452 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
454 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
455 s8 __percpu *p = pcp->vm_node_stat_diff + item;
458 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
460 v = __this_cpu_dec_return(*p);
461 t = __this_cpu_read(pcp->stat_threshold);
462 if (unlikely(v < - t)) {
463 s8 overstep = t >> 1;
465 node_page_state_add(v - overstep, pgdat, item);
466 __this_cpu_write(*p, overstep);
470 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
472 __dec_zone_state(page_zone(page), item);
474 EXPORT_SYMBOL(__dec_zone_page_state);
476 void __dec_node_page_state(struct page *page, enum node_stat_item item)
478 __dec_node_state(page_pgdat(page), item);
480 EXPORT_SYMBOL(__dec_node_page_state);
482 #ifdef CONFIG_HAVE_CMPXCHG_LOCAL
484 * If we have cmpxchg_local support then we do not need to incur the overhead
485 * that comes with local_irq_save/restore if we use this_cpu_cmpxchg.
487 * mod_state() modifies the zone counter state through atomic per cpu
490 * Overstep mode specifies how overstep should handled:
492 * 1 Overstepping half of threshold
493 * -1 Overstepping minus half of threshold
495 static inline void mod_zone_state(struct zone *zone,
496 enum zone_stat_item item, long delta, int overstep_mode)
498 struct per_cpu_zonestat __percpu *pcp = zone->per_cpu_zonestats;
499 s8 __percpu *p = pcp->vm_stat_diff + item;
503 z = 0; /* overflow to zone counters */
506 * The fetching of the stat_threshold is racy. We may apply
507 * a counter threshold to the wrong the cpu if we get
508 * rescheduled while executing here. However, the next
509 * counter update will apply the threshold again and
510 * therefore bring the counter under the threshold again.
512 * Most of the time the thresholds are the same anyways
513 * for all cpus in a zone.
515 t = this_cpu_read(pcp->stat_threshold);
517 o = this_cpu_read(*p);
521 int os = overstep_mode * (t >> 1) ;
523 /* Overflow must be added to zone counters */
527 } while (this_cpu_cmpxchg(*p, o, n) != o);
530 zone_page_state_add(z, zone, item);
533 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
536 mod_zone_state(zone, item, delta, 0);
538 EXPORT_SYMBOL(mod_zone_page_state);
540 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
542 mod_zone_state(page_zone(page), item, 1, 1);
544 EXPORT_SYMBOL(inc_zone_page_state);
546 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
548 mod_zone_state(page_zone(page), item, -1, -1);
550 EXPORT_SYMBOL(dec_zone_page_state);
552 static inline void mod_node_state(struct pglist_data *pgdat,
553 enum node_stat_item item, int delta, int overstep_mode)
555 struct per_cpu_nodestat __percpu *pcp = pgdat->per_cpu_nodestats;
556 s8 __percpu *p = pcp->vm_node_stat_diff + item;
559 if (vmstat_item_in_bytes(item)) {
561 * Only cgroups use subpage accounting right now; at
562 * the global level, these items still change in
563 * multiples of whole pages. Store them as pages
564 * internally to keep the per-cpu counters compact.
566 VM_WARN_ON_ONCE(delta & (PAGE_SIZE - 1));
567 delta >>= PAGE_SHIFT;
571 z = 0; /* overflow to node counters */
574 * The fetching of the stat_threshold is racy. We may apply
575 * a counter threshold to the wrong the cpu if we get
576 * rescheduled while executing here. However, the next
577 * counter update will apply the threshold again and
578 * therefore bring the counter under the threshold again.
580 * Most of the time the thresholds are the same anyways
581 * for all cpus in a node.
583 t = this_cpu_read(pcp->stat_threshold);
585 o = this_cpu_read(*p);
589 int os = overstep_mode * (t >> 1) ;
591 /* Overflow must be added to node counters */
595 } while (this_cpu_cmpxchg(*p, o, n) != o);
598 node_page_state_add(z, pgdat, item);
601 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
604 mod_node_state(pgdat, item, delta, 0);
606 EXPORT_SYMBOL(mod_node_page_state);
608 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
610 mod_node_state(pgdat, item, 1, 1);
613 void inc_node_page_state(struct page *page, enum node_stat_item item)
615 mod_node_state(page_pgdat(page), item, 1, 1);
617 EXPORT_SYMBOL(inc_node_page_state);
619 void dec_node_page_state(struct page *page, enum node_stat_item item)
621 mod_node_state(page_pgdat(page), item, -1, -1);
623 EXPORT_SYMBOL(dec_node_page_state);
626 * Use interrupt disable to serialize counter updates
628 void mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
633 local_irq_save(flags);
634 __mod_zone_page_state(zone, item, delta);
635 local_irq_restore(flags);
637 EXPORT_SYMBOL(mod_zone_page_state);
639 void inc_zone_page_state(struct page *page, enum zone_stat_item item)
644 zone = page_zone(page);
645 local_irq_save(flags);
646 __inc_zone_state(zone, item);
647 local_irq_restore(flags);
649 EXPORT_SYMBOL(inc_zone_page_state);
651 void dec_zone_page_state(struct page *page, enum zone_stat_item item)
655 local_irq_save(flags);
656 __dec_zone_page_state(page, item);
657 local_irq_restore(flags);
659 EXPORT_SYMBOL(dec_zone_page_state);
661 void inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
665 local_irq_save(flags);
666 __inc_node_state(pgdat, item);
667 local_irq_restore(flags);
669 EXPORT_SYMBOL(inc_node_state);
671 void mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
676 local_irq_save(flags);
677 __mod_node_page_state(pgdat, item, delta);
678 local_irq_restore(flags);
680 EXPORT_SYMBOL(mod_node_page_state);
682 void inc_node_page_state(struct page *page, enum node_stat_item item)
685 struct pglist_data *pgdat;
687 pgdat = page_pgdat(page);
688 local_irq_save(flags);
689 __inc_node_state(pgdat, item);
690 local_irq_restore(flags);
692 EXPORT_SYMBOL(inc_node_page_state);
694 void dec_node_page_state(struct page *page, enum node_stat_item item)
698 local_irq_save(flags);
699 __dec_node_page_state(page, item);
700 local_irq_restore(flags);
702 EXPORT_SYMBOL(dec_node_page_state);
706 * Fold a differential into the global counters.
707 * Returns the number of counters updated.
710 static int fold_diff(int *zone_diff, int *numa_diff, int *node_diff)
715 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
717 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
721 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
723 atomic_long_add(numa_diff[i], &vm_numa_stat[i]);
727 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
729 atomic_long_add(node_diff[i], &vm_node_stat[i]);
735 static int fold_diff(int *zone_diff, int *node_diff)
740 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
742 atomic_long_add(zone_diff[i], &vm_zone_stat[i]);
746 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
748 atomic_long_add(node_diff[i], &vm_node_stat[i]);
753 #endif /* CONFIG_NUMA */
756 * Update the zone counters for the current cpu.
758 * Note that refresh_cpu_vm_stats strives to only access
759 * node local memory. The per cpu pagesets on remote zones are placed
760 * in the memory local to the processor using that pageset. So the
761 * loop over all zones will access a series of cachelines local to
764 * The call to zone_page_state_add updates the cachelines with the
765 * statistics in the remote zone struct as well as the global cachelines
766 * with the global counters. These could cause remote node cache line
767 * bouncing and will have to be only done when necessary.
769 * The function returns the number of global counters updated.
771 static int refresh_cpu_vm_stats(bool do_pagesets)
773 struct pglist_data *pgdat;
776 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
778 int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
780 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
783 for_each_populated_zone(zone) {
784 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
786 struct per_cpu_pages __percpu *pcp = zone->per_cpu_pageset;
789 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
792 v = this_cpu_xchg(pzstats->vm_stat_diff[i], 0);
795 atomic_long_add(v, &zone->vm_stat[i]);
796 global_zone_diff[i] += v;
798 /* 3 seconds idle till flush */
799 __this_cpu_write(pcp->expire, 3);
804 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++) {
807 v = this_cpu_xchg(pzstats->vm_numa_stat_diff[i], 0);
810 atomic_long_add(v, &zone->vm_numa_stat[i]);
811 global_numa_diff[i] += v;
812 __this_cpu_write(pcp->expire, 3);
819 * Deal with draining the remote pageset of this
822 * Check if there are pages remaining in this pageset
823 * if not then there is nothing to expire.
825 if (!__this_cpu_read(pcp->expire) ||
826 !__this_cpu_read(pcp->count))
830 * We never drain zones local to this processor.
832 if (zone_to_nid(zone) == numa_node_id()) {
833 __this_cpu_write(pcp->expire, 0);
837 if (__this_cpu_dec_return(pcp->expire))
840 if (__this_cpu_read(pcp->count)) {
841 drain_zone_pages(zone, this_cpu_ptr(pcp));
848 for_each_online_pgdat(pgdat) {
849 struct per_cpu_nodestat __percpu *p = pgdat->per_cpu_nodestats;
851 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
854 v = this_cpu_xchg(p->vm_node_stat_diff[i], 0);
856 atomic_long_add(v, &pgdat->vm_stat[i]);
857 global_node_diff[i] += v;
863 changes += fold_diff(global_zone_diff, global_numa_diff,
866 changes += fold_diff(global_zone_diff, global_node_diff);
872 * Fold the data for an offline cpu into the global array.
873 * There cannot be any access by the offline cpu and therefore
874 * synchronization is simplified.
876 void cpu_vm_stats_fold(int cpu)
878 struct pglist_data *pgdat;
881 int global_zone_diff[NR_VM_ZONE_STAT_ITEMS] = { 0, };
883 int global_numa_diff[NR_VM_NUMA_STAT_ITEMS] = { 0, };
885 int global_node_diff[NR_VM_NODE_STAT_ITEMS] = { 0, };
887 for_each_populated_zone(zone) {
888 struct per_cpu_zonestat *pzstats;
890 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
892 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
893 if (pzstats->vm_stat_diff[i]) {
896 v = pzstats->vm_stat_diff[i];
897 pzstats->vm_stat_diff[i] = 0;
898 atomic_long_add(v, &zone->vm_stat[i]);
899 global_zone_diff[i] += v;
903 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
904 if (pzstats->vm_numa_stat_diff[i]) {
907 v = pzstats->vm_numa_stat_diff[i];
908 pzstats->vm_numa_stat_diff[i] = 0;
909 atomic_long_add(v, &zone->vm_numa_stat[i]);
910 global_numa_diff[i] += v;
915 for_each_online_pgdat(pgdat) {
916 struct per_cpu_nodestat *p;
918 p = per_cpu_ptr(pgdat->per_cpu_nodestats, cpu);
920 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++)
921 if (p->vm_node_stat_diff[i]) {
924 v = p->vm_node_stat_diff[i];
925 p->vm_node_stat_diff[i] = 0;
926 atomic_long_add(v, &pgdat->vm_stat[i]);
927 global_node_diff[i] += v;
932 fold_diff(global_zone_diff, global_numa_diff, global_node_diff);
934 fold_diff(global_zone_diff, global_node_diff);
939 * this is only called if !populated_zone(zone), which implies no other users of
940 * pset->vm_stat_diff[] exist.
942 void drain_zonestat(struct zone *zone, struct per_cpu_zonestat *pzstats)
946 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
947 if (pzstats->vm_stat_diff[i]) {
948 int v = pzstats->vm_stat_diff[i];
949 pzstats->vm_stat_diff[i] = 0;
950 atomic_long_add(v, &zone->vm_stat[i]);
951 atomic_long_add(v, &vm_zone_stat[i]);
955 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
956 if (pzstats->vm_numa_stat_diff[i]) {
957 int v = pzstats->vm_numa_stat_diff[i];
959 pzstats->vm_numa_stat_diff[i] = 0;
960 atomic_long_add(v, &zone->vm_numa_stat[i]);
961 atomic_long_add(v, &vm_numa_stat[i]);
968 void __inc_numa_state(struct zone *zone,
969 enum numa_stat_item item)
971 struct per_cpu_zonestat __percpu *pzstats = zone->per_cpu_zonestats;
972 u16 __percpu *p = pzstats->vm_numa_stat_diff + item;
975 v = __this_cpu_inc_return(*p);
977 if (unlikely(v > NUMA_STATS_THRESHOLD)) {
978 zone_numa_state_add(v, zone, item);
979 __this_cpu_write(*p, 0);
984 * Determine the per node value of a stat item. This function
985 * is called frequently in a NUMA machine, so try to be as
986 * frugal as possible.
988 unsigned long sum_zone_node_page_state(int node,
989 enum zone_stat_item item)
991 struct zone *zones = NODE_DATA(node)->node_zones;
993 unsigned long count = 0;
995 for (i = 0; i < MAX_NR_ZONES; i++)
996 count += zone_page_state(zones + i, item);
1002 * Determine the per node value of a numa stat item. To avoid deviation,
1003 * the per cpu stat number in vm_numa_stat_diff[] is also included.
1005 unsigned long sum_zone_numa_state(int node,
1006 enum numa_stat_item item)
1008 struct zone *zones = NODE_DATA(node)->node_zones;
1010 unsigned long count = 0;
1012 for (i = 0; i < MAX_NR_ZONES; i++)
1013 count += zone_numa_state_snapshot(zones + i, item);
1019 * Determine the per node value of a stat item.
1021 unsigned long node_page_state_pages(struct pglist_data *pgdat,
1022 enum node_stat_item item)
1024 long x = atomic_long_read(&pgdat->vm_stat[item]);
1032 unsigned long node_page_state(struct pglist_data *pgdat,
1033 enum node_stat_item item)
1035 VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
1037 return node_page_state_pages(pgdat, item);
1041 #ifdef CONFIG_COMPACTION
1043 struct contig_page_info {
1044 unsigned long free_pages;
1045 unsigned long free_blocks_total;
1046 unsigned long free_blocks_suitable;
1050 * Calculate the number of free pages in a zone, how many contiguous
1051 * pages are free and how many are large enough to satisfy an allocation of
1052 * the target size. Note that this function makes no attempt to estimate
1053 * how many suitable free blocks there *might* be if MOVABLE pages were
1054 * migrated. Calculating that is possible, but expensive and can be
1055 * figured out from userspace
1057 static void fill_contig_page_info(struct zone *zone,
1058 unsigned int suitable_order,
1059 struct contig_page_info *info)
1063 info->free_pages = 0;
1064 info->free_blocks_total = 0;
1065 info->free_blocks_suitable = 0;
1067 for (order = 0; order < MAX_ORDER; order++) {
1068 unsigned long blocks;
1070 /* Count number of free blocks */
1071 blocks = zone->free_area[order].nr_free;
1072 info->free_blocks_total += blocks;
1074 /* Count free base pages */
1075 info->free_pages += blocks << order;
1077 /* Count the suitable free blocks */
1078 if (order >= suitable_order)
1079 info->free_blocks_suitable += blocks <<
1080 (order - suitable_order);
1085 * A fragmentation index only makes sense if an allocation of a requested
1086 * size would fail. If that is true, the fragmentation index indicates
1087 * whether external fragmentation or a lack of memory was the problem.
1088 * The value can be used to determine if page reclaim or compaction
1091 static int __fragmentation_index(unsigned int order, struct contig_page_info *info)
1093 unsigned long requested = 1UL << order;
1095 if (WARN_ON_ONCE(order >= MAX_ORDER))
1098 if (!info->free_blocks_total)
1101 /* Fragmentation index only makes sense when a request would fail */
1102 if (info->free_blocks_suitable)
1106 * Index is between 0 and 1 so return within 3 decimal places
1108 * 0 => allocation would fail due to lack of memory
1109 * 1 => allocation would fail due to fragmentation
1111 return 1000 - div_u64( (1000+(div_u64(info->free_pages * 1000ULL, requested))), info->free_blocks_total);
1115 * Calculates external fragmentation within a zone wrt the given order.
1116 * It is defined as the percentage of pages found in blocks of size
1117 * less than 1 << order. It returns values in range [0, 100].
1119 unsigned int extfrag_for_order(struct zone *zone, unsigned int order)
1121 struct contig_page_info info;
1123 fill_contig_page_info(zone, order, &info);
1124 if (info.free_pages == 0)
1127 return div_u64((info.free_pages -
1128 (info.free_blocks_suitable << order)) * 100,
1132 /* Same as __fragmentation index but allocs contig_page_info on stack */
1133 int fragmentation_index(struct zone *zone, unsigned int order)
1135 struct contig_page_info info;
1137 fill_contig_page_info(zone, order, &info);
1138 return __fragmentation_index(order, &info);
1142 #if defined(CONFIG_PROC_FS) || defined(CONFIG_SYSFS) || \
1143 defined(CONFIG_NUMA) || defined(CONFIG_MEMCG)
1144 #ifdef CONFIG_ZONE_DMA
1145 #define TEXT_FOR_DMA(xx) xx "_dma",
1147 #define TEXT_FOR_DMA(xx)
1150 #ifdef CONFIG_ZONE_DMA32
1151 #define TEXT_FOR_DMA32(xx) xx "_dma32",
1153 #define TEXT_FOR_DMA32(xx)
1156 #ifdef CONFIG_HIGHMEM
1157 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
1159 #define TEXT_FOR_HIGHMEM(xx)
1162 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
1163 TEXT_FOR_HIGHMEM(xx) xx "_movable",
1165 const char * const vmstat_text[] = {
1166 /* enum zone_stat_item counters */
1168 "nr_zone_inactive_anon",
1169 "nr_zone_active_anon",
1170 "nr_zone_inactive_file",
1171 "nr_zone_active_file",
1172 "nr_zone_unevictable",
1173 "nr_zone_write_pending",
1176 #if IS_ENABLED(CONFIG_ZSMALLOC)
1181 /* enum numa_stat_item counters */
1191 /* enum node_stat_item counters */
1197 "nr_slab_reclaimable",
1198 "nr_slab_unreclaimable",
1202 "workingset_refault_anon",
1203 "workingset_refault_file",
1204 "workingset_activate_anon",
1205 "workingset_activate_file",
1206 "workingset_restore_anon",
1207 "workingset_restore_file",
1208 "workingset_nodereclaim",
1214 "nr_writeback_temp",
1216 "nr_shmem_hugepages",
1217 "nr_shmem_pmdmapped",
1218 "nr_file_hugepages",
1219 "nr_file_pmdmapped",
1220 "nr_anon_transparent_hugepages",
1222 "nr_vmscan_immediate_reclaim",
1225 "nr_kernel_misc_reclaimable",
1226 "nr_foll_pin_acquired",
1227 "nr_foll_pin_released",
1229 #if IS_ENABLED(CONFIG_SHADOW_CALL_STACK)
1230 "nr_shadow_call_stack",
1232 "nr_page_table_pages",
1237 /* enum writeback_stat_item counters */
1238 "nr_dirty_threshold",
1239 "nr_dirty_background_threshold",
1241 #if defined(CONFIG_VM_EVENT_COUNTERS) || defined(CONFIG_MEMCG)
1242 /* enum vm_event_item counters */
1248 TEXTS_FOR_ZONES("pgalloc")
1249 TEXTS_FOR_ZONES("allocstall")
1250 TEXTS_FOR_ZONES("pgskip")
1267 "pgscan_direct_throttle",
1274 "zone_reclaim_failed",
1278 "kswapd_inodesteal",
1279 "kswapd_low_wmark_hit_quickly",
1280 "kswapd_high_wmark_hit_quickly",
1289 #ifdef CONFIG_NUMA_BALANCING
1291 "numa_huge_pte_updates",
1293 "numa_hint_faults_local",
1294 "numa_pages_migrated",
1296 #ifdef CONFIG_MIGRATION
1297 "pgmigrate_success",
1299 "thp_migration_success",
1300 "thp_migration_fail",
1301 "thp_migration_split",
1303 #ifdef CONFIG_COMPACTION
1304 "compact_migrate_scanned",
1305 "compact_free_scanned",
1310 "compact_daemon_wake",
1311 "compact_daemon_migrate_scanned",
1312 "compact_daemon_free_scanned",
1315 #ifdef CONFIG_HUGETLB_PAGE
1316 "htlb_buddy_alloc_success",
1317 "htlb_buddy_alloc_fail",
1320 "cma_alloc_success",
1323 "unevictable_pgs_culled",
1324 "unevictable_pgs_scanned",
1325 "unevictable_pgs_rescued",
1326 "unevictable_pgs_mlocked",
1327 "unevictable_pgs_munlocked",
1328 "unevictable_pgs_cleared",
1329 "unevictable_pgs_stranded",
1331 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1333 "thp_fault_fallback",
1334 "thp_fault_fallback_charge",
1335 "thp_collapse_alloc",
1336 "thp_collapse_alloc_failed",
1338 "thp_file_fallback",
1339 "thp_file_fallback_charge",
1342 "thp_split_page_failed",
1343 "thp_deferred_split_page",
1345 #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
1348 "thp_zero_page_alloc",
1349 "thp_zero_page_alloc_failed",
1351 "thp_swpout_fallback",
1353 #ifdef CONFIG_MEMORY_BALLOON
1356 #ifdef CONFIG_BALLOON_COMPACTION
1359 #endif /* CONFIG_MEMORY_BALLOON */
1360 #ifdef CONFIG_DEBUG_TLBFLUSH
1361 "nr_tlb_remote_flush",
1362 "nr_tlb_remote_flush_received",
1363 "nr_tlb_local_flush_all",
1364 "nr_tlb_local_flush_one",
1365 #endif /* CONFIG_DEBUG_TLBFLUSH */
1367 #ifdef CONFIG_DEBUG_VM_VMACACHE
1368 "vmacache_find_calls",
1369 "vmacache_find_hits",
1376 "direct_map_level2_splits",
1377 "direct_map_level3_splits",
1379 #endif /* CONFIG_VM_EVENT_COUNTERS || CONFIG_MEMCG */
1381 #endif /* CONFIG_PROC_FS || CONFIG_SYSFS || CONFIG_NUMA || CONFIG_MEMCG */
1383 #if (defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)) || \
1384 defined(CONFIG_PROC_FS)
1385 static void *frag_start(struct seq_file *m, loff_t *pos)
1390 for (pgdat = first_online_pgdat();
1392 pgdat = next_online_pgdat(pgdat))
1398 static void *frag_next(struct seq_file *m, void *arg, loff_t *pos)
1400 pg_data_t *pgdat = (pg_data_t *)arg;
1403 return next_online_pgdat(pgdat);
1406 static void frag_stop(struct seq_file *m, void *arg)
1411 * Walk zones in a node and print using a callback.
1412 * If @assert_populated is true, only use callback for zones that are populated.
1414 static void walk_zones_in_node(struct seq_file *m, pg_data_t *pgdat,
1415 bool assert_populated, bool nolock,
1416 void (*print)(struct seq_file *m, pg_data_t *, struct zone *))
1419 struct zone *node_zones = pgdat->node_zones;
1420 unsigned long flags;
1422 for (zone = node_zones; zone - node_zones < MAX_NR_ZONES; ++zone) {
1423 if (assert_populated && !populated_zone(zone))
1427 spin_lock_irqsave(&zone->lock, flags);
1428 print(m, pgdat, zone);
1430 spin_unlock_irqrestore(&zone->lock, flags);
1435 #ifdef CONFIG_PROC_FS
1436 static void frag_show_print(struct seq_file *m, pg_data_t *pgdat,
1441 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1442 for (order = 0; order < MAX_ORDER; ++order)
1443 seq_printf(m, "%6lu ", zone->free_area[order].nr_free);
1448 * This walks the free areas for each zone.
1450 static int frag_show(struct seq_file *m, void *arg)
1452 pg_data_t *pgdat = (pg_data_t *)arg;
1453 walk_zones_in_node(m, pgdat, true, false, frag_show_print);
1457 static void pagetypeinfo_showfree_print(struct seq_file *m,
1458 pg_data_t *pgdat, struct zone *zone)
1462 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++) {
1463 seq_printf(m, "Node %4d, zone %8s, type %12s ",
1466 migratetype_names[mtype]);
1467 for (order = 0; order < MAX_ORDER; ++order) {
1468 unsigned long freecount = 0;
1469 struct free_area *area;
1470 struct list_head *curr;
1471 bool overflow = false;
1473 area = &(zone->free_area[order]);
1475 list_for_each(curr, &area->free_list[mtype]) {
1477 * Cap the free_list iteration because it might
1478 * be really large and we are under a spinlock
1479 * so a long time spent here could trigger a
1480 * hard lockup detector. Anyway this is a
1481 * debugging tool so knowing there is a handful
1482 * of pages of this order should be more than
1485 if (++freecount >= 100000) {
1490 seq_printf(m, "%s%6lu ", overflow ? ">" : "", freecount);
1491 spin_unlock_irq(&zone->lock);
1493 spin_lock_irq(&zone->lock);
1499 /* Print out the free pages at each order for each migatetype */
1500 static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
1503 pg_data_t *pgdat = (pg_data_t *)arg;
1506 seq_printf(m, "%-43s ", "Free pages count per migrate type at order");
1507 for (order = 0; order < MAX_ORDER; ++order)
1508 seq_printf(m, "%6d ", order);
1511 walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
1516 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
1517 pg_data_t *pgdat, struct zone *zone)
1521 unsigned long start_pfn = zone->zone_start_pfn;
1522 unsigned long end_pfn = zone_end_pfn(zone);
1523 unsigned long count[MIGRATE_TYPES] = { 0, };
1525 for (pfn = start_pfn; pfn < end_pfn; pfn += pageblock_nr_pages) {
1528 page = pfn_to_online_page(pfn);
1532 if (page_zone(page) != zone)
1535 mtype = get_pageblock_migratetype(page);
1537 if (mtype < MIGRATE_TYPES)
1542 seq_printf(m, "Node %d, zone %8s ", pgdat->node_id, zone->name);
1543 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1544 seq_printf(m, "%12lu ", count[mtype]);
1548 /* Print out the number of pageblocks for each migratetype */
1549 static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
1552 pg_data_t *pgdat = (pg_data_t *)arg;
1554 seq_printf(m, "\n%-23s", "Number of blocks type ");
1555 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1556 seq_printf(m, "%12s ", migratetype_names[mtype]);
1558 walk_zones_in_node(m, pgdat, true, false,
1559 pagetypeinfo_showblockcount_print);
1565 * Print out the number of pageblocks for each migratetype that contain pages
1566 * of other types. This gives an indication of how well fallbacks are being
1567 * contained by rmqueue_fallback(). It requires information from PAGE_OWNER
1568 * to determine what is going on
1570 static void pagetypeinfo_showmixedcount(struct seq_file *m, pg_data_t *pgdat)
1572 #ifdef CONFIG_PAGE_OWNER
1575 if (!static_branch_unlikely(&page_owner_inited))
1578 drain_all_pages(NULL);
1580 seq_printf(m, "\n%-23s", "Number of mixed blocks ");
1581 for (mtype = 0; mtype < MIGRATE_TYPES; mtype++)
1582 seq_printf(m, "%12s ", migratetype_names[mtype]);
1585 walk_zones_in_node(m, pgdat, true, true,
1586 pagetypeinfo_showmixedcount_print);
1587 #endif /* CONFIG_PAGE_OWNER */
1591 * This prints out statistics in relation to grouping pages by mobility.
1592 * It is expensive to collect so do not constantly read the file.
1594 static int pagetypeinfo_show(struct seq_file *m, void *arg)
1596 pg_data_t *pgdat = (pg_data_t *)arg;
1598 /* check memoryless node */
1599 if (!node_state(pgdat->node_id, N_MEMORY))
1602 seq_printf(m, "Page block order: %d\n", pageblock_order);
1603 seq_printf(m, "Pages per block: %lu\n", pageblock_nr_pages);
1605 pagetypeinfo_showfree(m, pgdat);
1606 pagetypeinfo_showblockcount(m, pgdat);
1607 pagetypeinfo_showmixedcount(m, pgdat);
1612 static const struct seq_operations fragmentation_op = {
1613 .start = frag_start,
1619 static const struct seq_operations pagetypeinfo_op = {
1620 .start = frag_start,
1623 .show = pagetypeinfo_show,
1626 static bool is_zone_first_populated(pg_data_t *pgdat, struct zone *zone)
1630 for (zid = 0; zid < MAX_NR_ZONES; zid++) {
1631 struct zone *compare = &pgdat->node_zones[zid];
1633 if (populated_zone(compare))
1634 return zone == compare;
1640 static void zoneinfo_show_print(struct seq_file *m, pg_data_t *pgdat,
1644 seq_printf(m, "Node %d, zone %8s", pgdat->node_id, zone->name);
1645 if (is_zone_first_populated(pgdat, zone)) {
1646 seq_printf(m, "\n per-node stats");
1647 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1648 unsigned long pages = node_page_state_pages(pgdat, i);
1650 if (vmstat_item_print_in_thp(i))
1651 pages /= HPAGE_PMD_NR;
1652 seq_printf(m, "\n %-12s %lu", node_stat_name(i),
1665 zone_page_state(zone, NR_FREE_PAGES),
1666 min_wmark_pages(zone),
1667 low_wmark_pages(zone),
1668 high_wmark_pages(zone),
1669 zone->spanned_pages,
1670 zone->present_pages,
1671 zone_managed_pages(zone),
1672 zone_cma_pages(zone));
1675 "\n protection: (%ld",
1676 zone->lowmem_reserve[0]);
1677 for (i = 1; i < ARRAY_SIZE(zone->lowmem_reserve); i++)
1678 seq_printf(m, ", %ld", zone->lowmem_reserve[i]);
1681 /* If unpopulated, no other information is useful */
1682 if (!populated_zone(zone)) {
1687 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1688 seq_printf(m, "\n %-12s %lu", zone_stat_name(i),
1689 zone_page_state(zone, i));
1692 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
1693 seq_printf(m, "\n %-12s %lu", numa_stat_name(i),
1694 zone_numa_state_snapshot(zone, i));
1697 seq_printf(m, "\n pagesets");
1698 for_each_online_cpu(i) {
1699 struct per_cpu_pages *pcp;
1700 struct per_cpu_zonestat __maybe_unused *pzstats;
1702 pcp = per_cpu_ptr(zone->per_cpu_pageset, i);
1713 pzstats = per_cpu_ptr(zone->per_cpu_zonestats, i);
1714 seq_printf(m, "\n vm stats threshold: %d",
1715 pzstats->stat_threshold);
1719 "\n node_unreclaimable: %u"
1720 "\n start_pfn: %lu",
1721 pgdat->kswapd_failures >= MAX_RECLAIM_RETRIES,
1722 zone->zone_start_pfn);
1727 * Output information about zones in @pgdat. All zones are printed regardless
1728 * of whether they are populated or not: lowmem_reserve_ratio operates on the
1729 * set of all zones and userspace would not be aware of such zones if they are
1730 * suppressed here (zoneinfo displays the effect of lowmem_reserve_ratio).
1732 static int zoneinfo_show(struct seq_file *m, void *arg)
1734 pg_data_t *pgdat = (pg_data_t *)arg;
1735 walk_zones_in_node(m, pgdat, false, false, zoneinfo_show_print);
1739 static const struct seq_operations zoneinfo_op = {
1740 .start = frag_start, /* iterate over all zones. The same as in
1744 .show = zoneinfo_show,
1747 #define NR_VMSTAT_ITEMS (NR_VM_ZONE_STAT_ITEMS + \
1748 NR_VM_NUMA_STAT_ITEMS + \
1749 NR_VM_NODE_STAT_ITEMS + \
1750 NR_VM_WRITEBACK_STAT_ITEMS + \
1751 (IS_ENABLED(CONFIG_VM_EVENT_COUNTERS) ? \
1752 NR_VM_EVENT_ITEMS : 0))
1754 static void *vmstat_start(struct seq_file *m, loff_t *pos)
1759 if (*pos >= NR_VMSTAT_ITEMS)
1762 BUILD_BUG_ON(ARRAY_SIZE(vmstat_text) < NR_VMSTAT_ITEMS);
1763 v = kmalloc_array(NR_VMSTAT_ITEMS, sizeof(unsigned long), GFP_KERNEL);
1766 return ERR_PTR(-ENOMEM);
1767 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++)
1768 v[i] = global_zone_page_state(i);
1769 v += NR_VM_ZONE_STAT_ITEMS;
1772 for (i = 0; i < NR_VM_NUMA_STAT_ITEMS; i++)
1773 v[i] = global_numa_state(i);
1774 v += NR_VM_NUMA_STAT_ITEMS;
1777 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1778 v[i] = global_node_page_state_pages(i);
1779 if (vmstat_item_print_in_thp(i))
1780 v[i] /= HPAGE_PMD_NR;
1782 v += NR_VM_NODE_STAT_ITEMS;
1784 global_dirty_limits(v + NR_DIRTY_BG_THRESHOLD,
1785 v + NR_DIRTY_THRESHOLD);
1786 v += NR_VM_WRITEBACK_STAT_ITEMS;
1788 #ifdef CONFIG_VM_EVENT_COUNTERS
1790 v[PGPGIN] /= 2; /* sectors -> kbytes */
1793 return (unsigned long *)m->private + *pos;
1796 static void *vmstat_next(struct seq_file *m, void *arg, loff_t *pos)
1799 if (*pos >= NR_VMSTAT_ITEMS)
1801 return (unsigned long *)m->private + *pos;
1804 static int vmstat_show(struct seq_file *m, void *arg)
1806 unsigned long *l = arg;
1807 unsigned long off = l - (unsigned long *)m->private;
1809 seq_puts(m, vmstat_text[off]);
1810 seq_put_decimal_ull(m, " ", *l);
1813 if (off == NR_VMSTAT_ITEMS - 1) {
1815 * We've come to the end - add any deprecated counters to avoid
1816 * breaking userspace which might depend on them being present.
1818 seq_puts(m, "nr_unstable 0\n");
1823 static void vmstat_stop(struct seq_file *m, void *arg)
1829 static const struct seq_operations vmstat_op = {
1830 .start = vmstat_start,
1831 .next = vmstat_next,
1832 .stop = vmstat_stop,
1833 .show = vmstat_show,
1835 #endif /* CONFIG_PROC_FS */
1838 static DEFINE_PER_CPU(struct delayed_work, vmstat_work);
1839 int sysctl_stat_interval __read_mostly = HZ;
1841 #ifdef CONFIG_PROC_FS
1842 static void refresh_vm_stats(struct work_struct *work)
1844 refresh_cpu_vm_stats(true);
1847 int vmstat_refresh(struct ctl_table *table, int write,
1848 void *buffer, size_t *lenp, loff_t *ppos)
1855 * The regular update, every sysctl_stat_interval, may come later
1856 * than expected: leaving a significant amount in per_cpu buckets.
1857 * This is particularly misleading when checking a quantity of HUGE
1858 * pages, immediately after running a test. /proc/sys/vm/stat_refresh,
1859 * which can equally be echo'ed to or cat'ted from (by root),
1860 * can be used to update the stats just before reading them.
1862 * Oh, and since global_zone_page_state() etc. are so careful to hide
1863 * transiently negative values, report an error here if any of
1864 * the stats is negative, so we know to go looking for imbalance.
1866 err = schedule_on_each_cpu(refresh_vm_stats);
1869 for (i = 0; i < NR_VM_ZONE_STAT_ITEMS; i++) {
1871 * Skip checking stats known to go negative occasionally.
1874 case NR_ZONE_WRITE_PENDING:
1875 case NR_FREE_CMA_PAGES:
1878 val = atomic_long_read(&vm_zone_stat[i]);
1880 pr_warn("%s: %s %ld\n",
1881 __func__, zone_stat_name(i), val);
1884 for (i = 0; i < NR_VM_NODE_STAT_ITEMS; i++) {
1886 * Skip checking stats known to go negative occasionally.
1892 val = atomic_long_read(&vm_node_stat[i]);
1894 pr_warn("%s: %s %ld\n",
1895 __func__, node_stat_name(i), val);
1904 #endif /* CONFIG_PROC_FS */
1906 static void vmstat_update(struct work_struct *w)
1908 if (refresh_cpu_vm_stats(true)) {
1910 * Counters were updated so we expect more updates
1911 * to occur in the future. Keep on running the
1912 * update worker thread.
1914 queue_delayed_work_on(smp_processor_id(), mm_percpu_wq,
1915 this_cpu_ptr(&vmstat_work),
1916 round_jiffies_relative(sysctl_stat_interval));
1921 * Switch off vmstat processing and then fold all the remaining differentials
1922 * until the diffs stay at zero. The function is used by NOHZ and can only be
1923 * invoked when tick processing is not active.
1926 * Check if the diffs for a certain cpu indicate that
1927 * an update is needed.
1929 static bool need_update(int cpu)
1931 pg_data_t *last_pgdat = NULL;
1934 for_each_populated_zone(zone) {
1935 struct per_cpu_zonestat *pzstats = per_cpu_ptr(zone->per_cpu_zonestats, cpu);
1936 struct per_cpu_nodestat *n;
1939 * The fast way of checking if there are any vmstat diffs.
1941 if (memchr_inv(pzstats->vm_stat_diff, 0, NR_VM_ZONE_STAT_ITEMS *
1942 sizeof(pzstats->vm_stat_diff[0])))
1945 if (memchr_inv(pzstats->vm_numa_stat_diff, 0, NR_VM_NUMA_STAT_ITEMS *
1946 sizeof(pzstats->vm_numa_stat_diff[0])))
1949 if (last_pgdat == zone->zone_pgdat)
1951 last_pgdat = zone->zone_pgdat;
1952 n = per_cpu_ptr(zone->zone_pgdat->per_cpu_nodestats, cpu);
1953 if (memchr_inv(n->vm_node_stat_diff, 0, NR_VM_NODE_STAT_ITEMS *
1954 sizeof(n->vm_node_stat_diff[0])))
1961 * Switch off vmstat processing and then fold all the remaining differentials
1962 * until the diffs stay at zero. The function is used by NOHZ and can only be
1963 * invoked when tick processing is not active.
1965 void quiet_vmstat(void)
1967 if (system_state != SYSTEM_RUNNING)
1970 if (!delayed_work_pending(this_cpu_ptr(&vmstat_work)))
1973 if (!need_update(smp_processor_id()))
1977 * Just refresh counters and do not care about the pending delayed
1978 * vmstat_update. It doesn't fire that often to matter and canceling
1979 * it would be too expensive from this path.
1980 * vmstat_shepherd will take care about that for us.
1982 refresh_cpu_vm_stats(false);
1986 * Shepherd worker thread that checks the
1987 * differentials of processors that have their worker
1988 * threads for vm statistics updates disabled because of
1991 static void vmstat_shepherd(struct work_struct *w);
1993 static DECLARE_DEFERRABLE_WORK(shepherd, vmstat_shepherd);
1995 static void vmstat_shepherd(struct work_struct *w)
2000 /* Check processors whose vmstat worker threads have been disabled */
2001 for_each_online_cpu(cpu) {
2002 struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
2004 if (!delayed_work_pending(dw) && need_update(cpu))
2005 queue_delayed_work_on(cpu, mm_percpu_wq, dw, 0);
2011 schedule_delayed_work(&shepherd,
2012 round_jiffies_relative(sysctl_stat_interval));
2015 static void __init start_shepherd_timer(void)
2019 for_each_possible_cpu(cpu)
2020 INIT_DEFERRABLE_WORK(per_cpu_ptr(&vmstat_work, cpu),
2023 schedule_delayed_work(&shepherd,
2024 round_jiffies_relative(sysctl_stat_interval));
2027 static void __init init_cpu_node_state(void)
2031 for_each_online_node(node) {
2032 if (cpumask_weight(cpumask_of_node(node)) > 0)
2033 node_set_state(node, N_CPU);
2037 static int vmstat_cpu_online(unsigned int cpu)
2039 refresh_zone_stat_thresholds();
2040 node_set_state(cpu_to_node(cpu), N_CPU);
2044 static int vmstat_cpu_down_prep(unsigned int cpu)
2046 cancel_delayed_work_sync(&per_cpu(vmstat_work, cpu));
2050 static int vmstat_cpu_dead(unsigned int cpu)
2052 const struct cpumask *node_cpus;
2055 node = cpu_to_node(cpu);
2057 refresh_zone_stat_thresholds();
2058 node_cpus = cpumask_of_node(node);
2059 if (cpumask_weight(node_cpus) > 0)
2062 node_clear_state(node, N_CPU);
2068 struct workqueue_struct *mm_percpu_wq;
2070 void __init init_mm_internals(void)
2072 int ret __maybe_unused;
2074 mm_percpu_wq = alloc_workqueue("mm_percpu_wq", WQ_MEM_RECLAIM, 0);
2077 ret = cpuhp_setup_state_nocalls(CPUHP_MM_VMSTAT_DEAD, "mm/vmstat:dead",
2078 NULL, vmstat_cpu_dead);
2080 pr_err("vmstat: failed to register 'dead' hotplug state\n");
2082 ret = cpuhp_setup_state_nocalls(CPUHP_AP_ONLINE_DYN, "mm/vmstat:online",
2084 vmstat_cpu_down_prep);
2086 pr_err("vmstat: failed to register 'online' hotplug state\n");
2089 init_cpu_node_state();
2092 start_shepherd_timer();
2094 #ifdef CONFIG_PROC_FS
2095 proc_create_seq("buddyinfo", 0444, NULL, &fragmentation_op);
2096 proc_create_seq("pagetypeinfo", 0400, NULL, &pagetypeinfo_op);
2097 proc_create_seq("vmstat", 0444, NULL, &vmstat_op);
2098 proc_create_seq("zoneinfo", 0444, NULL, &zoneinfo_op);
2102 #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_COMPACTION)
2105 * Return an index indicating how much of the available free memory is
2106 * unusable for an allocation of the requested size.
2108 static int unusable_free_index(unsigned int order,
2109 struct contig_page_info *info)
2111 /* No free memory is interpreted as all free memory is unusable */
2112 if (info->free_pages == 0)
2116 * Index should be a value between 0 and 1. Return a value to 3
2119 * 0 => no fragmentation
2120 * 1 => high fragmentation
2122 return div_u64((info->free_pages - (info->free_blocks_suitable << order)) * 1000ULL, info->free_pages);
2126 static void unusable_show_print(struct seq_file *m,
2127 pg_data_t *pgdat, struct zone *zone)
2131 struct contig_page_info info;
2133 seq_printf(m, "Node %d, zone %8s ",
2136 for (order = 0; order < MAX_ORDER; ++order) {
2137 fill_contig_page_info(zone, order, &info);
2138 index = unusable_free_index(order, &info);
2139 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2146 * Display unusable free space index
2148 * The unusable free space index measures how much of the available free
2149 * memory cannot be used to satisfy an allocation of a given size and is a
2150 * value between 0 and 1. The higher the value, the more of free memory is
2151 * unusable and by implication, the worse the external fragmentation is. This
2152 * can be expressed as a percentage by multiplying by 100.
2154 static int unusable_show(struct seq_file *m, void *arg)
2156 pg_data_t *pgdat = (pg_data_t *)arg;
2158 /* check memoryless node */
2159 if (!node_state(pgdat->node_id, N_MEMORY))
2162 walk_zones_in_node(m, pgdat, true, false, unusable_show_print);
2167 static const struct seq_operations unusable_sops = {
2168 .start = frag_start,
2171 .show = unusable_show,
2174 DEFINE_SEQ_ATTRIBUTE(unusable);
2176 static void extfrag_show_print(struct seq_file *m,
2177 pg_data_t *pgdat, struct zone *zone)
2182 /* Alloc on stack as interrupts are disabled for zone walk */
2183 struct contig_page_info info;
2185 seq_printf(m, "Node %d, zone %8s ",
2188 for (order = 0; order < MAX_ORDER; ++order) {
2189 fill_contig_page_info(zone, order, &info);
2190 index = __fragmentation_index(order, &info);
2191 seq_printf(m, "%d.%03d ", index / 1000, index % 1000);
2198 * Display fragmentation index for orders that allocations would fail for
2200 static int extfrag_show(struct seq_file *m, void *arg)
2202 pg_data_t *pgdat = (pg_data_t *)arg;
2204 walk_zones_in_node(m, pgdat, true, false, extfrag_show_print);
2209 static const struct seq_operations extfrag_sops = {
2210 .start = frag_start,
2213 .show = extfrag_show,
2216 DEFINE_SEQ_ATTRIBUTE(extfrag);
2218 static int __init extfrag_debug_init(void)
2220 struct dentry *extfrag_debug_root;
2222 extfrag_debug_root = debugfs_create_dir("extfrag", NULL);
2224 debugfs_create_file("unusable_index", 0444, extfrag_debug_root, NULL,
2227 debugfs_create_file("extfrag_index", 0444, extfrag_debug_root, NULL,
2233 module_init(extfrag_debug_init);