Merge tag 'acpi-5.15-rc1-3' of git://git.kernel.org/pub/scm/linux/kernel/git/rafael...

[linux-2.6-microblaze.git] / mm / vmstat.c

diff --git a/mm/vmstat.c b/mm/vmstat.c
index c4634dc..8ce2620 100644 (file)
--- a/mm/vmstat.c
+++ b/mm/vmstat.c
@@ -129,9 +129,9 @@ static void sum_vm_events(unsigned long *ret)
 */
 void all_vm_events(unsigned long *ret)
 {
-       get_online_cpus();
+       cpus_read_lock();
        sum_vm_events(ret);
-       put_online_cpus();
+       cpus_read_unlock();
 }
 EXPORT_SYMBOL_GPL(all_vm_events);
 
@@ -319,6 +319,16 @@ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
        long x;
        long t;
 
+       /*
+        * Accurate vmstat updates require a RMW. On !PREEMPT_RT kernels,
+        * atomicity is provided by IRQs being disabled -- either explicitly
+        * or via local_lock_irq. On PREEMPT_RT, local_lock_irq only disables
+        * CPU migrations and preemption potentially corrupts a counter so
+        * disable preemption.
+        */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        x = delta + __this_cpu_read(*p);
 
        t = __this_cpu_read(pcp->stat_threshold);
@@ -328,6 +338,9 @@ void __mod_zone_page_state(struct zone *zone, enum zone_stat_item item,
                x = 0;
        }
        __this_cpu_write(*p, x);
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 EXPORT_SYMBOL(__mod_zone_page_state);
 
@@ -350,6 +363,10 @@ void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
                delta >>= PAGE_SHIFT;
        }
 
+       /* See __mod_node_page_state */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        x = delta + __this_cpu_read(*p);
 
        t = __this_cpu_read(pcp->stat_threshold);
@@ -359,6 +376,9 @@ void __mod_node_page_state(struct pglist_data *pgdat, enum node_stat_item item,
                x = 0;
        }
        __this_cpu_write(*p, x);
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 EXPORT_SYMBOL(__mod_node_page_state);
 
@@ -391,6 +411,10 @@ void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
        s8 __percpu *p = pcp->vm_stat_diff + item;
        s8 v, t;
 
+       /* See __mod_node_page_state */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        v = __this_cpu_inc_return(*p);
        t = __this_cpu_read(pcp->stat_threshold);
        if (unlikely(v > t)) {
@@ -399,6 +423,9 @@ void __inc_zone_state(struct zone *zone, enum zone_stat_item item)
                zone_page_state_add(v + overstep, zone, item);
                __this_cpu_write(*p, -overstep);
        }
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 
 void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
@@ -409,6 +436,10 @@ void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 
        VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
 
+       /* See __mod_node_page_state */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        v = __this_cpu_inc_return(*p);
        t = __this_cpu_read(pcp->stat_threshold);
        if (unlikely(v > t)) {
@@ -417,6 +448,9 @@ void __inc_node_state(struct pglist_data *pgdat, enum node_stat_item item)
                node_page_state_add(v + overstep, pgdat, item);
                __this_cpu_write(*p, -overstep);
        }
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 
 void __inc_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -437,6 +471,10 @@ void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
        s8 __percpu *p = pcp->vm_stat_diff + item;
        s8 v, t;
 
+       /* See __mod_node_page_state */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        v = __this_cpu_dec_return(*p);
        t = __this_cpu_read(pcp->stat_threshold);
        if (unlikely(v < - t)) {
@@ -445,6 +483,9 @@ void __dec_zone_state(struct zone *zone, enum zone_stat_item item)
                zone_page_state_add(v - overstep, zone, item);
                __this_cpu_write(*p, overstep);
        }
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 
 void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
@@ -455,6 +496,10 @@ void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
 
        VM_WARN_ON_ONCE(vmstat_item_in_bytes(item));
 
+       /* See __mod_node_page_state */
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_disable();
+
        v = __this_cpu_dec_return(*p);
        t = __this_cpu_read(pcp->stat_threshold);
        if (unlikely(v < - t)) {
@@ -463,6 +508,9 @@ void __dec_node_state(struct pglist_data *pgdat, enum node_stat_item item)
                node_page_state_add(v - overstep, pgdat, item);
                __this_cpu_write(*p, overstep);
        }
+
+       if (IS_ENABLED(CONFIG_PREEMPT_RT))
+               preempt_enable();
 }
 
 void __dec_zone_page_state(struct page *page, enum zone_stat_item item)
@@ -1454,7 +1502,7 @@ static void pagetypeinfo_showfree_print(struct seq_file *m,
 }
 
 /* Print out the free pages at each order for each migatetype */
-static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
+static void pagetypeinfo_showfree(struct seq_file *m, void *arg)
 {
        int order;
        pg_data_t *pgdat = (pg_data_t *)arg;
@@ -1466,8 +1514,6 @@ static int pagetypeinfo_showfree(struct seq_file *m, void *arg)
        seq_putc(m, '\n');
 
        walk_zones_in_node(m, pgdat, true, false, pagetypeinfo_showfree_print);
-
-       return 0;
 }
 
 static void pagetypeinfo_showblockcount_print(struct seq_file *m,
@@ -1503,7 +1549,7 @@ static void pagetypeinfo_showblockcount_print(struct seq_file *m,
 }
 
 /* Print out the number of pageblocks for each migratetype */
-static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
+static void pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
 {
        int mtype;
        pg_data_t *pgdat = (pg_data_t *)arg;
@@ -1514,8 +1560,6 @@ static int pagetypeinfo_showblockcount(struct seq_file *m, void *arg)
        seq_putc(m, '\n');
        walk_zones_in_node(m, pgdat, true, false,
                pagetypeinfo_showblockcount_print);
-
-       return 0;
 }
 
 /*
@@ -1943,7 +1987,7 @@ static void vmstat_shepherd(struct work_struct *w)
 {
        int cpu;
 
-       get_online_cpus();
+       cpus_read_lock();
        /* Check processors whose vmstat worker threads have been disabled */
        for_each_online_cpu(cpu) {
                struct delayed_work *dw = &per_cpu(vmstat_work, cpu);
@@ -1953,7 +1997,7 @@ static void vmstat_shepherd(struct work_struct *w)
 
                cond_resched();
        }
-       put_online_cpus();
+       cpus_read_unlock();
 
        schedule_delayed_work(&shepherd,
                round_jiffies_relative(sysctl_stat_interval));
@@ -2032,9 +2076,9 @@ void __init init_mm_internals(void)
        if (ret < 0)
                pr_err("vmstat: failed to register 'online' hotplug state\n");
 
-       get_online_cpus();
+       cpus_read_lock();
        init_cpu_node_state();
-       put_online_cpus();
+       cpus_read_unlock();
 
        start_shepherd_timer();
 #endif