Merge branches 'pm-cpufreq', 'pm-sleep' and 'pm-em'

[linux-2.6-microblaze.git] / mm / memcontrol.c

diff --git a/mm/memcontrol.c b/mm/memcontrol.c
index 400d210..b762215 100644 (file)
--- a/mm/memcontrol.c
+++ b/mm/memcontrol.c
@@ -103,6 +103,14 @@ static bool do_memsw_account(void)
        return !cgroup_subsys_on_dfl(memory_cgrp_subsys) && !cgroup_memory_noswap;
 }
 
+/* memcg and lruvec stats flushing */
+static void flush_memcg_stats_dwork(struct work_struct *w);
+static DECLARE_DEFERRABLE_WORK(stats_flush_dwork, flush_memcg_stats_dwork);
+static void flush_memcg_stats_work(struct work_struct *w);
+static DECLARE_WORK(stats_flush_work, flush_memcg_stats_work);
+static DEFINE_PER_CPU(unsigned int, stats_flush_threshold);
+static DEFINE_SPINLOCK(stats_flush_lock);
+
 #define THRESHOLDS_EVENTS_TARGET 128
 #define SOFTLIMIT_EVENTS_TARGET 1024
 
@@ -248,9 +256,9 @@ struct vmpressure *memcg_to_vmpressure(struct mem_cgroup *memcg)
        return &memcg->vmpressure;
 }
 
-struct cgroup_subsys_state *vmpressure_to_css(struct vmpressure *vmpr)
+struct mem_cgroup *vmpressure_to_memcg(struct vmpressure *vmpr)
 {
-       return &container_of(vmpr, struct mem_cgroup, vmpressure)->css;
+       return container_of(vmpr, struct mem_cgroup, vmpressure);
 }
 
 #ifdef CONFIG_MEMCG_KMEM
@@ -674,6 +682,8 @@ void __mod_memcg_lruvec_state(struct lruvec *lruvec, enum node_stat_item idx,
 
        /* Update lruvec */
        __this_cpu_add(pn->lruvec_stats_percpu->state[idx], val);
+       if (!(__this_cpu_inc_return(stats_flush_threshold) % MEMCG_CHARGE_BATCH))
+               queue_work(system_unbound_wq, &stats_flush_work);
 }
 
 /**
@@ -868,7 +878,7 @@ EXPORT_SYMBOL(mem_cgroup_from_task);
 
 static __always_inline struct mem_cgroup *active_memcg(void)
 {
-       if (in_interrupt())
+       if (!in_task())
                return this_cpu_read(int_active_memcg);
        else
                return current->active_memcg;
@@ -931,7 +941,7 @@ static __always_inline bool memcg_kmem_bypass(void)
                return false;
 
        /* Memcg to charge can't be determined. */
-       if (in_interrupt() || !current->mm || (current->flags & PF_KTHREAD))
+       if (!in_task() || !current->mm || (current->flags & PF_KTHREAD))
                return true;
 
        return false;
@@ -2168,8 +2178,9 @@ static void drain_local_stock(struct work_struct *dummy)
        unsigned long flags;
 
        /*
-        * The only protection from memory hotplug vs. drain_stock races is
-        * that we always operate on local CPU stock here with IRQ disabled
+        * The only protection from cpu hotplug (memcg_hotplug_cpu_dead) vs.
+        * drain_stock races is that we always operate on local CPU stock
+        * here with IRQ disabled
         */
        local_irq_save(flags);
 
@@ -2236,7 +2247,7 @@ static void drain_all_stock(struct mem_cgroup *root_memcg)
                if (memcg && stock->nr_pages &&
                    mem_cgroup_is_descendant(memcg, root_memcg))
                        flush = true;
-               if (obj_stock_flush_required(stock, root_memcg))
+               else if (obj_stock_flush_required(stock, root_memcg))
                        flush = true;
                rcu_read_unlock();
 
@@ -4052,7 +4063,7 @@ static int mem_cgroup_swappiness_write(struct cgroup_subsys_state *css,
 {
        struct mem_cgroup *memcg = mem_cgroup_from_css(css);
 
-       if (val > 100)
+       if (val > 200)
                return -EINVAL;
 
        if (!mem_cgroup_is_root(memcg))
@@ -4828,9 +4839,9 @@ static ssize_t memcg_write_event_control(struct kernfs_open_file *of,
 
        vfs_poll(efile.file, &event->pt);
 
-       spin_lock(&memcg->event_list_lock);
+       spin_lock_irq(&memcg->event_list_lock);
        list_add(&event->list, &memcg->event_list);
-       spin_unlock(&memcg->event_list_lock);
+       spin_unlock_irq(&memcg->event_list_lock);
 
        fdput(cfile);
        fdput(efile);
@@ -5240,6 +5251,10 @@ static int mem_cgroup_css_online(struct cgroup_subsys_state *css)
        /* Online state pins memcg ID, memcg ID pins CSS */
        refcount_set(&memcg->id.ref, 1);
        css_get(css);
+
+       if (unlikely(mem_cgroup_is_root(memcg)))
+               queue_delayed_work(system_unbound_wq, &stats_flush_dwork,
+                                  2UL*HZ);
        return 0;
 }
 
@@ -5253,12 +5268,12 @@ static void mem_cgroup_css_offline(struct cgroup_subsys_state *css)
         * Notify userspace about cgroup removing only after rmdir of cgroup
         * directory to avoid race between userspace and kernelspace.
         */
-       spin_lock(&memcg->event_list_lock);
+       spin_lock_irq(&memcg->event_list_lock);
        list_for_each_entry_safe(event, tmp, &memcg->event_list, list) {
                list_del_init(&event->list);
                schedule_work(&event->remove);
        }
-       spin_unlock(&memcg->event_list_lock);
+       spin_unlock_irq(&memcg->event_list_lock);
 
        page_counter_set_min(&memcg->memory, 0);
        page_counter_set_low(&memcg->memory, 0);
@@ -5331,6 +5346,26 @@ static void mem_cgroup_css_reset(struct cgroup_subsys_state *css)
        memcg_wb_domain_size_changed(memcg);
 }
 
+void mem_cgroup_flush_stats(void)
+{
+       if (!spin_trylock(&stats_flush_lock))
+               return;
+
+       cgroup_rstat_flush_irqsafe(root_mem_cgroup->css.cgroup);
+       spin_unlock(&stats_flush_lock);
+}
+
+static void flush_memcg_stats_dwork(struct work_struct *w)
+{
+       mem_cgroup_flush_stats();
+       queue_delayed_work(system_unbound_wq, &stats_flush_dwork, 2UL*HZ);
+}
+
+static void flush_memcg_stats_work(struct work_struct *w)
+{
+       mem_cgroup_flush_stats();
+}
+
 static void mem_cgroup_css_rstat_flush(struct cgroup_subsys_state *css, int cpu)
 {
        struct mem_cgroup *memcg = mem_cgroup_from_css(css);
@@ -6992,14 +7027,14 @@ void mem_cgroup_sk_free(struct sock *sk)
  * mem_cgroup_charge_skmem - charge socket memory
  * @memcg: memcg to charge
  * @nr_pages: number of pages to charge
+ * @gfp_mask: reclaim mode
  *
  * Charges @nr_pages to @memcg. Returns %true if the charge fit within
- * @memcg's configured limit, %false if the charge had to be forced.
+ * @memcg's configured limit, %false if it doesn't.
  */
-bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
+bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages,
+                            gfp_t gfp_mask)
 {
-       gfp_t gfp_mask = GFP_KERNEL;
-
        if (!cgroup_subsys_on_dfl(memory_cgrp_subsys)) {
                struct page_counter *fail;
 
@@ -7007,21 +7042,19 @@ bool mem_cgroup_charge_skmem(struct mem_cgroup *memcg, unsigned int nr_pages)
                        memcg->tcpmem_pressure = 0;
                        return true;
                }
-               page_counter_charge(&memcg->tcpmem, nr_pages);
                memcg->tcpmem_pressure = 1;
+               if (gfp_mask & __GFP_NOFAIL) {
+                       page_counter_charge(&memcg->tcpmem, nr_pages);
+                       return true;
+               }
                return false;
        }
 
-       /* Don't block in the packet receive path */
-       if (in_softirq())
-               gfp_mask = GFP_NOWAIT;
-
-       mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
-
-       if (try_charge(memcg, gfp_mask, nr_pages) == 0)
+       if (try_charge(memcg, gfp_mask, nr_pages) == 0) {
+               mod_memcg_state(memcg, MEMCG_SOCK, nr_pages);
                return true;
+       }
 
-       try_charge(memcg, gfp_mask|__GFP_NOFAIL, nr_pages);
        return false;
 }