mm, slub: call deactivate_slab() without disabling irqs

The function is now safe to be called with irqs enabled, so move the calls
outside of irq disabled sections.

When called from ___slab_alloc() -> flush_slab() we have irqs disabled, so to
reenable them before deactivate_slab() we need to open-code flush_slab() in
___slab_alloc() and reenable irqs after modifying the kmem_cache_cpu fields.
But that means a IRQ handler meanwhile might have assigned a new page to
kmem_cache_cpu.page so we have to retry the whole check.

The remaining callers of flush_slab() are the IPI handler which has disabled
irqs anyway, and slub_cpu_dead() which will be dealt with in the following
patch.

Signed-off-by: Vlastimil Babka <vbabka@suse.cz>

diff --git a/mm/slub.c b/mm/slub.c
index 6deb408..cb12a07 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -2779,8 +2779,8 @@ deactivate_slab:
        freelist = c->freelist;
        c->page = NULL;
        c->freelist = NULL;
-       deactivate_slab(s, page, freelist);
        local_irq_restore(flags);
+       deactivate_slab(s, page, freelist);
 
 new_slab:
 
@@ -2848,18 +2848,32 @@ check_new_page:
                 */
                goto return_single;
 
+retry_load_page:
+
        local_irq_save(flags);
-       if (unlikely(c->page))
-               flush_slab(s, c);
+       if (unlikely(c->page)) {
+               void *flush_freelist = c->freelist;
+               struct page *flush_page = c->page;
+
+               c->page = NULL;
+               c->freelist = NULL;
+               c->tid = next_tid(c->tid);
+
+               local_irq_restore(flags);
+
+               deactivate_slab(s, flush_page, flush_freelist);
+
+               stat(s, CPUSLAB_FLUSH);
+
+               goto retry_load_page;
+       }
        c->page = page;
 
        goto load_freelist;
 
 return_single:
 
-       local_irq_save(flags);
        deactivate_slab(s, page, get_freepointer(s, freelist));
-       local_irq_restore(flags);
        return freelist;
 }