Merge tag 'pwm/for-5.13-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/thierry...

[linux-2.6-microblaze.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 4e212cd..df45c43 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -3216,6 +3216,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_
        void *ptr;
        int slab_node = numa_mem_id();
        struct obj_cgroup *objcg = NULL;
+       bool init = false;
 
        flags &= gfp_allowed_mask;
        cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags);
@@ -3254,12 +3255,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_
   out:
        local_irq_restore(save_flags);
        ptr = cache_alloc_debugcheck_after(cachep, flags, ptr, caller);
-
-       if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
-               memset(ptr, 0, cachep->object_size);
+       init = slab_want_init_on_alloc(flags, cachep);
 
 out_hooks:
-       slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
+       slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr, init);
        return ptr;
 }
 
@@ -3301,6 +3300,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo
        unsigned long save_flags;
        void *objp;
        struct obj_cgroup *objcg = NULL;
+       bool init = false;
 
        flags &= gfp_allowed_mask;
        cachep = slab_pre_alloc_hook(cachep, &objcg, 1, flags);
@@ -3317,12 +3317,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned lo
        local_irq_restore(save_flags);
        objp = cache_alloc_debugcheck_after(cachep, flags, objp, caller);
        prefetchw(objp);
-
-       if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
-               memset(objp, 0, cachep->object_size);
+       init = slab_want_init_on_alloc(flags, cachep);
 
 out:
-       slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
+       slab_post_alloc_hook(cachep, objcg, flags, 1, &objp, init);
        return objp;
 }
 
@@ -3427,17 +3425,24 @@ free_done:
 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
                                         unsigned long caller)
 {
+       bool init;
+
        if (is_kfence_address(objp)) {
                kmemleak_free_recursive(objp, cachep->flags);
                __kfence_free(objp);
                return;
        }
 
-       if (unlikely(slab_want_init_on_free(cachep)))
+       /*
+        * As memory initialization might be integrated into KASAN,
+        * kasan_slab_free and initialization memset must be
+        * kept together to avoid discrepancies in behavior.
+        */
+       init = slab_want_init_on_free(cachep);
+       if (init && !kasan_has_integrated_init())
                memset(objp, 0, cachep->object_size);
-
-       /* Put the object into the quarantine, don't touch it for now. */
-       if (kasan_slab_free(cachep, objp))
+       /* KASAN might put objp into memory quarantine, delaying its reuse. */
+       if (kasan_slab_free(cachep, objp, init))
                return;
 
        /* Use KCSAN to help debug racy use-after-free. */
@@ -3542,18 +3547,18 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 
        cache_alloc_debugcheck_after_bulk(s, flags, size, p, _RET_IP_);
 
-       /* Clear memory outside IRQ disabled section */
-       if (unlikely(slab_want_init_on_alloc(flags, s)))
-               for (i = 0; i < size; i++)
-                       memset(p[i], 0, s->object_size);
-
-       slab_post_alloc_hook(s, objcg, flags, size, p);
+       /*
+        * memcg and kmem_cache debug support and memory initialization.
+        * Done outside of the IRQ disabled section.
+        */
+       slab_post_alloc_hook(s, objcg, flags, size, p,
+                               slab_want_init_on_alloc(flags, s));
        /* FIXME: Trace call missing. Christoph would like a bulk variant */
        return size;
 error:
        local_irq_enable();
        cache_alloc_debugcheck_after_bulk(s, flags, i, p, _RET_IP_);
-       slab_post_alloc_hook(s, objcg, flags, i, p);
+       slab_post_alloc_hook(s, objcg, flags, i, p, false);
        __kmem_cache_free_bulk(s, i, p);
        return 0;
 }