perf map: Tighten snprintf() string precision to pass gcc check on some 32-bit arches

[linux-2.6-microblaze.git] / mm / slab.c

diff --git a/mm/slab.c b/mm/slab.c
index 35c68d9..51fd424 100644 (file)
--- a/mm/slab.c
+++ b/mm/slab.c
@@ -100,6 +100,7 @@
 #include       <linux/seq_file.h>
 #include       <linux/notifier.h>
 #include       <linux/kallsyms.h>
+#include       <linux/kfence.h>
 #include       <linux/cpu.h>
 #include       <linux/sysctl.h>
 #include       <linux/module.h>
@@ -3208,7 +3209,7 @@ must_grow:
 }
 
 static __always_inline void *
-slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
+slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid, size_t orig_size,
                   unsigned long caller)
 {
        unsigned long save_flags;
@@ -3221,6 +3222,10 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        if (unlikely(!cachep))
                return NULL;
 
+       ptr = kfence_alloc(cachep, orig_size, flags);
+       if (unlikely(ptr))
+               goto out_hooks;
+
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
 
@@ -3253,6 +3258,7 @@ slab_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid,
        if (unlikely(slab_want_init_on_alloc(flags, cachep)) && ptr)
                memset(ptr, 0, cachep->object_size);
 
+out_hooks:
        slab_post_alloc_hook(cachep, objcg, flags, 1, &ptr);
        return ptr;
 }
@@ -3290,7 +3296,7 @@ __do_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 #endif /* CONFIG_NUMA */
 
 static __always_inline void *
-slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
+slab_alloc(struct kmem_cache *cachep, gfp_t flags, size_t orig_size, unsigned long caller)
 {
        unsigned long save_flags;
        void *objp;
@@ -3301,6 +3307,10 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
        if (unlikely(!cachep))
                return NULL;
 
+       objp = kfence_alloc(cachep, orig_size, flags);
+       if (unlikely(objp))
+               goto out;
+
        cache_alloc_debugcheck_before(cachep, flags);
        local_irq_save(save_flags);
        objp = __do_cache_alloc(cachep, flags);
@@ -3311,6 +3321,7 @@ slab_alloc(struct kmem_cache *cachep, gfp_t flags, unsigned long caller)
        if (unlikely(slab_want_init_on_alloc(flags, cachep)) && objp)
                memset(objp, 0, cachep->object_size);
 
+out:
        slab_post_alloc_hook(cachep, objcg, flags, 1, &objp);
        return objp;
 }
@@ -3416,6 +3427,12 @@ free_done:
 static __always_inline void __cache_free(struct kmem_cache *cachep, void *objp,
                                         unsigned long caller)
 {
+       if (is_kfence_address(objp)) {
+               kmemleak_free_recursive(objp, cachep->flags);
+               __kfence_free(objp);
+               return;
+       }
+
        if (unlikely(slab_want_init_on_free(cachep)))
                memset(objp, 0, cachep->object_size);
 
@@ -3482,7 +3499,7 @@ void ___cache_free(struct kmem_cache *cachep, void *objp,
  */
 void *kmem_cache_alloc(struct kmem_cache *cachep, gfp_t flags)
 {
-       void *ret = slab_alloc(cachep, flags, _RET_IP_);
+       void *ret = slab_alloc(cachep, flags, cachep->object_size, _RET_IP_);
 
        trace_kmem_cache_alloc(_RET_IP_, ret,
                               cachep->object_size, cachep->size, flags);
@@ -3515,7 +3532,7 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
 
        local_irq_disable();
        for (i = 0; i < size; i++) {
-               void *objp = __do_cache_alloc(s, flags);
+               void *objp = kfence_alloc(s, s->object_size, flags) ?: __do_cache_alloc(s, flags);
 
                if (unlikely(!objp))
                        goto error;
@@ -3548,7 +3565,7 @@ kmem_cache_alloc_trace(struct kmem_cache *cachep, gfp_t flags, size_t size)
 {
        void *ret;
 
-       ret = slab_alloc(cachep, flags, _RET_IP_);
+       ret = slab_alloc(cachep, flags, size, _RET_IP_);
 
        ret = kasan_kmalloc(cachep, ret, size, flags);
        trace_kmalloc(_RET_IP_, ret,
@@ -3574,7 +3591,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace);
  */
 void *kmem_cache_alloc_node(struct kmem_cache *cachep, gfp_t flags, int nodeid)
 {
-       void *ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
+       void *ret = slab_alloc_node(cachep, flags, nodeid, cachep->object_size, _RET_IP_);
 
        trace_kmem_cache_alloc_node(_RET_IP_, ret,
                                    cachep->object_size, cachep->size,
@@ -3592,7 +3609,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *cachep,
 {
        void *ret;
 
-       ret = slab_alloc_node(cachep, flags, nodeid, _RET_IP_);
+       ret = slab_alloc_node(cachep, flags, nodeid, size, _RET_IP_);
 
        ret = kasan_kmalloc(cachep, ret, size, flags);
        trace_kmalloc_node(_RET_IP_, ret,
@@ -3673,7 +3690,7 @@ static __always_inline void *__do_kmalloc(size_t size, gfp_t flags,
        cachep = kmalloc_slab(size, flags);
        if (unlikely(ZERO_OR_NULL_PTR(cachep)))
                return cachep;
-       ret = slab_alloc(cachep, flags, caller);
+       ret = slab_alloc(cachep, flags, size, caller);
 
        ret = kasan_kmalloc(cachep, ret, size, flags);
        trace_kmalloc(caller, ret,
@@ -4172,7 +4189,10 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
        BUG_ON(objnr >= cachep->num);
 
        /* Find offset within object. */
-       offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
+       if (is_kfence_address(ptr))
+               offset = ptr - kfence_object_start(ptr);
+       else
+               offset = ptr - index_to_obj(cachep, page, objnr) - obj_offset(cachep);
 
        /* Allow address range falling entirely within usercopy region. */
        if (offset >= cachep->useroffset &&