arm64: Introduce prctl(PR_PAC_{SET,GET}_ENABLED_KEYS)

[linux-2.6-microblaze.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index b2833ce..3021ce9 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -27,6 +27,7 @@
 #include <linux/ctype.h>
 #include <linux/debugobjects.h>
 #include <linux/kallsyms.h>
+#include <linux/kfence.h>
 #include <linux/memory.h>
 #include <linux/math64.h>
 #include <linux/fault-inject.h>
@@ -1570,6 +1571,11 @@ static inline bool slab_free_freelist_hook(struct kmem_cache *s,
        void *old_tail = *tail ? *tail : *head;
        int rsize;
 
+       if (is_kfence_address(next)) {
+               slab_free_hook(s, next);
+               return true;
+       }
+
        /* Head and tail of the reconstructed freelist */
        *head = NULL;
        *tail = NULL;
@@ -1987,7 +1993,7 @@ static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
 
                t = acquire_slab(s, n, page, object == NULL, &objects);
                if (!t)
-                       continue; /* cmpxchg raced */
+                       break;
 
                available += objects;
                if (!object) {
@@ -2809,7 +2815,7 @@ static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
  * Otherwise we can simply pick the next object from the lockless free list.
  */
 static __always_inline void *slab_alloc_node(struct kmem_cache *s,
-               gfp_t gfpflags, int node, unsigned long addr)
+               gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
 {
        void *object;
        struct kmem_cache_cpu *c;
@@ -2820,6 +2826,11 @@ static __always_inline void *slab_alloc_node(struct kmem_cache *s,
        s = slab_pre_alloc_hook(s, &objcg, 1, gfpflags);
        if (!s)
                return NULL;
+
+       object = kfence_alloc(s, orig_size, gfpflags);
+       if (unlikely(object))
+               goto out;
+
 redo:
        /*
         * Must read kmem_cache cpu data via this cpu ptr. Preemption is
@@ -2892,20 +2903,21 @@ redo:
        if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object)
                memset(kasan_reset_tag(object), 0, s->object_size);
 
+out:
        slab_post_alloc_hook(s, objcg, gfpflags, 1, &object);
 
        return object;
 }
 
 static __always_inline void *slab_alloc(struct kmem_cache *s,
-               gfp_t gfpflags, unsigned long addr)
+               gfp_t gfpflags, unsigned long addr, size_t orig_size)
 {
-       return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr);
+       return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr, orig_size);
 }
 
 void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
 {
-       void *ret = slab_alloc(s, gfpflags, _RET_IP_);
+       void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size);
 
        trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
                                s->size, gfpflags);
@@ -2917,7 +2929,7 @@ EXPORT_SYMBOL(kmem_cache_alloc);
 #ifdef CONFIG_TRACING
 void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
 {
-       void *ret = slab_alloc(s, gfpflags, _RET_IP_);
+       void *ret = slab_alloc(s, gfpflags, _RET_IP_, size);
        trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
        ret = kasan_kmalloc(s, ret, size, gfpflags);
        return ret;
@@ -2928,7 +2940,7 @@ EXPORT_SYMBOL(kmem_cache_alloc_trace);
 #ifdef CONFIG_NUMA
 void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
 {
-       void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
+       void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size);
 
        trace_kmem_cache_alloc_node(_RET_IP_, ret,
                                    s->object_size, s->size, gfpflags, node);
@@ -2942,7 +2954,7 @@ void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
                                    gfp_t gfpflags,
                                    int node, size_t size)
 {
-       void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_);
+       void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size);
 
        trace_kmalloc_node(_RET_IP_, ret,
                           size, s->size, gfpflags, node);
@@ -2976,6 +2988,9 @@ static void __slab_free(struct kmem_cache *s, struct page *page,
 
        stat(s, FREE_SLOWPATH);
 
+       if (kfence_free(head))
+               return;
+
        if (kmem_cache_debug(s) &&
            !free_debug_processing(s, page, head, tail, cnt, addr))
                return;
@@ -3220,6 +3235,13 @@ int build_detached_freelist(struct kmem_cache *s, size_t size,
                df->s = cache_from_obj(s, object); /* Support for memcg */
        }
 
+       if (is_kfence_address(object)) {
+               slab_free_hook(df->s, object);
+               __kfence_free(object);
+               p[size] = NULL; /* mark object processed */
+               return size;
+       }
+
        /* Start new detached freelist */
        df->page = page;
        set_freepointer(df->s, object, NULL);
@@ -3295,8 +3317,14 @@ int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
        c = this_cpu_ptr(s->cpu_slab);
 
        for (i = 0; i < size; i++) {
-               void *object = c->freelist;
+               void *object = kfence_alloc(s, s->object_size, flags);
 
+               if (unlikely(object)) {
+                       p[i] = object;
+                       continue;
+               }
+
+               object = c->freelist;
                if (unlikely(!object)) {
                        /*
                         * We may have removed an object from c->freelist using
@@ -3551,8 +3579,7 @@ static void early_kmem_cache_node_alloc(int node)
        init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
        init_tracking(kmem_cache_node, n);
 #endif
-       n = kasan_kmalloc(kmem_cache_node, n, sizeof(struct kmem_cache_node),
-                     GFP_KERNEL);
+       n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL);
        page->freelist = get_freepointer(kmem_cache_node, n);
        page->inuse = 1;
        page->frozen = 0;
@@ -4021,7 +4048,7 @@ void *__kmalloc(size_t size, gfp_t flags)
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
-       ret = slab_alloc(s, flags, _RET_IP_);
+       ret = slab_alloc(s, flags, _RET_IP_, size);
 
        trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
 
@@ -4069,7 +4096,7 @@ void *__kmalloc_node(size_t size, gfp_t flags, int node)
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
-       ret = slab_alloc_node(s, flags, node, _RET_IP_);
+       ret = slab_alloc_node(s, flags, node, _RET_IP_, size);
 
        trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
 
@@ -4095,6 +4122,7 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
        struct kmem_cache *s;
        unsigned int offset;
        size_t object_size;
+       bool is_kfence = is_kfence_address(ptr);
 
        ptr = kasan_reset_tag(ptr);
 
@@ -4107,10 +4135,13 @@ void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
                               to_user, 0, n);
 
        /* Find offset within object. */
-       offset = (ptr - page_address(page)) % s->size;
+       if (is_kfence)
+               offset = ptr - kfence_object_start(ptr);
+       else
+               offset = (ptr - page_address(page)) % s->size;
 
        /* Adjust for redzone and reject if within the redzone. */
-       if (kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
+       if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
                if (offset < s->red_left_pad)
                        usercopy_abort("SLUB object in left red zone",
                                       s->name, to_user, offset, n);
@@ -4527,7 +4558,7 @@ void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
-       ret = slab_alloc(s, gfpflags, caller);
+       ret = slab_alloc(s, gfpflags, caller, size);
 
        /* Honor the call site pointer we received. */
        trace_kmalloc(caller, ret, size, s->size, gfpflags);
@@ -4558,7 +4589,7 @@ void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
        if (unlikely(ZERO_OR_NULL_PTR(s)))
                return s;
 
-       ret = slab_alloc_node(s, gfpflags, node, caller);
+       ret = slab_alloc_node(s, gfpflags, node, caller, size);
 
        /* Honor the call site pointer we received. */
        trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);