mm: slub: call account_slab_page() after slab page initialization

[linux-2.6-microblaze.git] / mm / slub.c

diff --git a/mm/slub.c b/mm/slub.c
index 4552319..dc5b42e 100644 (file)
--- a/mm/slub.c
+++ b/mm/slub.c
@@ -249,7 +249,7 @@ static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
 {
 #ifdef CONFIG_SLAB_FREELIST_HARDENED
        /*
-        * When CONFIG_KASAN_SW_TAGS is enabled, ptr_addr might be tagged.
+        * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
         * Normally, this doesn't cause any issues, as both set_freepointer()
         * and get_freepointer() are called with a pointer with the same tag.
         * However, there are some issues with CONFIG_SLUB_DEBUG code. For
@@ -275,6 +275,7 @@ static inline void *freelist_dereference(const struct kmem_cache *s,
 
 static inline void *get_freepointer(struct kmem_cache *s, void *object)
 {
+       object = kasan_reset_tag(object);
        return freelist_dereference(s, object + s->offset);
 }
 
@@ -304,6 +305,7 @@ static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
        BUG_ON(object == fp); /* naive detection of double free or corruption */
 #endif
 
+       freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
        *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
 }
 
@@ -538,8 +540,8 @@ static void print_section(char *level, char *text, u8 *addr,
                          unsigned int length)
 {
        metadata_access_enable();
-       print_hex_dump(level, text, DUMP_PREFIX_ADDRESS, 16, 1, addr,
-                       length, 1);
+       print_hex_dump(level, kasan_reset_tag(text), DUMP_PREFIX_ADDRESS,
+                       16, 1, addr, length, 1);
        metadata_access_disable();
 }
 
@@ -570,7 +572,7 @@ static struct track *get_track(struct kmem_cache *s, void *object,
 
        p = object + get_info_end(s);
 
-       return p + alloc;
+       return kasan_reset_tag(p + alloc);
 }
 
 static void set_track(struct kmem_cache *s, void *object,
@@ -583,7 +585,8 @@ static void set_track(struct kmem_cache *s, void *object,
                unsigned int nr_entries;
 
                metadata_access_enable();
-               nr_entries = stack_trace_save(p->addrs, TRACK_ADDRS_COUNT, 3);
+               nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
+                                             TRACK_ADDRS_COUNT, 3);
                metadata_access_disable();
 
                if (nr_entries < TRACK_ADDRS_COUNT)
@@ -747,7 +750,7 @@ static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
 
 static void init_object(struct kmem_cache *s, void *object, u8 val)
 {
-       u8 *p = object;
+       u8 *p = kasan_reset_tag(object);
 
        if (s->flags & SLAB_RED_ZONE)
                memset(p - s->red_left_pad, val, s->red_left_pad);
@@ -777,7 +780,7 @@ static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
        u8 *addr = page_address(page);
 
        metadata_access_enable();
-       fault = memchr_inv(start, value, bytes);
+       fault = memchr_inv(kasan_reset_tag(start), value, bytes);
        metadata_access_disable();
        if (!fault)
                return 1;
@@ -873,7 +876,7 @@ static int slab_pad_check(struct kmem_cache *s, struct page *page)
 
        pad = end - remainder;
        metadata_access_enable();
-       fault = memchr_inv(pad, POISON_INUSE, remainder);
+       fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
        metadata_access_disable();
        if (!fault)
                return 1;
@@ -1118,7 +1121,7 @@ void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr)
                return;
 
        metadata_access_enable();
-       memset(addr, POISON_INUSE, page_size(page));
+       memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page));
        metadata_access_disable();
 }
 
@@ -1566,10 +1569,10 @@ static inline bool slab_free_freelist_hook(struct kmem_cache *s,
                         * Clear the object and the metadata, but don't touch
                         * the redzone.
                         */
-                       memset(object, 0, s->object_size);
+                       memset(kasan_reset_tag(object), 0, s->object_size);
                        rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad
                                                           : 0;
-                       memset((char *)object + s->inuse, 0,
+                       memset((char *)kasan_reset_tag(object) + s->inuse, 0,
                               s->size - s->inuse - rsize);
 
                }
@@ -1616,9 +1619,6 @@ static inline struct page *alloc_slab_page(struct kmem_cache *s,
        else
                page = __alloc_pages_node(node, flags, order);
 
-       if (page)
-               account_slab_page(page, order, s);
-
        return page;
 }
 
@@ -1771,6 +1771,8 @@ static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
 
        page->objects = oo_objects(oo);
 
+       account_slab_page(page, oo_order(oo), s);
+
        page->slab_cache = s;
        __SetPageSlab(page);
        if (page_is_pfmemalloc(page))
@@ -2881,10 +2883,10 @@ redo:
                stat(s, ALLOC_FASTPATH);
        }
 
-       maybe_wipe_obj_freeptr(s, object);
+       maybe_wipe_obj_freeptr(s, kasan_reset_tag(object));
 
        if (unlikely(slab_want_init_on_alloc(gfpflags, s)) && object)
-               memset(object, 0, s->object_size);
+               memset(kasan_reset_tag(object), 0, s->object_size);
 
        slab_post_alloc_hook(s, objcg, gfpflags, 1, &object);