rbd: introduce rbd_is_snap()

[linux-2.6-microblaze.git] / mm / slob.c

diff --git a/mm/slob.c b/mm/slob.c
index cf377be..fa53e9f 100644 (file)
--- a/mm/slob.c
+++ b/mm/slob.c
@@ -190,7 +190,7 @@ static int slob_last(slob_t *s)
 
 static void *slob_new_pages(gfp_t gfp, int order, int node)
 {
-       void *page;
+       struct page *page;
 
 #ifdef CONFIG_NUMA
        if (node != NUMA_NO_NODE)
@@ -202,14 +202,21 @@ static void *slob_new_pages(gfp_t gfp, int order, int node)
        if (!page)
                return NULL;
 
+       mod_node_page_state(page_pgdat(page), NR_SLAB_UNRECLAIMABLE,
+                           1 << order);
        return page_address(page);
 }
 
 static void slob_free_pages(void *b, int order)
 {
+       struct page *sp = virt_to_page(b);
+
        if (current->reclaim_state)
                current->reclaim_state->reclaimed_slab += 1 << order;
-       free_pages((unsigned long)b, order);
+
+       mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE,
+                           -(1 << order));
+       __free_pages(sp, order);
 }
 
 /*
@@ -217,6 +224,7 @@ static void slob_free_pages(void *b, int order)
  * @sp: Page to look in.
  * @size: Size of the allocation.
  * @align: Allocation alignment.
+ * @align_offset: Offset in the allocated block that will be aligned.
  * @page_removed_from_list: Return parameter.
  *
  * Tries to find a chunk of memory at least @size bytes big within @page.
@@ -227,7 +235,7 @@ static void slob_free_pages(void *b, int order)
  *         true (set to false otherwise).
  */
 static void *slob_page_alloc(struct page *sp, size_t size, int align,
-                            bool *page_removed_from_list)
+                             int align_offset, bool *page_removed_from_list)
 {
        slob_t *prev, *cur, *aligned = NULL;
        int delta = 0, units = SLOB_UNITS(size);
@@ -236,8 +244,17 @@ static void *slob_page_alloc(struct page *sp, size_t size, int align,
        for (prev = NULL, cur = sp->freelist; ; prev = cur, cur = slob_next(cur)) {
                slobidx_t avail = slob_units(cur);
 
+               /*
+                * 'aligned' will hold the address of the slob block so that the
+                * address 'aligned'+'align_offset' is aligned according to the
+                * 'align' parameter. This is for kmalloc() which prepends the
+                * allocated block with its size, so that the block itself is
+                * aligned when needed.
+                */
                if (align) {
-                       aligned = (slob_t *)ALIGN((unsigned long)cur, align);
+                       aligned = (slob_t *)
+                               (ALIGN((unsigned long)cur + align_offset, align)
+                                - align_offset);
                        delta = aligned - cur;
                }
                if (avail >= units + delta) { /* room enough? */
@@ -281,7 +298,8 @@ static void *slob_page_alloc(struct page *sp, size_t size, int align,
 /*
  * slob_alloc: entry point into the slob allocator.
  */
-static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
+static void *slob_alloc(size_t size, gfp_t gfp, int align, int node,
+                                                       int align_offset)
 {
        struct page *sp;
        struct list_head *slob_list;
@@ -312,7 +330,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
                if (sp->units < SLOB_UNITS(size))
                        continue;
 
-               b = slob_page_alloc(sp, size, align, &page_removed_from_list);
+               b = slob_page_alloc(sp, size, align, align_offset, &page_removed_from_list);
                if (!b)
                        continue;
 
@@ -349,7 +367,7 @@ static void *slob_alloc(size_t size, gfp_t gfp, int align, int node)
                INIT_LIST_HEAD(&sp->slab_list);
                set_slob(b, SLOB_UNITS(PAGE_SIZE), b + SLOB_UNITS(PAGE_SIZE));
                set_slob_page_free(sp, slob_list);
-               b = slob_page_alloc(sp, size, align, &_unused);
+               b = slob_page_alloc(sp, size, align, align_offset, &_unused);
                BUG_ON(!b);
                spin_unlock_irqrestore(&slob_lock, flags);
        }
@@ -451,7 +469,7 @@ static __always_inline void *
 __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
 {
        unsigned int *m;
-       int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
+       int minalign = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
        void *ret;
 
        gfp &= gfp_allowed_mask;
@@ -459,19 +477,28 @@ __do_kmalloc_node(size_t size, gfp_t gfp, int node, unsigned long caller)
        fs_reclaim_acquire(gfp);
        fs_reclaim_release(gfp);
 
-       if (size < PAGE_SIZE - align) {
+       if (size < PAGE_SIZE - minalign) {
+               int align = minalign;
+
+               /*
+                * For power of two sizes, guarantee natural alignment for
+                * kmalloc()'d objects.
+                */
+               if (is_power_of_2(size))
+                       align = max(minalign, (int) size);
+
                if (!size)
                        return ZERO_SIZE_PTR;
 
-               m = slob_alloc(size + align, gfp, align, node);
+               m = slob_alloc(size + minalign, gfp, align, node, minalign);
 
                if (!m)
                        return NULL;
                *m = size;
-               ret = (void *)m + align;
+               ret = (void *)m + minalign;
 
                trace_kmalloc_node(caller, ret,
-                                  size, size + align, gfp, node);
+                                  size, size + minalign, gfp, node);
        } else {
                unsigned int order = get_order(size);
 
@@ -521,8 +548,13 @@ void kfree(const void *block)
                int align = max_t(size_t, ARCH_KMALLOC_MINALIGN, ARCH_SLAB_MINALIGN);
                unsigned int *m = (unsigned int *)(block - align);
                slob_free(m, *m + align);
-       } else
-               __free_pages(sp, compound_order(sp));
+       } else {
+               unsigned int order = compound_order(sp);
+               mod_node_page_state(page_pgdat(sp), NR_SLAB_UNRECLAIMABLE,
+                                   -(1 << order));
+               __free_pages(sp, order);
+
+       }
 }
 EXPORT_SYMBOL(kfree);
 
@@ -567,7 +599,7 @@ static void *slob_alloc_node(struct kmem_cache *c, gfp_t flags, int node)
        fs_reclaim_release(flags);
 
        if (c->size < PAGE_SIZE) {
-               b = slob_alloc(c->size, flags, c->align, node);
+               b = slob_alloc(c->size, flags, c->align, node, 0);
                trace_kmem_cache_alloc_node(_RET_IP_, b, c->object_size,
                                            SLOB_UNITS(c->size) * SLOB_UNIT,
                                            flags, node);