mm, slab/slub: introduce kmalloc-reclaimable caches

[linux-2.6-microblaze.git] / mm / slab_common.c

diff --git a/mm/slab_common.c b/mm/slab_common.c
index fea3376..5b19439 100644 (file)
--- a/mm/slab_common.c
+++ b/mm/slab_common.c
@@ -973,14 +973,10 @@ struct kmem_cache *__init create_kmalloc_cache(const char *name,
        return s;
 }
 
-struct kmem_cache *kmalloc_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
+struct kmem_cache *
+kmalloc_caches[NR_KMALLOC_TYPES][KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
 EXPORT_SYMBOL(kmalloc_caches);
 
-#ifdef CONFIG_ZONE_DMA
-struct kmem_cache *kmalloc_dma_caches[KMALLOC_SHIFT_HIGH + 1] __ro_after_init;
-EXPORT_SYMBOL(kmalloc_dma_caches);
-#endif
-
 /*
  * Conversion table for small slabs sizes / 8 to the index in the
  * kmalloc array. This is necessary for slabs < 192 since we have non power
@@ -1027,25 +1023,20 @@ struct kmem_cache *kmalloc_slab(size_t size, gfp_t flags)
 {
        unsigned int index;
 
-       if (unlikely(size > KMALLOC_MAX_SIZE)) {
-               WARN_ON_ONCE(!(flags & __GFP_NOWARN));
-               return NULL;
-       }
-
        if (size <= 192) {
                if (!size)
                        return ZERO_SIZE_PTR;
 
                index = size_index[size_index_elem(size)];
-       } else
+       } else {
+               if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
+                       WARN_ON(1);
+                       return NULL;
+               }
                index = fls(size - 1);
+       }
 
-#ifdef CONFIG_ZONE_DMA
-       if (unlikely((flags & GFP_DMA)))
-               return kmalloc_dma_caches[index];
-
-#endif
-       return kmalloc_caches[index];
+       return kmalloc_caches[kmalloc_type(flags)][index];
 }
 
 /*
@@ -1117,9 +1108,21 @@ void __init setup_kmalloc_cache_index_table(void)
        }
 }
 
-static void __init new_kmalloc_cache(int idx, slab_flags_t flags)
+static void __init
+new_kmalloc_cache(int idx, int type, slab_flags_t flags)
 {
-       kmalloc_caches[idx] = create_kmalloc_cache(kmalloc_info[idx].name,
+       const char *name;
+
+       if (type == KMALLOC_RECLAIM) {
+               flags |= SLAB_RECLAIM_ACCOUNT;
+               name = kasprintf(GFP_NOWAIT, "kmalloc-rcl-%u",
+                                               kmalloc_info[idx].size);
+               BUG_ON(!name);
+       } else {
+               name = kmalloc_info[idx].name;
+       }
+
+       kmalloc_caches[type][idx] = create_kmalloc_cache(name,
                                        kmalloc_info[idx].size, flags, 0,
                                        kmalloc_info[idx].size);
 }
@@ -1131,21 +1134,25 @@ static void __init new_kmalloc_cache(int idx, slab_flags_t flags)
  */
 void __init create_kmalloc_caches(slab_flags_t flags)
 {
-       int i;
+       int i, type;
 
-       for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
-               if (!kmalloc_caches[i])
-                       new_kmalloc_cache(i, flags);
+       for (type = KMALLOC_NORMAL; type <= KMALLOC_RECLAIM; type++) {
+               for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++) {
+                       if (!kmalloc_caches[type][i])
+                               new_kmalloc_cache(i, type, flags);
 
-               /*
-                * Caches that are not of the two-to-the-power-of size.
-                * These have to be created immediately after the
-                * earlier power of two caches
-                */
-               if (KMALLOC_MIN_SIZE <= 32 && !kmalloc_caches[1] && i == 6)
-                       new_kmalloc_cache(1, flags);
-               if (KMALLOC_MIN_SIZE <= 64 && !kmalloc_caches[2] && i == 7)
-                       new_kmalloc_cache(2, flags);
+                       /*
+                        * Caches that are not of the two-to-the-power-of size.
+                        * These have to be created immediately after the
+                        * earlier power of two caches
+                        */
+                       if (KMALLOC_MIN_SIZE <= 32 && i == 6 &&
+                                       !kmalloc_caches[type][1])
+                               new_kmalloc_cache(1, type, flags);
+                       if (KMALLOC_MIN_SIZE <= 64 && i == 7 &&
+                                       !kmalloc_caches[type][2])
+                               new_kmalloc_cache(2, type, flags);
+               }
        }
 
        /* Kmalloc array is now usable */
@@ -1153,7 +1160,7 @@ void __init create_kmalloc_caches(slab_flags_t flags)
 
 #ifdef CONFIG_ZONE_DMA
        for (i = 0; i <= KMALLOC_SHIFT_HIGH; i++) {
-               struct kmem_cache *s = kmalloc_caches[i];
+               struct kmem_cache *s = kmalloc_caches[KMALLOC_NORMAL][i];
 
                if (s) {
                        unsigned int size = kmalloc_size(i);
@@ -1161,8 +1168,8 @@ void __init create_kmalloc_caches(slab_flags_t flags)
                                 "dma-kmalloc-%u", size);
 
                        BUG_ON(!n);
-                       kmalloc_dma_caches[i] = create_kmalloc_cache(n,
-                               size, SLAB_CACHE_DMA | flags, 0, 0);
+                       kmalloc_caches[KMALLOC_DMA][i] = create_kmalloc_cache(
+                               n, size, SLAB_CACHE_DMA | flags, 0, 0);
                }
        }
 #endif