drm/sun4i: Introduce color encoding and range properties
[linux-2.6-microblaze.git] / lib / debugobjects.c
1 /*
2  * Generic infrastructure for lifetime debugging of objects.
3  *
4  * Started by Thomas Gleixner
5  *
6  * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
7  *
8  * For licencing details see kernel-base/COPYING
9  */
10
11 #define pr_fmt(fmt) "ODEBUG: " fmt
12
13 #include <linux/debugobjects.h>
14 #include <linux/interrupt.h>
15 #include <linux/sched.h>
16 #include <linux/sched/task_stack.h>
17 #include <linux/seq_file.h>
18 #include <linux/debugfs.h>
19 #include <linux/slab.h>
20 #include <linux/hash.h>
21 #include <linux/kmemleak.h>
22
23 #define ODEBUG_HASH_BITS        14
24 #define ODEBUG_HASH_SIZE        (1 << ODEBUG_HASH_BITS)
25
26 #define ODEBUG_POOL_SIZE        1024
27 #define ODEBUG_POOL_MIN_LEVEL   256
28
29 #define ODEBUG_CHUNK_SHIFT      PAGE_SHIFT
30 #define ODEBUG_CHUNK_SIZE       (1 << ODEBUG_CHUNK_SHIFT)
31 #define ODEBUG_CHUNK_MASK       (~(ODEBUG_CHUNK_SIZE - 1))
32
33 struct debug_bucket {
34         struct hlist_head       list;
35         raw_spinlock_t          lock;
36 };
37
38 static struct debug_bucket      obj_hash[ODEBUG_HASH_SIZE];
39
40 static struct debug_obj         obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
41
42 static DEFINE_RAW_SPINLOCK(pool_lock);
43
44 static HLIST_HEAD(obj_pool);
45 static HLIST_HEAD(obj_to_free);
46
47 static int                      obj_pool_min_free = ODEBUG_POOL_SIZE;
48 static int                      obj_pool_free = ODEBUG_POOL_SIZE;
49 static int                      obj_pool_used;
50 static int                      obj_pool_max_used;
51 /* The number of objs on the global free list */
52 static int                      obj_nr_tofree;
53 static struct kmem_cache        *obj_cache;
54
55 static int                      debug_objects_maxchain __read_mostly;
56 static int __maybe_unused       debug_objects_maxchecked __read_mostly;
57 static int                      debug_objects_fixups __read_mostly;
58 static int                      debug_objects_warnings __read_mostly;
59 static int                      debug_objects_enabled __read_mostly
60                                 = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
61 static int                      debug_objects_pool_size __read_mostly
62                                 = ODEBUG_POOL_SIZE;
63 static int                      debug_objects_pool_min_level __read_mostly
64                                 = ODEBUG_POOL_MIN_LEVEL;
65 static struct debug_obj_descr   *descr_test  __read_mostly;
66
67 /*
68  * Track numbers of kmem_cache_alloc()/free() calls done.
69  */
70 static int                      debug_objects_allocated;
71 static int                      debug_objects_freed;
72
73 static void free_obj_work(struct work_struct *work);
74 static DECLARE_WORK(debug_obj_work, free_obj_work);
75
76 static int __init enable_object_debug(char *str)
77 {
78         debug_objects_enabled = 1;
79         return 0;
80 }
81
82 static int __init disable_object_debug(char *str)
83 {
84         debug_objects_enabled = 0;
85         return 0;
86 }
87
88 early_param("debug_objects", enable_object_debug);
89 early_param("no_debug_objects", disable_object_debug);
90
91 static const char *obj_states[ODEBUG_STATE_MAX] = {
92         [ODEBUG_STATE_NONE]             = "none",
93         [ODEBUG_STATE_INIT]             = "initialized",
94         [ODEBUG_STATE_INACTIVE]         = "inactive",
95         [ODEBUG_STATE_ACTIVE]           = "active",
96         [ODEBUG_STATE_DESTROYED]        = "destroyed",
97         [ODEBUG_STATE_NOTAVAILABLE]     = "not available",
98 };
99
100 static void fill_pool(void)
101 {
102         gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
103         struct debug_obj *new, *obj;
104         unsigned long flags;
105
106         if (likely(obj_pool_free >= debug_objects_pool_min_level))
107                 return;
108
109         /*
110          * Reuse objs from the global free list; they will be reinitialized
111          * when allocating.
112          */
113         while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
114                 raw_spin_lock_irqsave(&pool_lock, flags);
115                 /*
116                  * Recheck with the lock held as the worker thread might have
117                  * won the race and freed the global free list already.
118                  */
119                 if (obj_nr_tofree) {
120                         obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
121                         hlist_del(&obj->node);
122                         obj_nr_tofree--;
123                         hlist_add_head(&obj->node, &obj_pool);
124                         obj_pool_free++;
125                 }
126                 raw_spin_unlock_irqrestore(&pool_lock, flags);
127         }
128
129         if (unlikely(!obj_cache))
130                 return;
131
132         while (obj_pool_free < debug_objects_pool_min_level) {
133
134                 new = kmem_cache_zalloc(obj_cache, gfp);
135                 if (!new)
136                         return;
137
138                 raw_spin_lock_irqsave(&pool_lock, flags);
139                 hlist_add_head(&new->node, &obj_pool);
140                 debug_objects_allocated++;
141                 obj_pool_free++;
142                 raw_spin_unlock_irqrestore(&pool_lock, flags);
143         }
144 }
145
146 /*
147  * Lookup an object in the hash bucket.
148  */
149 static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
150 {
151         struct debug_obj *obj;
152         int cnt = 0;
153
154         hlist_for_each_entry(obj, &b->list, node) {
155                 cnt++;
156                 if (obj->object == addr)
157                         return obj;
158         }
159         if (cnt > debug_objects_maxchain)
160                 debug_objects_maxchain = cnt;
161
162         return NULL;
163 }
164
165 /*
166  * Allocate a new object. If the pool is empty, switch off the debugger.
167  * Must be called with interrupts disabled.
168  */
169 static struct debug_obj *
170 alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
171 {
172         struct debug_obj *obj = NULL;
173
174         raw_spin_lock(&pool_lock);
175         if (obj_pool.first) {
176                 obj         = hlist_entry(obj_pool.first, typeof(*obj), node);
177
178                 obj->object = addr;
179                 obj->descr  = descr;
180                 obj->state  = ODEBUG_STATE_NONE;
181                 obj->astate = 0;
182                 hlist_del(&obj->node);
183
184                 hlist_add_head(&obj->node, &b->list);
185
186                 obj_pool_used++;
187                 if (obj_pool_used > obj_pool_max_used)
188                         obj_pool_max_used = obj_pool_used;
189
190                 obj_pool_free--;
191                 if (obj_pool_free < obj_pool_min_free)
192                         obj_pool_min_free = obj_pool_free;
193         }
194         raw_spin_unlock(&pool_lock);
195
196         return obj;
197 }
198
199 /*
200  * workqueue function to free objects.
201  *
202  * To reduce contention on the global pool_lock, the actual freeing of
203  * debug objects will be delayed if the pool_lock is busy.
204  */
205 static void free_obj_work(struct work_struct *work)
206 {
207         struct hlist_node *tmp;
208         struct debug_obj *obj;
209         unsigned long flags;
210         HLIST_HEAD(tofree);
211
212         if (!raw_spin_trylock_irqsave(&pool_lock, flags))
213                 return;
214
215         /*
216          * The objs on the pool list might be allocated before the work is
217          * run, so recheck if pool list it full or not, if not fill pool
218          * list from the global free list
219          */
220         while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
221                 obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
222                 hlist_del(&obj->node);
223                 hlist_add_head(&obj->node, &obj_pool);
224                 obj_pool_free++;
225                 obj_nr_tofree--;
226         }
227
228         /*
229          * Pool list is already full and there are still objs on the free
230          * list. Move remaining free objs to a temporary list to free the
231          * memory outside the pool_lock held region.
232          */
233         if (obj_nr_tofree) {
234                 hlist_move_list(&obj_to_free, &tofree);
235                 debug_objects_freed += obj_nr_tofree;
236                 obj_nr_tofree = 0;
237         }
238         raw_spin_unlock_irqrestore(&pool_lock, flags);
239
240         hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
241                 hlist_del(&obj->node);
242                 kmem_cache_free(obj_cache, obj);
243         }
244 }
245
246 static bool __free_object(struct debug_obj *obj)
247 {
248         unsigned long flags;
249         bool work;
250
251         raw_spin_lock_irqsave(&pool_lock, flags);
252         work = (obj_pool_free > debug_objects_pool_size) && obj_cache;
253         obj_pool_used--;
254
255         if (work) {
256                 obj_nr_tofree++;
257                 hlist_add_head(&obj->node, &obj_to_free);
258         } else {
259                 obj_pool_free++;
260                 hlist_add_head(&obj->node, &obj_pool);
261         }
262         raw_spin_unlock_irqrestore(&pool_lock, flags);
263         return work;
264 }
265
266 /*
267  * Put the object back into the pool and schedule work to free objects
268  * if necessary.
269  */
270 static void free_object(struct debug_obj *obj)
271 {
272         if (__free_object(obj))
273                 schedule_work(&debug_obj_work);
274 }
275
276 /*
277  * We run out of memory. That means we probably have tons of objects
278  * allocated.
279  */
280 static void debug_objects_oom(void)
281 {
282         struct debug_bucket *db = obj_hash;
283         struct hlist_node *tmp;
284         HLIST_HEAD(freelist);
285         struct debug_obj *obj;
286         unsigned long flags;
287         int i;
288
289         pr_warn("Out of memory. ODEBUG disabled\n");
290
291         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
292                 raw_spin_lock_irqsave(&db->lock, flags);
293                 hlist_move_list(&db->list, &freelist);
294                 raw_spin_unlock_irqrestore(&db->lock, flags);
295
296                 /* Now free them */
297                 hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
298                         hlist_del(&obj->node);
299                         free_object(obj);
300                 }
301         }
302 }
303
304 /*
305  * We use the pfn of the address for the hash. That way we can check
306  * for freed objects simply by checking the affected bucket.
307  */
308 static struct debug_bucket *get_bucket(unsigned long addr)
309 {
310         unsigned long hash;
311
312         hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
313         return &obj_hash[hash];
314 }
315
316 static void debug_print_object(struct debug_obj *obj, char *msg)
317 {
318         struct debug_obj_descr *descr = obj->descr;
319         static int limit;
320
321         if (limit < 5 && descr != descr_test) {
322                 void *hint = descr->debug_hint ?
323                         descr->debug_hint(obj->object) : NULL;
324                 limit++;
325                 WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
326                                  "object type: %s hint: %pS\n",
327                         msg, obj_states[obj->state], obj->astate,
328                         descr->name, hint);
329         }
330         debug_objects_warnings++;
331 }
332
333 /*
334  * Try to repair the damage, so we have a better chance to get useful
335  * debug output.
336  */
337 static bool
338 debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
339                    void * addr, enum debug_obj_state state)
340 {
341         if (fixup && fixup(addr, state)) {
342                 debug_objects_fixups++;
343                 return true;
344         }
345         return false;
346 }
347
348 static void debug_object_is_on_stack(void *addr, int onstack)
349 {
350         int is_on_stack;
351         static int limit;
352
353         if (limit > 4)
354                 return;
355
356         is_on_stack = object_is_on_stack(addr);
357         if (is_on_stack == onstack)
358                 return;
359
360         limit++;
361         if (is_on_stack)
362                 pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
363                          task_stack_page(current));
364         else
365                 pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
366                          task_stack_page(current));
367
368         WARN_ON(1);
369 }
370
371 static void
372 __debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
373 {
374         enum debug_obj_state state;
375         struct debug_bucket *db;
376         struct debug_obj *obj;
377         unsigned long flags;
378
379         fill_pool();
380
381         db = get_bucket((unsigned long) addr);
382
383         raw_spin_lock_irqsave(&db->lock, flags);
384
385         obj = lookup_object(addr, db);
386         if (!obj) {
387                 obj = alloc_object(addr, db, descr);
388                 if (!obj) {
389                         debug_objects_enabled = 0;
390                         raw_spin_unlock_irqrestore(&db->lock, flags);
391                         debug_objects_oom();
392                         return;
393                 }
394                 debug_object_is_on_stack(addr, onstack);
395         }
396
397         switch (obj->state) {
398         case ODEBUG_STATE_NONE:
399         case ODEBUG_STATE_INIT:
400         case ODEBUG_STATE_INACTIVE:
401                 obj->state = ODEBUG_STATE_INIT;
402                 break;
403
404         case ODEBUG_STATE_ACTIVE:
405                 debug_print_object(obj, "init");
406                 state = obj->state;
407                 raw_spin_unlock_irqrestore(&db->lock, flags);
408                 debug_object_fixup(descr->fixup_init, addr, state);
409                 return;
410
411         case ODEBUG_STATE_DESTROYED:
412                 debug_print_object(obj, "init");
413                 break;
414         default:
415                 break;
416         }
417
418         raw_spin_unlock_irqrestore(&db->lock, flags);
419 }
420
421 /**
422  * debug_object_init - debug checks when an object is initialized
423  * @addr:       address of the object
424  * @descr:      pointer to an object specific debug description structure
425  */
426 void debug_object_init(void *addr, struct debug_obj_descr *descr)
427 {
428         if (!debug_objects_enabled)
429                 return;
430
431         __debug_object_init(addr, descr, 0);
432 }
433 EXPORT_SYMBOL_GPL(debug_object_init);
434
435 /**
436  * debug_object_init_on_stack - debug checks when an object on stack is
437  *                              initialized
438  * @addr:       address of the object
439  * @descr:      pointer to an object specific debug description structure
440  */
441 void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
442 {
443         if (!debug_objects_enabled)
444                 return;
445
446         __debug_object_init(addr, descr, 1);
447 }
448 EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
449
450 /**
451  * debug_object_activate - debug checks when an object is activated
452  * @addr:       address of the object
453  * @descr:      pointer to an object specific debug description structure
454  * Returns 0 for success, -EINVAL for check failed.
455  */
456 int debug_object_activate(void *addr, struct debug_obj_descr *descr)
457 {
458         enum debug_obj_state state;
459         struct debug_bucket *db;
460         struct debug_obj *obj;
461         unsigned long flags;
462         int ret;
463         struct debug_obj o = { .object = addr,
464                                .state = ODEBUG_STATE_NOTAVAILABLE,
465                                .descr = descr };
466
467         if (!debug_objects_enabled)
468                 return 0;
469
470         db = get_bucket((unsigned long) addr);
471
472         raw_spin_lock_irqsave(&db->lock, flags);
473
474         obj = lookup_object(addr, db);
475         if (obj) {
476                 switch (obj->state) {
477                 case ODEBUG_STATE_INIT:
478                 case ODEBUG_STATE_INACTIVE:
479                         obj->state = ODEBUG_STATE_ACTIVE;
480                         ret = 0;
481                         break;
482
483                 case ODEBUG_STATE_ACTIVE:
484                         debug_print_object(obj, "activate");
485                         state = obj->state;
486                         raw_spin_unlock_irqrestore(&db->lock, flags);
487                         ret = debug_object_fixup(descr->fixup_activate, addr, state);
488                         return ret ? 0 : -EINVAL;
489
490                 case ODEBUG_STATE_DESTROYED:
491                         debug_print_object(obj, "activate");
492                         ret = -EINVAL;
493                         break;
494                 default:
495                         ret = 0;
496                         break;
497                 }
498                 raw_spin_unlock_irqrestore(&db->lock, flags);
499                 return ret;
500         }
501
502         raw_spin_unlock_irqrestore(&db->lock, flags);
503         /*
504          * We are here when a static object is activated. We
505          * let the type specific code confirm whether this is
506          * true or not. if true, we just make sure that the
507          * static object is tracked in the object tracker. If
508          * not, this must be a bug, so we try to fix it up.
509          */
510         if (descr->is_static_object && descr->is_static_object(addr)) {
511                 /* track this static object */
512                 debug_object_init(addr, descr);
513                 debug_object_activate(addr, descr);
514         } else {
515                 debug_print_object(&o, "activate");
516                 ret = debug_object_fixup(descr->fixup_activate, addr,
517                                         ODEBUG_STATE_NOTAVAILABLE);
518                 return ret ? 0 : -EINVAL;
519         }
520         return 0;
521 }
522 EXPORT_SYMBOL_GPL(debug_object_activate);
523
524 /**
525  * debug_object_deactivate - debug checks when an object is deactivated
526  * @addr:       address of the object
527  * @descr:      pointer to an object specific debug description structure
528  */
529 void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
530 {
531         struct debug_bucket *db;
532         struct debug_obj *obj;
533         unsigned long flags;
534
535         if (!debug_objects_enabled)
536                 return;
537
538         db = get_bucket((unsigned long) addr);
539
540         raw_spin_lock_irqsave(&db->lock, flags);
541
542         obj = lookup_object(addr, db);
543         if (obj) {
544                 switch (obj->state) {
545                 case ODEBUG_STATE_INIT:
546                 case ODEBUG_STATE_INACTIVE:
547                 case ODEBUG_STATE_ACTIVE:
548                         if (!obj->astate)
549                                 obj->state = ODEBUG_STATE_INACTIVE;
550                         else
551                                 debug_print_object(obj, "deactivate");
552                         break;
553
554                 case ODEBUG_STATE_DESTROYED:
555                         debug_print_object(obj, "deactivate");
556                         break;
557                 default:
558                         break;
559                 }
560         } else {
561                 struct debug_obj o = { .object = addr,
562                                        .state = ODEBUG_STATE_NOTAVAILABLE,
563                                        .descr = descr };
564
565                 debug_print_object(&o, "deactivate");
566         }
567
568         raw_spin_unlock_irqrestore(&db->lock, flags);
569 }
570 EXPORT_SYMBOL_GPL(debug_object_deactivate);
571
572 /**
573  * debug_object_destroy - debug checks when an object is destroyed
574  * @addr:       address of the object
575  * @descr:      pointer to an object specific debug description structure
576  */
577 void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
578 {
579         enum debug_obj_state state;
580         struct debug_bucket *db;
581         struct debug_obj *obj;
582         unsigned long flags;
583
584         if (!debug_objects_enabled)
585                 return;
586
587         db = get_bucket((unsigned long) addr);
588
589         raw_spin_lock_irqsave(&db->lock, flags);
590
591         obj = lookup_object(addr, db);
592         if (!obj)
593                 goto out_unlock;
594
595         switch (obj->state) {
596         case ODEBUG_STATE_NONE:
597         case ODEBUG_STATE_INIT:
598         case ODEBUG_STATE_INACTIVE:
599                 obj->state = ODEBUG_STATE_DESTROYED;
600                 break;
601         case ODEBUG_STATE_ACTIVE:
602                 debug_print_object(obj, "destroy");
603                 state = obj->state;
604                 raw_spin_unlock_irqrestore(&db->lock, flags);
605                 debug_object_fixup(descr->fixup_destroy, addr, state);
606                 return;
607
608         case ODEBUG_STATE_DESTROYED:
609                 debug_print_object(obj, "destroy");
610                 break;
611         default:
612                 break;
613         }
614 out_unlock:
615         raw_spin_unlock_irqrestore(&db->lock, flags);
616 }
617 EXPORT_SYMBOL_GPL(debug_object_destroy);
618
619 /**
620  * debug_object_free - debug checks when an object is freed
621  * @addr:       address of the object
622  * @descr:      pointer to an object specific debug description structure
623  */
624 void debug_object_free(void *addr, struct debug_obj_descr *descr)
625 {
626         enum debug_obj_state state;
627         struct debug_bucket *db;
628         struct debug_obj *obj;
629         unsigned long flags;
630
631         if (!debug_objects_enabled)
632                 return;
633
634         db = get_bucket((unsigned long) addr);
635
636         raw_spin_lock_irqsave(&db->lock, flags);
637
638         obj = lookup_object(addr, db);
639         if (!obj)
640                 goto out_unlock;
641
642         switch (obj->state) {
643         case ODEBUG_STATE_ACTIVE:
644                 debug_print_object(obj, "free");
645                 state = obj->state;
646                 raw_spin_unlock_irqrestore(&db->lock, flags);
647                 debug_object_fixup(descr->fixup_free, addr, state);
648                 return;
649         default:
650                 hlist_del(&obj->node);
651                 raw_spin_unlock_irqrestore(&db->lock, flags);
652                 free_object(obj);
653                 return;
654         }
655 out_unlock:
656         raw_spin_unlock_irqrestore(&db->lock, flags);
657 }
658 EXPORT_SYMBOL_GPL(debug_object_free);
659
660 /**
661  * debug_object_assert_init - debug checks when object should be init-ed
662  * @addr:       address of the object
663  * @descr:      pointer to an object specific debug description structure
664  */
665 void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
666 {
667         struct debug_bucket *db;
668         struct debug_obj *obj;
669         unsigned long flags;
670
671         if (!debug_objects_enabled)
672                 return;
673
674         db = get_bucket((unsigned long) addr);
675
676         raw_spin_lock_irqsave(&db->lock, flags);
677
678         obj = lookup_object(addr, db);
679         if (!obj) {
680                 struct debug_obj o = { .object = addr,
681                                        .state = ODEBUG_STATE_NOTAVAILABLE,
682                                        .descr = descr };
683
684                 raw_spin_unlock_irqrestore(&db->lock, flags);
685                 /*
686                  * Maybe the object is static, and we let the type specific
687                  * code confirm. Track this static object if true, else invoke
688                  * fixup.
689                  */
690                 if (descr->is_static_object && descr->is_static_object(addr)) {
691                         /* Track this static object */
692                         debug_object_init(addr, descr);
693                 } else {
694                         debug_print_object(&o, "assert_init");
695                         debug_object_fixup(descr->fixup_assert_init, addr,
696                                            ODEBUG_STATE_NOTAVAILABLE);
697                 }
698                 return;
699         }
700
701         raw_spin_unlock_irqrestore(&db->lock, flags);
702 }
703 EXPORT_SYMBOL_GPL(debug_object_assert_init);
704
705 /**
706  * debug_object_active_state - debug checks object usage state machine
707  * @addr:       address of the object
708  * @descr:      pointer to an object specific debug description structure
709  * @expect:     expected state
710  * @next:       state to move to if expected state is found
711  */
712 void
713 debug_object_active_state(void *addr, struct debug_obj_descr *descr,
714                           unsigned int expect, unsigned int next)
715 {
716         struct debug_bucket *db;
717         struct debug_obj *obj;
718         unsigned long flags;
719
720         if (!debug_objects_enabled)
721                 return;
722
723         db = get_bucket((unsigned long) addr);
724
725         raw_spin_lock_irqsave(&db->lock, flags);
726
727         obj = lookup_object(addr, db);
728         if (obj) {
729                 switch (obj->state) {
730                 case ODEBUG_STATE_ACTIVE:
731                         if (obj->astate == expect)
732                                 obj->astate = next;
733                         else
734                                 debug_print_object(obj, "active_state");
735                         break;
736
737                 default:
738                         debug_print_object(obj, "active_state");
739                         break;
740                 }
741         } else {
742                 struct debug_obj o = { .object = addr,
743                                        .state = ODEBUG_STATE_NOTAVAILABLE,
744                                        .descr = descr };
745
746                 debug_print_object(&o, "active_state");
747         }
748
749         raw_spin_unlock_irqrestore(&db->lock, flags);
750 }
751 EXPORT_SYMBOL_GPL(debug_object_active_state);
752
753 #ifdef CONFIG_DEBUG_OBJECTS_FREE
754 static void __debug_check_no_obj_freed(const void *address, unsigned long size)
755 {
756         unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
757         struct debug_obj_descr *descr;
758         enum debug_obj_state state;
759         struct debug_bucket *db;
760         struct hlist_node *tmp;
761         struct debug_obj *obj;
762         int cnt, objs_checked = 0;
763         bool work = false;
764
765         saddr = (unsigned long) address;
766         eaddr = saddr + size;
767         paddr = saddr & ODEBUG_CHUNK_MASK;
768         chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
769         chunks >>= ODEBUG_CHUNK_SHIFT;
770
771         for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
772                 db = get_bucket(paddr);
773
774 repeat:
775                 cnt = 0;
776                 raw_spin_lock_irqsave(&db->lock, flags);
777                 hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
778                         cnt++;
779                         oaddr = (unsigned long) obj->object;
780                         if (oaddr < saddr || oaddr >= eaddr)
781                                 continue;
782
783                         switch (obj->state) {
784                         case ODEBUG_STATE_ACTIVE:
785                                 debug_print_object(obj, "free");
786                                 descr = obj->descr;
787                                 state = obj->state;
788                                 raw_spin_unlock_irqrestore(&db->lock, flags);
789                                 debug_object_fixup(descr->fixup_free,
790                                                    (void *) oaddr, state);
791                                 goto repeat;
792                         default:
793                                 hlist_del(&obj->node);
794                                 work |= __free_object(obj);
795                                 break;
796                         }
797                 }
798                 raw_spin_unlock_irqrestore(&db->lock, flags);
799
800                 if (cnt > debug_objects_maxchain)
801                         debug_objects_maxchain = cnt;
802
803                 objs_checked += cnt;
804         }
805
806         if (objs_checked > debug_objects_maxchecked)
807                 debug_objects_maxchecked = objs_checked;
808
809         /* Schedule work to actually kmem_cache_free() objects */
810         if (work)
811                 schedule_work(&debug_obj_work);
812 }
813
814 void debug_check_no_obj_freed(const void *address, unsigned long size)
815 {
816         if (debug_objects_enabled)
817                 __debug_check_no_obj_freed(address, size);
818 }
819 #endif
820
821 #ifdef CONFIG_DEBUG_FS
822
823 static int debug_stats_show(struct seq_file *m, void *v)
824 {
825         seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
826         seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
827         seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
828         seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
829         seq_printf(m, "pool_free     :%d\n", obj_pool_free);
830         seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
831         seq_printf(m, "pool_used     :%d\n", obj_pool_used);
832         seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
833         seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
834         seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
835         seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
836         return 0;
837 }
838
839 static int debug_stats_open(struct inode *inode, struct file *filp)
840 {
841         return single_open(filp, debug_stats_show, NULL);
842 }
843
844 static const struct file_operations debug_stats_fops = {
845         .open           = debug_stats_open,
846         .read           = seq_read,
847         .llseek         = seq_lseek,
848         .release        = single_release,
849 };
850
851 static int __init debug_objects_init_debugfs(void)
852 {
853         struct dentry *dbgdir, *dbgstats;
854
855         if (!debug_objects_enabled)
856                 return 0;
857
858         dbgdir = debugfs_create_dir("debug_objects", NULL);
859         if (!dbgdir)
860                 return -ENOMEM;
861
862         dbgstats = debugfs_create_file("stats", 0444, dbgdir, NULL,
863                                        &debug_stats_fops);
864         if (!dbgstats)
865                 goto err;
866
867         return 0;
868
869 err:
870         debugfs_remove(dbgdir);
871
872         return -ENOMEM;
873 }
874 __initcall(debug_objects_init_debugfs);
875
876 #else
877 static inline void debug_objects_init_debugfs(void) { }
878 #endif
879
880 #ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
881
882 /* Random data structure for the self test */
883 struct self_test {
884         unsigned long   dummy1[6];
885         int             static_init;
886         unsigned long   dummy2[3];
887 };
888
889 static __initdata struct debug_obj_descr descr_type_test;
890
891 static bool __init is_static_object(void *addr)
892 {
893         struct self_test *obj = addr;
894
895         return obj->static_init;
896 }
897
898 /*
899  * fixup_init is called when:
900  * - an active object is initialized
901  */
902 static bool __init fixup_init(void *addr, enum debug_obj_state state)
903 {
904         struct self_test *obj = addr;
905
906         switch (state) {
907         case ODEBUG_STATE_ACTIVE:
908                 debug_object_deactivate(obj, &descr_type_test);
909                 debug_object_init(obj, &descr_type_test);
910                 return true;
911         default:
912                 return false;
913         }
914 }
915
916 /*
917  * fixup_activate is called when:
918  * - an active object is activated
919  * - an unknown non-static object is activated
920  */
921 static bool __init fixup_activate(void *addr, enum debug_obj_state state)
922 {
923         struct self_test *obj = addr;
924
925         switch (state) {
926         case ODEBUG_STATE_NOTAVAILABLE:
927                 return true;
928         case ODEBUG_STATE_ACTIVE:
929                 debug_object_deactivate(obj, &descr_type_test);
930                 debug_object_activate(obj, &descr_type_test);
931                 return true;
932
933         default:
934                 return false;
935         }
936 }
937
938 /*
939  * fixup_destroy is called when:
940  * - an active object is destroyed
941  */
942 static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
943 {
944         struct self_test *obj = addr;
945
946         switch (state) {
947         case ODEBUG_STATE_ACTIVE:
948                 debug_object_deactivate(obj, &descr_type_test);
949                 debug_object_destroy(obj, &descr_type_test);
950                 return true;
951         default:
952                 return false;
953         }
954 }
955
956 /*
957  * fixup_free is called when:
958  * - an active object is freed
959  */
960 static bool __init fixup_free(void *addr, enum debug_obj_state state)
961 {
962         struct self_test *obj = addr;
963
964         switch (state) {
965         case ODEBUG_STATE_ACTIVE:
966                 debug_object_deactivate(obj, &descr_type_test);
967                 debug_object_free(obj, &descr_type_test);
968                 return true;
969         default:
970                 return false;
971         }
972 }
973
974 static int __init
975 check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
976 {
977         struct debug_bucket *db;
978         struct debug_obj *obj;
979         unsigned long flags;
980         int res = -EINVAL;
981
982         db = get_bucket((unsigned long) addr);
983
984         raw_spin_lock_irqsave(&db->lock, flags);
985
986         obj = lookup_object(addr, db);
987         if (!obj && state != ODEBUG_STATE_NONE) {
988                 WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
989                 goto out;
990         }
991         if (obj && obj->state != state) {
992                 WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
993                        obj->state, state);
994                 goto out;
995         }
996         if (fixups != debug_objects_fixups) {
997                 WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
998                        fixups, debug_objects_fixups);
999                 goto out;
1000         }
1001         if (warnings != debug_objects_warnings) {
1002                 WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1003                        warnings, debug_objects_warnings);
1004                 goto out;
1005         }
1006         res = 0;
1007 out:
1008         raw_spin_unlock_irqrestore(&db->lock, flags);
1009         if (res)
1010                 debug_objects_enabled = 0;
1011         return res;
1012 }
1013
1014 static __initdata struct debug_obj_descr descr_type_test = {
1015         .name                   = "selftest",
1016         .is_static_object       = is_static_object,
1017         .fixup_init             = fixup_init,
1018         .fixup_activate         = fixup_activate,
1019         .fixup_destroy          = fixup_destroy,
1020         .fixup_free             = fixup_free,
1021 };
1022
1023 static __initdata struct self_test obj = { .static_init = 0 };
1024
1025 static void __init debug_objects_selftest(void)
1026 {
1027         int fixups, oldfixups, warnings, oldwarnings;
1028         unsigned long flags;
1029
1030         local_irq_save(flags);
1031
1032         fixups = oldfixups = debug_objects_fixups;
1033         warnings = oldwarnings = debug_objects_warnings;
1034         descr_test = &descr_type_test;
1035
1036         debug_object_init(&obj, &descr_type_test);
1037         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1038                 goto out;
1039         debug_object_activate(&obj, &descr_type_test);
1040         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1041                 goto out;
1042         debug_object_activate(&obj, &descr_type_test);
1043         if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1044                 goto out;
1045         debug_object_deactivate(&obj, &descr_type_test);
1046         if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1047                 goto out;
1048         debug_object_destroy(&obj, &descr_type_test);
1049         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1050                 goto out;
1051         debug_object_init(&obj, &descr_type_test);
1052         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1053                 goto out;
1054         debug_object_activate(&obj, &descr_type_test);
1055         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1056                 goto out;
1057         debug_object_deactivate(&obj, &descr_type_test);
1058         if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1059                 goto out;
1060         debug_object_free(&obj, &descr_type_test);
1061         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1062                 goto out;
1063
1064         obj.static_init = 1;
1065         debug_object_activate(&obj, &descr_type_test);
1066         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1067                 goto out;
1068         debug_object_init(&obj, &descr_type_test);
1069         if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1070                 goto out;
1071         debug_object_free(&obj, &descr_type_test);
1072         if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1073                 goto out;
1074
1075 #ifdef CONFIG_DEBUG_OBJECTS_FREE
1076         debug_object_init(&obj, &descr_type_test);
1077         if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1078                 goto out;
1079         debug_object_activate(&obj, &descr_type_test);
1080         if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1081                 goto out;
1082         __debug_check_no_obj_freed(&obj, sizeof(obj));
1083         if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1084                 goto out;
1085 #endif
1086         pr_info("selftest passed\n");
1087
1088 out:
1089         debug_objects_fixups = oldfixups;
1090         debug_objects_warnings = oldwarnings;
1091         descr_test = NULL;
1092
1093         local_irq_restore(flags);
1094 }
1095 #else
1096 static inline void debug_objects_selftest(void) { }
1097 #endif
1098
1099 /*
1100  * Called during early boot to initialize the hash buckets and link
1101  * the static object pool objects into the poll list. After this call
1102  * the object tracker is fully operational.
1103  */
1104 void __init debug_objects_early_init(void)
1105 {
1106         int i;
1107
1108         for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1109                 raw_spin_lock_init(&obj_hash[i].lock);
1110
1111         for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1112                 hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1113 }
1114
1115 /*
1116  * Convert the statically allocated objects to dynamic ones:
1117  */
1118 static int __init debug_objects_replace_static_objects(void)
1119 {
1120         struct debug_bucket *db = obj_hash;
1121         struct hlist_node *tmp;
1122         struct debug_obj *obj, *new;
1123         HLIST_HEAD(objects);
1124         int i, cnt = 0;
1125
1126         for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1127                 obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1128                 if (!obj)
1129                         goto free;
1130                 hlist_add_head(&obj->node, &objects);
1131         }
1132
1133         /*
1134          * debug_objects_mem_init() is now called early that only one CPU is up
1135          * and interrupts have been disabled, so it is safe to replace the
1136          * active object references.
1137          */
1138
1139         /* Remove the statically allocated objects from the pool */
1140         hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1141                 hlist_del(&obj->node);
1142         /* Move the allocated objects to the pool */
1143         hlist_move_list(&objects, &obj_pool);
1144
1145         /* Replace the active object references */
1146         for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1147                 hlist_move_list(&db->list, &objects);
1148
1149                 hlist_for_each_entry(obj, &objects, node) {
1150                         new = hlist_entry(obj_pool.first, typeof(*obj), node);
1151                         hlist_del(&new->node);
1152                         /* copy object data */
1153                         *new = *obj;
1154                         hlist_add_head(&new->node, &db->list);
1155                         cnt++;
1156                 }
1157         }
1158
1159         pr_debug("%d of %d active objects replaced\n",
1160                  cnt, obj_pool_used);
1161         return 0;
1162 free:
1163         hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1164                 hlist_del(&obj->node);
1165                 kmem_cache_free(obj_cache, obj);
1166         }
1167         return -ENOMEM;
1168 }
1169
1170 /*
1171  * Called after the kmem_caches are functional to setup a dedicated
1172  * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
1173  * prevents that the debug code is called on kmem_cache_free() for the
1174  * debug tracker objects to avoid recursive calls.
1175  */
1176 void __init debug_objects_mem_init(void)
1177 {
1178         if (!debug_objects_enabled)
1179                 return;
1180
1181         obj_cache = kmem_cache_create("debug_objects_cache",
1182                                       sizeof (struct debug_obj), 0,
1183                                       SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1184                                       NULL);
1185
1186         if (!obj_cache || debug_objects_replace_static_objects()) {
1187                 debug_objects_enabled = 0;
1188                 kmem_cache_destroy(obj_cache);
1189                 pr_warn("out of memory.\n");
1190         } else
1191                 debug_objects_selftest();
1192
1193         /*
1194          * Increase the thresholds for allocating and freeing objects
1195          * according to the number of possible CPUs available in the system.
1196          */
1197         debug_objects_pool_size += num_possible_cpus() * 32;
1198         debug_objects_pool_min_level += num_possible_cpus() * 4;
1199 }