^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Generic infrastructure for lifetime debugging of objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Started by Thomas Gleixner
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * For licencing details see kernel-base/COPYING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define pr_fmt(fmt) "ODEBUG: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/debugobjects.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/sched/task_stack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/kmemleak.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #define ODEBUG_HASH_BITS 14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #define ODEBUG_HASH_SIZE (1 << ODEBUG_HASH_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #define ODEBUG_POOL_SIZE 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #define ODEBUG_POOL_MIN_LEVEL 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #define ODEBUG_POOL_PERCPU_SIZE 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #define ODEBUG_BATCH_SIZE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define ODEBUG_CHUNK_SHIFT PAGE_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define ODEBUG_CHUNK_SIZE (1 << ODEBUG_CHUNK_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define ODEBUG_CHUNK_MASK (~(ODEBUG_CHUNK_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * We limit the freeing of debug objects via workqueue at a maximum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * frequency of 10Hz and about 1024 objects for each freeing operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * So it is freeing at most 10k debug objects per second.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define ODEBUG_FREE_WORK_MAX 1024
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #define ODEBUG_FREE_WORK_DELAY DIV_ROUND_UP(HZ, 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) struct debug_bucket {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) struct hlist_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) raw_spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * Debug object percpu free list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * Access is protected by disabling irq
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) struct debug_percpu_free {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct hlist_head free_objs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) int obj_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static struct debug_bucket obj_hash[ODEBUG_HASH_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) static struct debug_obj obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static DEFINE_RAW_SPINLOCK(pool_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) static HLIST_HEAD(obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static HLIST_HEAD(obj_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * Because of the presence of percpu free pools, obj_pool_free will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * under-count those in the percpu free pools. Similarly, obj_pool_used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * will over-count those in the percpu free pools. Adjustments will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * can be off.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static int obj_pool_min_free = ODEBUG_POOL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static int obj_pool_free = ODEBUG_POOL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static int obj_pool_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static int obj_pool_max_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static bool obj_freeing;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* The number of objs on the global free list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static int obj_nr_tofree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static int debug_objects_maxchain __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) static int __maybe_unused debug_objects_maxchecked __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static int debug_objects_fixups __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static int debug_objects_warnings __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int debug_objects_enabled __read_mostly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static int debug_objects_pool_size __read_mostly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) = ODEBUG_POOL_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int debug_objects_pool_min_level __read_mostly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) = ODEBUG_POOL_MIN_LEVEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static const struct debug_obj_descr *descr_test __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static struct kmem_cache *obj_cache __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * Track numbers of kmem_cache_alloc()/free() calls done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static int debug_objects_allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) static int debug_objects_freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static void free_obj_work(struct work_struct *work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) static int __init enable_object_debug(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) debug_objects_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static int __init disable_object_debug(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) debug_objects_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) early_param("debug_objects", enable_object_debug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) early_param("no_debug_objects", disable_object_debug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static const char *obj_states[ODEBUG_STATE_MAX] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) [ODEBUG_STATE_NONE] = "none",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) [ODEBUG_STATE_INIT] = "initialized",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) [ODEBUG_STATE_INACTIVE] = "inactive",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) [ODEBUG_STATE_ACTIVE] = "active",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) [ODEBUG_STATE_DESTROYED] = "destroyed",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) [ODEBUG_STATE_NOTAVAILABLE] = "not available",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) static void fill_pool(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (likely(READ_ONCE(obj_pool_free) >= debug_objects_pool_min_level))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * Reuse objs from the global free list; they will be reinitialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * when allocating.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * Both obj_nr_tofree and obj_pool_free are checked locklessly; the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * READ_ONCE()s pair with the WRITE_ONCE()s in pool_lock critical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * sections.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) while (READ_ONCE(obj_nr_tofree) && (READ_ONCE(obj_pool_free) < obj_pool_min_free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) raw_spin_lock_irqsave(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * Recheck with the lock held as the worker thread might have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * won the race and freed the global free list already.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) hlist_add_head(&obj->node, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) raw_spin_unlock_irqrestore(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) if (unlikely(!obj_cache))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) while (READ_ONCE(obj_pool_free) < debug_objects_pool_min_level) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) struct debug_obj *new[ODEBUG_BATCH_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) int cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (!new[cnt])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) if (!cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) raw_spin_lock_irqsave(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) while (cnt) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) hlist_add_head(&new[--cnt]->node, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) debug_objects_allocated++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) raw_spin_unlock_irqrestore(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * Lookup an object in the hash bucket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) int cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) hlist_for_each_entry(obj, &b->list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (obj->object == addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) return obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (cnt > debug_objects_maxchain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) debug_objects_maxchain = cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) * Allocate a new object from the hlist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static struct debug_obj *__alloc_object(struct hlist_head *list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) struct debug_obj *obj = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (list->first) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) obj = hlist_entry(list->first, typeof(*obj), node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) return obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * Allocate a new object. If the pool is empty, switch off the debugger.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * Must be called with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static struct debug_obj *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) alloc_object(void *addr, struct debug_bucket *b, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (likely(obj_cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) obj = __alloc_object(&percpu_pool->free_objs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) percpu_pool->obj_free--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) goto init_obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) raw_spin_lock(&pool_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) obj = __alloc_object(&obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) obj_pool_used++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * Looking ahead, allocate one batch of debug objects and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * put them into the percpu free pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (likely(obj_cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct debug_obj *obj2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) obj2 = __alloc_object(&obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (!obj2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) hlist_add_head(&obj2->node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) &percpu_pool->free_objs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) percpu_pool->obj_free++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) obj_pool_used++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (obj_pool_used > obj_pool_max_used)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) obj_pool_max_used = obj_pool_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) if (obj_pool_free < obj_pool_min_free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) obj_pool_min_free = obj_pool_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) raw_spin_unlock(&pool_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) init_obj:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) obj->object = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) obj->descr = descr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) obj->state = ODEBUG_STATE_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) obj->astate = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) hlist_add_head(&obj->node, &b->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * workqueue function to free objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) * To reduce contention on the global pool_lock, the actual freeing of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * debug objects will be delayed if the pool_lock is busy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) static void free_obj_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) HLIST_HEAD(tofree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) WRITE_ONCE(obj_freeing, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) if (!raw_spin_trylock_irqsave(&pool_lock, flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (obj_pool_free >= debug_objects_pool_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) goto free_objs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * The objs on the pool list might be allocated before the work is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * run, so recheck if pool list it full or not, if not fill pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * list from the global free list. As it is likely that a workload
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * may be gearing up to use more and more objects, don't free any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * of them until the next round.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) hlist_add_head(&obj->node, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) raw_spin_unlock_irqrestore(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) free_objs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * Pool list is already full and there are still objs on the free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * list. Move remaining free objs to a temporary list to free the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * memory outside the pool_lock held region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) if (obj_nr_tofree) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) hlist_move_list(&obj_to_free, &tofree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) debug_objects_freed += obj_nr_tofree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) WRITE_ONCE(obj_nr_tofree, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) raw_spin_unlock_irqrestore(&pool_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) kmem_cache_free(obj_cache, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static void __free_object(struct debug_obj *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct debug_obj *objs[ODEBUG_BATCH_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) struct debug_percpu_free *percpu_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) int lookahead_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) bool work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (!obj_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) goto free_to_obj_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * Try to free it into the percpu pool first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) percpu_pool = this_cpu_ptr(&percpu_obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) hlist_add_head(&obj->node, &percpu_pool->free_objs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) percpu_pool->obj_free++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) * As the percpu pool is full, look ahead and pull out a batch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * of objects from the percpu pool and free them as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (!objs[lookahead_count])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) percpu_pool->obj_free--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) free_to_obj_pool:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) raw_spin_lock(&pool_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) obj_pool_used--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (work) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) hlist_add_head(&obj->node, &obj_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) if (lookahead_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + lookahead_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) obj_pool_used -= lookahead_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) while (lookahead_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) hlist_add_head(&objs[--lookahead_count]->node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) &obj_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) if ((obj_pool_free > debug_objects_pool_size) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) * Free one more batch of objects from obj_pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) obj = __alloc_object(&obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) hlist_add_head(&obj->node, &obj_to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) WRITE_ONCE(obj_pool_free, obj_pool_free - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) WRITE_ONCE(obj_nr_tofree, obj_nr_tofree + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) WRITE_ONCE(obj_pool_free, obj_pool_free + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) hlist_add_head(&obj->node, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) if (lookahead_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) WRITE_ONCE(obj_pool_free, obj_pool_free + lookahead_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) obj_pool_used -= lookahead_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) while (lookahead_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) hlist_add_head(&objs[--lookahead_count]->node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) raw_spin_unlock(&pool_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Put the object back into the pool and schedule work to free objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) * if necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static void free_object(struct debug_obj *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) __free_object(obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) WRITE_ONCE(obj_freeing, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static int object_cpu_offline(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) struct debug_percpu_free *percpu_pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) /* Remote access is safe as the CPU is dead already */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) percpu_pool = per_cpu_ptr(&percpu_obj_pool, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) hlist_for_each_entry_safe(obj, tmp, &percpu_pool->free_objs, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) kmem_cache_free(obj_cache, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) percpu_pool->obj_free = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) * We run out of memory. That means we probably have tons of objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static void debug_objects_oom(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) struct debug_bucket *db = obj_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) HLIST_HEAD(freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) pr_warn("Out of memory. ODEBUG disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) hlist_move_list(&db->list, &freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) /* Now free them */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) free_object(obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * We use the pfn of the address for the hash. That way we can check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * for freed objects simply by checking the affected bucket.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) static struct debug_bucket *get_bucket(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) unsigned long hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return &obj_hash[hash];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) static void debug_print_object(struct debug_obj *obj, char *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) const struct debug_obj_descr *descr = obj->descr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) static int limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) if (limit < 5 && descr != descr_test) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) void *hint = descr->debug_hint ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) descr->debug_hint(obj->object) : NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) limit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) "object type: %s hint: %pS\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) msg, obj_states[obj->state], obj->astate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) descr->name, hint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) debug_objects_warnings++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * Try to repair the damage, so we have a better chance to get useful
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * debug output.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) static bool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) void * addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) if (fixup && fixup(addr, state)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) debug_objects_fixups++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) static void debug_object_is_on_stack(void *addr, int onstack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) int is_on_stack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) static int limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (limit > 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) is_on_stack = object_is_on_stack(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (is_on_stack == onstack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) limit++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (is_on_stack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) task_stack_page(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) task_stack_page(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) __debug_object_init(void *addr, const struct debug_obj_descr *descr, int onstack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) enum debug_obj_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) bool check_stack = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) fill_pool();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) obj = alloc_object(addr, db, descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) debug_objects_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) debug_objects_oom();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) check_stack = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) case ODEBUG_STATE_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) case ODEBUG_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) case ODEBUG_STATE_INACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) obj->state = ODEBUG_STATE_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) state = obj->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) debug_print_object(obj, "init");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) debug_object_fixup(descr->fixup_init, addr, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) case ODEBUG_STATE_DESTROYED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) debug_print_object(obj, "init");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (check_stack)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) debug_object_is_on_stack(addr, onstack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) * debug_object_init - debug checks when an object is initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void debug_object_init(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) __debug_object_init(addr, descr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) EXPORT_SYMBOL_GPL(debug_object_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * debug_object_init_on_stack - debug checks when an object on stack is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) void debug_object_init_on_stack(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) __debug_object_init(addr, descr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * debug_object_activate - debug checks when an object is activated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * Returns 0 for success, -EINVAL for check failed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) int debug_object_activate(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) enum debug_obj_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct debug_obj o = { .object = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) .state = ODEBUG_STATE_NOTAVAILABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) .descr = descr };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) bool print_object = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) case ODEBUG_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) case ODEBUG_STATE_INACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) obj->state = ODEBUG_STATE_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) state = obj->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) debug_print_object(obj, "activate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) ret = debug_object_fixup(descr->fixup_activate, addr, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return ret ? 0 : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) case ODEBUG_STATE_DESTROYED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) if (print_object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) debug_print_object(obj, "activate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * We are here when a static object is activated. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * let the type specific code confirm whether this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) * true or not. if true, we just make sure that the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * static object is tracked in the object tracker. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * not, this must be a bug, so we try to fix it up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) if (descr->is_static_object && descr->is_static_object(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) /* track this static object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) debug_object_init(addr, descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) debug_object_activate(addr, descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) debug_print_object(&o, "activate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) ret = debug_object_fixup(descr->fixup_activate, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) ODEBUG_STATE_NOTAVAILABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) return ret ? 0 : -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) EXPORT_SYMBOL_GPL(debug_object_activate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * debug_object_deactivate - debug checks when an object is deactivated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) void debug_object_deactivate(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) bool print_object = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) case ODEBUG_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) case ODEBUG_STATE_INACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) if (!obj->astate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) obj->state = ODEBUG_STATE_INACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) case ODEBUG_STATE_DESTROYED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) struct debug_obj o = { .object = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) .state = ODEBUG_STATE_NOTAVAILABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) .descr = descr };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) debug_print_object(&o, "deactivate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) } else if (print_object) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) debug_print_object(obj, "deactivate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) EXPORT_SYMBOL_GPL(debug_object_deactivate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) * debug_object_destroy - debug checks when an object is destroyed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) void debug_object_destroy(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) enum debug_obj_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) bool print_object = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) case ODEBUG_STATE_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) case ODEBUG_STATE_INIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) case ODEBUG_STATE_INACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) obj->state = ODEBUG_STATE_DESTROYED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) state = obj->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) debug_print_object(obj, "destroy");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) debug_object_fixup(descr->fixup_destroy, addr, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) case ODEBUG_STATE_DESTROYED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) if (print_object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) debug_print_object(obj, "destroy");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) EXPORT_SYMBOL_GPL(debug_object_destroy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) * debug_object_free - debug checks when an object is freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) void debug_object_free(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) enum debug_obj_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) state = obj->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) debug_print_object(obj, "free");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) debug_object_fixup(descr->fixup_free, addr, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) free_object(obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) EXPORT_SYMBOL_GPL(debug_object_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * debug_object_assert_init - debug checks when object should be init-ed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) void debug_object_assert_init(void *addr, const struct debug_obj_descr *descr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) struct debug_obj o = { .object = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) .state = ODEBUG_STATE_NOTAVAILABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) .descr = descr };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * Maybe the object is static, and we let the type specific
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) * code confirm. Track this static object if true, else invoke
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * fixup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) if (descr->is_static_object && descr->is_static_object(addr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) /* Track this static object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) debug_object_init(addr, descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) debug_print_object(&o, "assert_init");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) debug_object_fixup(descr->fixup_assert_init, addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) ODEBUG_STATE_NOTAVAILABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) EXPORT_SYMBOL_GPL(debug_object_assert_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) * debug_object_active_state - debug checks object usage state machine
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * @addr: address of the object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * @descr: pointer to an object specific debug description structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * @expect: expected state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * @next: state to move to if expected state is found
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) debug_object_active_state(void *addr, const struct debug_obj_descr *descr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) unsigned int expect, unsigned int next)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) bool print_object = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) if (obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (obj->astate == expect)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) obj->astate = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) print_object = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (!obj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) struct debug_obj o = { .object = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) .state = ODEBUG_STATE_NOTAVAILABLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) .descr = descr };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) debug_print_object(&o, "active_state");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) } else if (print_object) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) debug_print_object(obj, "active_state");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) EXPORT_SYMBOL_GPL(debug_object_active_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) #ifdef CONFIG_DEBUG_OBJECTS_FREE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) static void __debug_check_no_obj_freed(const void *address, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) const struct debug_obj_descr *descr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) enum debug_obj_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) int cnt, objs_checked = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) saddr = (unsigned long) address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) eaddr = saddr + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) paddr = saddr & ODEBUG_CHUNK_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) chunks >>= ODEBUG_CHUNK_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) db = get_bucket(paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) oaddr = (unsigned long) obj->object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (oaddr < saddr || oaddr >= eaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) switch (obj->state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) descr = obj->descr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) state = obj->state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) debug_print_object(obj, "free");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) debug_object_fixup(descr->fixup_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) (void *) oaddr, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) __free_object(obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) if (cnt > debug_objects_maxchain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) debug_objects_maxchain = cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) objs_checked += cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (objs_checked > debug_objects_maxchecked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) debug_objects_maxchecked = objs_checked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) /* Schedule work to actually kmem_cache_free() objects */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) if (!READ_ONCE(obj_freeing) && READ_ONCE(obj_nr_tofree)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) WRITE_ONCE(obj_freeing, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) void debug_check_no_obj_freed(const void *address, unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) __debug_check_no_obj_freed(address, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) #ifdef CONFIG_DEBUG_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) static int debug_stats_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) int cpu, obj_percpu_free = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) seq_printf(m, "max_chain :%d\n", debug_objects_maxchain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) seq_printf(m, "max_checked :%d\n", debug_objects_maxchecked);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) seq_printf(m, "warnings :%d\n", debug_objects_warnings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) seq_printf(m, "fixups :%d\n", debug_objects_fixups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) seq_printf(m, "pool_free :%d\n", READ_ONCE(obj_pool_free) + obj_percpu_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) seq_printf(m, "pool_used :%d\n", obj_pool_used - obj_percpu_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) seq_printf(m, "on_free_list :%d\n", READ_ONCE(obj_nr_tofree));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) seq_printf(m, "objs_freed :%d\n", debug_objects_freed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) DEFINE_SHOW_ATTRIBUTE(debug_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) static int __init debug_objects_init_debugfs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) struct dentry *dbgdir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) dbgdir = debugfs_create_dir("debug_objects", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) __initcall(debug_objects_init_debugfs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) static inline void debug_objects_init_debugfs(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) #ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) /* Random data structure for the self test */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) struct self_test {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) unsigned long dummy1[6];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) int static_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) unsigned long dummy2[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) static __initconst const struct debug_obj_descr descr_type_test;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) static bool __init is_static_object(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) struct self_test *obj = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return obj->static_init;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * fixup_init is called when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) * - an active object is initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) static bool __init fixup_init(void *addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) struct self_test *obj = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) debug_object_deactivate(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) debug_object_init(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) * fixup_activate is called when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) * - an active object is activated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) * - an unknown non-static object is activated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) static bool __init fixup_activate(void *addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) struct self_test *obj = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) case ODEBUG_STATE_NOTAVAILABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) debug_object_deactivate(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) debug_object_activate(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * fixup_destroy is called when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) * - an active object is destroyed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) struct self_test *obj = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) debug_object_deactivate(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) debug_object_destroy(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) * fixup_free is called when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) * - an active object is freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) static bool __init fixup_free(void *addr, enum debug_obj_state state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) struct self_test *obj = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) switch (state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) case ODEBUG_STATE_ACTIVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) debug_object_deactivate(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) debug_object_free(obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static int __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) struct debug_bucket *db;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) struct debug_obj *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) int res = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) db = get_bucket((unsigned long) addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) raw_spin_lock_irqsave(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) obj = lookup_object(addr, db);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (!obj && state != ODEBUG_STATE_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) if (obj && obj->state != state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) obj->state, state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) if (fixups != debug_objects_fixups) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) fixups, debug_objects_fixups);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) if (warnings != debug_objects_warnings) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) warnings, debug_objects_warnings);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) res = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) raw_spin_unlock_irqrestore(&db->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) if (res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) debug_objects_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) static __initconst const struct debug_obj_descr descr_type_test = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) .name = "selftest",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) .is_static_object = is_static_object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) .fixup_init = fixup_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) .fixup_activate = fixup_activate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) .fixup_destroy = fixup_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) .fixup_free = fixup_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) static __initdata struct self_test obj = { .static_init = 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) static void __init debug_objects_selftest(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) int fixups, oldfixups, warnings, oldwarnings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) fixups = oldfixups = debug_objects_fixups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) warnings = oldwarnings = debug_objects_warnings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) descr_test = &descr_type_test;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) debug_object_init(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) debug_object_activate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) debug_object_activate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) debug_object_deactivate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) debug_object_destroy(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) debug_object_init(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) debug_object_activate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) debug_object_deactivate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) debug_object_free(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) obj.static_init = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) debug_object_activate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) debug_object_init(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) debug_object_free(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) #ifdef CONFIG_DEBUG_OBJECTS_FREE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) debug_object_init(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) debug_object_activate(&obj, &descr_type_test);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) __debug_check_no_obj_freed(&obj, sizeof(obj));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) pr_info("selftest passed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) debug_objects_fixups = oldfixups;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) debug_objects_warnings = oldwarnings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) descr_test = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) static inline void debug_objects_selftest(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) * Called during early boot to initialize the hash buckets and link
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) * the static object pool objects into the poll list. After this call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) * the object tracker is fully operational.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) void __init debug_objects_early_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) for (i = 0; i < ODEBUG_HASH_SIZE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) raw_spin_lock_init(&obj_hash[i].lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) for (i = 0; i < ODEBUG_POOL_SIZE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) hlist_add_head(&obj_static_pool[i].node, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * Convert the statically allocated objects to dynamic ones:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) static int __init debug_objects_replace_static_objects(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) struct debug_bucket *db = obj_hash;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct hlist_node *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) struct debug_obj *obj, *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) HLIST_HEAD(objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) int i, cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) if (!obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) goto free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) hlist_add_head(&obj->node, &objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) * debug_objects_mem_init() is now called early that only one CPU is up
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) * and interrupts have been disabled, so it is safe to replace the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) * active object references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) /* Remove the statically allocated objects from the pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) /* Move the allocated objects to the pool */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) hlist_move_list(&objects, &obj_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) /* Replace the active object references */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) hlist_move_list(&db->list, &objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) hlist_for_each_entry(obj, &objects, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) new = hlist_entry(obj_pool.first, typeof(*obj), node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) hlist_del(&new->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) /* copy object data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) *new = *obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) hlist_add_head(&new->node, &db->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) pr_debug("%d of %d active objects replaced\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) cnt, obj_pool_used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) hlist_for_each_entry_safe(obj, tmp, &objects, node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) hlist_del(&obj->node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) kmem_cache_free(obj_cache, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) * Called after the kmem_caches are functional to setup a dedicated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) * prevents that the debug code is called on kmem_cache_free() for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) * debug tracker objects to avoid recursive calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) void __init debug_objects_mem_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) int cpu, extras;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) if (!debug_objects_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) * Initialize the percpu object pools
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) * Initialization is not strictly necessary, but was done for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) * completeness.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) obj_cache = kmem_cache_create("debug_objects_cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) sizeof (struct debug_obj), 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) if (!obj_cache || debug_objects_replace_static_objects()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) debug_objects_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) kmem_cache_destroy(obj_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) pr_warn("out of memory.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) debug_objects_selftest();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) cpuhp_setup_state_nocalls(CPUHP_DEBUG_OBJ_DEAD, "object:offline", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) object_cpu_offline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) * Increase the thresholds for allocating and freeing objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) * according to the number of possible CPUs available in the system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) debug_objects_pool_size += extras;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) debug_objects_pool_min_level += extras;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags