^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * SLUB: A slab allocator that limits cache line use instead of queuing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * objects in per cpu and per node lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * The allocator synchronizes using per slab locks or atomic operatios
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * and only uses a centralized lock to manage a pool of partial slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * (C) 2007 SGI, Christoph Lameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * (C) 2011 Linux Foundation, Christoph Lameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/swap.h> /* struct reclaim_state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/bit_spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/swab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include "slab.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/kasan.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/cpuset.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mempolicy.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/ctype.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/debugobjects.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/kfence.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/math64.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/fault-inject.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/stacktrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/prefetch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/memcontrol.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/random.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <trace/events/kmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <trace/hooks/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) * Lock order:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * 1. slab_mutex (Global Mutex)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * 2. node->list_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * 3. slab_lock(page) (Only on some arches and for debugging)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * slab_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * The role of the slab_mutex is to protect the list of all the slabs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * and to synchronize major metadata changes to slab cache structures.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * The slab_lock is only used for debugging and on arches that do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) * have the ability to do a cmpxchg_double. It only protects:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * A. page->freelist -> List of object free in a page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * B. page->inuse -> Number of objects in use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * C. page->objects -> Number of objects in page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * D. page->frozen -> frozen state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * If a slab is frozen then it is exempt from list management. It is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * on any list except per cpu partial list. The processor that froze the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * slab is the one who can perform list operations on the page. Other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * processors may put objects onto the freelist but the processor that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * froze the slab is the only one that can retrieve the objects from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * page's freelist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * The list_lock protects the partial and full list on each node and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * the partial slab counter. If taken then no new slabs may be added or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * removed from the lists nor make the number of partial slabs be modified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) * (Note that the total number of slabs is an atomic value that may be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) * modified without taking the list lock).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) * The list_lock is a centralized lock and thus we avoid taking it as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) * much as possible. As long as SLUB does not have to handle partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) * slabs, operations can continue without any centralized lock. F.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * allocating a long series of objects that fill up slabs does not require
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * the list lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * Interrupts are disabled during allocation and deallocation in order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) * make the slab allocator safe to use in the context of an irq. In addition
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * interrupts are disabled to ensure that the processor does not change
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) * while handling per_cpu slabs, due to kernel preemption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) * SLUB assigns one slab for allocation to each processor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * Allocations only occur from these slabs called cpu slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) * Slabs with free elements are kept on a partial list and during regular
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * operations no list for full slabs is used. If an object in a full slab is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * freed then the slab will show up again on the partial lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * We track full slabs for debugging purposes though because otherwise we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * cannot scan all objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * Slabs are freed when they become empty. Teardown and setup is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * minimal so we rely on the page allocators per cpu caches for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * fast frees and allocs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * page->frozen The slab is frozen and exempt from list processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * This means that the slab is dedicated to a purpose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * such as satisfying allocations for a specific
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * processor. Objects may be freed in the slab while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * it is frozen but slab_free will then skip the usual
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * list operations. It is up to the processor holding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * the slab to integrate the slab into the slab lists
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * when the slab is no longer needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * One use of this flag is to mark slabs that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * used for allocations. Then such a slab becomes a cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * slab. The cpu slab may be equipped with an additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * freelist that allows lockless access to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * free objects in addition to the regular freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * that requires the slab lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * SLAB_DEBUG_FLAGS Slab requires special handling due to debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * options set. This moves slab handling out of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * the fast path and disables lockless freelists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #ifdef CONFIG_SLUB_DEBUG_ON
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) DEFINE_STATIC_KEY_TRUE(slub_debug_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) DEFINE_STATIC_KEY_FALSE(slub_debug_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) static inline bool kmem_cache_debug(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) void *fixup_red_left(struct kmem_cache *s, void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) p += s->red_left_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #ifdef CONFIG_SLUB_CPU_PARTIAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return !kmem_cache_debug(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * Issues still to be resolved:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * - Variable sizing of the per node arrays
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* Enable to test recovery from slab corruption on boot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #undef SLUB_RESILIENCY_TEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) /* Enable to log cmpxchg failures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #undef SLUB_DEBUG_CMPXCHG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) * Mininum number of partial slabs. These will be left on the partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * lists even if they are empty. kmem_cache_shrink may reclaim them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #define MIN_PARTIAL 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * Maximum number of desirable partial slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * The existence of more partial slabs makes kmem_cache_shrink
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * sort the partial list by the number of objects in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) #define MAX_PARTIAL 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #define DEBUG_DEFAULT_FLAGS (SLAB_CONSISTENCY_CHECKS | SLAB_RED_ZONE | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) SLAB_POISON | SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * These debug flags cannot use CMPXCHG because there might be consistency
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * issues when checking or reading debug information
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) #define SLAB_NO_CMPXCHG (SLAB_CONSISTENCY_CHECKS | SLAB_STORE_USER | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) SLAB_TRACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * Debugging flags that require metadata to be stored in the slab. These get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * disabled when slub_debug=O is used and a cache's min order increases with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * metadata.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #define DEBUG_METADATA_FLAGS (SLAB_RED_ZONE | SLAB_POISON | SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #define OO_SHIFT 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #define OO_MASK ((1 << OO_SHIFT) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) #define MAX_OBJS_PER_PAGE 32767 /* since page.objects is u15 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /* Internal SLUB flags */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Poison object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) #define __OBJECT_POISON ((slab_flags_t __force)0x80000000U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Use cmpxchg_double */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #define __CMPXCHG_DOUBLE ((slab_flags_t __force)0x40000000U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) #ifdef CONFIG_SLUB_SYSFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static int sysfs_slab_add(struct kmem_cache *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) static int sysfs_slab_alias(struct kmem_cache *, const char *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #if defined(CONFIG_DEBUG_FS) && defined(CONFIG_SLUB_DEBUG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static void debugfs_slab_add(struct kmem_cache *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static inline void debugfs_slab_add(struct kmem_cache *s) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static inline void stat(const struct kmem_cache *s, enum stat_item si)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) #ifdef CONFIG_SLUB_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * The rmw is racy on a preemptible kernel but this is acceptable, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * avoid this_cpu_add()'s irq-disable overhead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) raw_cpu_inc(s->cpu_slab->stat[si]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /********************************************************************
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * Core slab cache functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) *******************************************************************/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * Returns freelist pointer (ptr). With hardening, this is obfuscated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * with an XOR of the address where the pointer is held and a per-cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * random number.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static inline void *freelist_ptr(const struct kmem_cache *s, void *ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) unsigned long ptr_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #ifdef CONFIG_SLAB_FREELIST_HARDENED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * When CONFIG_KASAN_SW/HW_TAGS is enabled, ptr_addr might be tagged.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * Normally, this doesn't cause any issues, as both set_freepointer()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * and get_freepointer() are called with a pointer with the same tag.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * However, there are some issues with CONFIG_SLUB_DEBUG code. For
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * example, when __free_slub() iterates over objects in a cache, it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * passes untagged pointers to check_object(). check_object() in turns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) * calls get_freepointer() with an untagged pointer, which causes the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * freepointer to be restored incorrectly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return (void *)((unsigned long)ptr ^ s->random ^
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) swab((unsigned long)kasan_reset_tag((void *)ptr_addr)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* Returns the freelist pointer recorded at location ptr_addr. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static inline void *freelist_dereference(const struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void *ptr_addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return freelist_ptr(s, (void *)*(unsigned long *)(ptr_addr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) (unsigned long)ptr_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) static inline void *get_freepointer(struct kmem_cache *s, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) object = kasan_reset_tag(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return freelist_dereference(s, object + s->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void prefetch_freepointer(const struct kmem_cache *s, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) prefetch(object + s->offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned long freepointer_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (!debug_pagealloc_enabled_static())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) return get_freepointer(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) object = kasan_reset_tag(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) freepointer_addr = (unsigned long)object + s->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) copy_from_kernel_nofault(&p, (void **)freepointer_addr, sizeof(p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return freelist_ptr(s, p, freepointer_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) unsigned long freeptr_addr = (unsigned long)object + s->offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) #ifdef CONFIG_SLAB_FREELIST_HARDENED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) BUG_ON(object == fp); /* naive detection of double free or corruption */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) freeptr_addr = (unsigned long)kasan_reset_tag((void *)freeptr_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) *(void **)freeptr_addr = freelist_ptr(s, fp, freeptr_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /* Loop over all objects in a slab */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) #define for_each_object(__p, __s, __addr, __objects) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) for (__p = fixup_red_left(__s, __addr); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) __p < (__addr) + (__objects) * (__s)->size; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) __p += (__s)->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static inline unsigned int order_objects(unsigned int order, unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) return ((unsigned int)PAGE_SIZE << order) / size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) static inline struct kmem_cache_order_objects oo_make(unsigned int order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct kmem_cache_order_objects x = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) (order << OO_SHIFT) + order_objects(order, size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static inline unsigned int oo_order(struct kmem_cache_order_objects x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) return x.x >> OO_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) static inline unsigned int oo_objects(struct kmem_cache_order_objects x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return x.x & OO_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) * Per slab locking using the pagelock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static __always_inline void slab_lock(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) VM_BUG_ON_PAGE(PageTail(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) bit_spin_lock(PG_locked, &page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static __always_inline void slab_unlock(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) VM_BUG_ON_PAGE(PageTail(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) __bit_spin_unlock(PG_locked, &page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) /* Interrupts must be disabled (for the fallback code to work right) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static inline bool __cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) void *freelist_old, unsigned long counters_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) void *freelist_new, unsigned long counters_new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) const char *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) VM_BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (s->flags & __CMPXCHG_DOUBLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (cmpxchg_double(&page->freelist, &page->counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) freelist_old, counters_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) freelist_new, counters_new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (page->freelist == freelist_old &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) page->counters == counters_old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) page->freelist = freelist_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) page->counters = counters_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) stat(s, CMPXCHG_DOUBLE_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #ifdef SLUB_DEBUG_CMPXCHG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) pr_info("%s %s: cmpxchg double redo ", n, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) static inline bool cmpxchg_double_slab(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) void *freelist_old, unsigned long counters_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void *freelist_new, unsigned long counters_new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) const char *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (s->flags & __CMPXCHG_DOUBLE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) if (cmpxchg_double(&page->freelist, &page->counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) freelist_old, counters_old,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) freelist_new, counters_new))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (page->freelist == freelist_old &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) page->counters == counters_old) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) page->freelist = freelist_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) page->counters = counters_new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) stat(s, CMPXCHG_DOUBLE_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) #ifdef SLUB_DEBUG_CMPXCHG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) pr_info("%s %s: cmpxchg double redo ", n, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static unsigned long object_map[BITS_TO_LONGS(MAX_OBJS_PER_PAGE)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) static DEFINE_SPINLOCK(object_map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) void *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) bitmap_zero(obj_map, page->objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) for (p = page->freelist; p; p = get_freepointer(s, p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) set_bit(__obj_to_index(s, addr, p), obj_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * Determine a map of object in use on a page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * Node listlock must be held to guarantee that the page does
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * not vanish from under us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static unsigned long *get_map(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) __acquires(&object_map_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) VM_BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) spin_lock(&object_map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) __fill_map(object_map, s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return object_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static void put_map(unsigned long *map) __releases(&object_map_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) VM_BUG_ON(map != object_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) spin_unlock(&object_map_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) static inline unsigned int size_from_object(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return s->size - s->red_left_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) return s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) static inline void *restore_red_left(struct kmem_cache *s, void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) p -= s->red_left_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) * Debug settings:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) #if defined(CONFIG_SLUB_DEBUG_ON)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) slab_flags_t slub_debug = DEBUG_DEFAULT_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) slab_flags_t slub_debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) static char *slub_debug_string;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) static int disable_higher_order_debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * slub is about to manipulate internal object metadata. This memory lies
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * outside the range of the allocated object, so accessing it would normally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * be reported by kasan as a bounds error. metadata_access_enable() is used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * to tell kasan that these accesses are OK.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) static inline void metadata_access_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) kasan_disable_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) static inline void metadata_access_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) kasan_enable_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) * Object debugging
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) /* Verify that a pointer has an address that is valid within a slab page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) static inline int check_valid_pointer(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) struct page *page, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) void *base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (!object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) base = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) object = kasan_reset_tag(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) object = restore_red_left(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (object < base || object >= base + page->objects * s->size ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) (object - base) % s->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) static void print_section(char *level, char *text, u8 *addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) unsigned int length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) print_hex_dump(level, text, DUMP_PREFIX_ADDRESS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) 16, 1, kasan_reset_tag((void *)addr), length, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * See comment in calculate_sizes().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) static inline bool freeptr_outside_object(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) return s->offset >= s->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) * Return offset of the end of info block which is inuse + free pointer if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) * not overlapping with object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) static inline unsigned int get_info_end(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (freeptr_outside_object(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return s->inuse + sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return s->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) static struct track *get_track(struct kmem_cache *s, void *object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) enum track_item alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct track *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) p = object + get_info_end(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return kasan_reset_tag(p + alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) * This function will be used to loop through all the slab objects in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) * a page to give track structure for each object, the function fn will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) * be using this track structure and extract required info into its private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) * data, the return value will be the number of track structures that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) * processed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) unsigned long get_each_object_track(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct page *page, enum track_item alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) int (*fn)(const struct kmem_cache *, const void *,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) const struct track *, void *), void *private)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) struct track *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) unsigned long num_track = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) if (!slub_debug || !(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) for_each_object(p, s, page_address(page), page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) t = get_track(s, p, alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) ret = fn(s, p, t, private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) num_track += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) return num_track;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) EXPORT_SYMBOL_GPL(get_each_object_track);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static void set_track(struct kmem_cache *s, void *object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) enum track_item alloc, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) struct track *p = get_track(s, object, alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) if (addr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) #ifdef CONFIG_STACKTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) nr_entries = stack_trace_save(kasan_reset_tag(p->addrs),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) TRACK_ADDRS_COUNT, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (nr_entries < TRACK_ADDRS_COUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) p->addrs[nr_entries] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) trace_android_vh_save_track_hash(alloc == TRACK_ALLOC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) (unsigned long)p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) p->addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) p->cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) p->pid = current->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) p->when = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) memset(p, 0, sizeof(struct track));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) static void init_tracking(struct kmem_cache *s, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (!(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) set_track(s, object, TRACK_FREE, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) set_track(s, object, TRACK_ALLOC, 0UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) static void print_track(const char *s, struct track *t, unsigned long pr_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) if (!t->addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) pr_err("INFO: %s in %pS age=%lu cpu=%u pid=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) #ifdef CONFIG_STACKTRACE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) for (i = 0; i < TRACK_ADDRS_COUNT; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (t->addrs[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) pr_err("\t%pS\n", (void *)t->addrs[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) void print_tracking(struct kmem_cache *s, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) unsigned long pr_time = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) if (!(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) static void print_page_info(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) pr_err("INFO: Slab 0x%p objects=%u used=%u fp=0x%p flags=0x%04lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) page, page->objects, page->inuse, page->freelist, page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) static void slab_bug(struct kmem_cache *s, char *fmt, ...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) struct va_format vaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) va_list args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) va_start(args, fmt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) vaf.fmt = fmt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) vaf.va = &args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) pr_err("=============================================================================\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) pr_err("BUG %s (%s): %pV\n", s->name, print_tainted(), &vaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) pr_err("-----------------------------------------------------------------------------\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) va_end(args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) static void slab_fix(struct kmem_cache *s, char *fmt, ...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) struct va_format vaf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) va_list args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) va_start(args, fmt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) vaf.fmt = fmt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) vaf.va = &args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) pr_err("FIX %s: %pV\n", s->name, &vaf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) va_end(args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) void **freelist, void *nextfree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) !check_valid_pointer(s, page, nextfree) && freelist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) object_err(s, page, *freelist, "Freechain corrupt");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) *freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) slab_fix(s, "Isolate corrupted freechain");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static void print_trailer(struct kmem_cache *s, struct page *page, u8 *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) unsigned int off; /* Offset of last byte */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) u8 *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) print_tracking(s, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) print_page_info(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) pr_err("INFO: Object 0x%p @offset=%tu fp=0x%p\n\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) p, p - addr, get_freepointer(s, p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) s->red_left_pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) else if (p > addr + 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) print_section(KERN_ERR, "Bytes b4 ", p - 16, 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) print_section(KERN_ERR, "Object ", p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) min_t(unsigned int, s->object_size, PAGE_SIZE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) print_section(KERN_ERR, "Redzone ", p + s->object_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) s->inuse - s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) off = get_info_end(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) off += 2 * sizeof(struct track);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) off += kasan_metadata_size(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) if (off != size_from_object(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) /* Beginning of the filler is the free pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) print_section(KERN_ERR, "Padding ", p + off,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) size_from_object(s) - off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) void object_err(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) u8 *object, char *reason)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) slab_bug(s, "%s", reason);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) print_trailer(s, page, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) static __printf(3, 4) void slab_err(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) const char *fmt, ...)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) va_list args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) char buf[100];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) va_start(args, fmt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) vsnprintf(buf, sizeof(buf), fmt, args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) va_end(args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) slab_bug(s, "%s", buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) print_page_info(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) static void init_object(struct kmem_cache *s, void *object, u8 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) u8 *p = kasan_reset_tag(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) memset(p - s->red_left_pad, val, s->red_left_pad);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (s->flags & __OBJECT_POISON) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) memset(p, POISON_FREE, s->object_size - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) p[s->object_size - 1] = POISON_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) if (s->flags & SLAB_RED_ZONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) memset(p + s->object_size, val, s->inuse - s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) static void restore_bytes(struct kmem_cache *s, char *message, u8 data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) void *from, void *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) slab_fix(s, "Restoring 0x%p-0x%p=0x%x\n", from, to - 1, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) memset(from, data, to - from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) static int check_bytes_and_report(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) u8 *object, char *what,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) u8 *start, unsigned int value, unsigned int bytes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) u8 *fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) u8 *end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) u8 *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) fault = memchr_inv(kasan_reset_tag(start), value, bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (!fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) end = start + bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) while (end > fault && end[-1] == value)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) end--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) slab_bug(s, "%s overwritten", what);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) pr_err("INFO: 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) fault, end - 1, fault - addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) fault[0], value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) print_trailer(s, page, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) add_taint(TAINT_BAD_PAGE, LOCKDEP_NOW_UNRELIABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) restore_bytes(s, what, value, fault, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * Object layout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * object address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * Bytes of the object to be managed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * If the freepointer may overlay the object then the free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) * pointer is at the middle of the object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) * Poisoning uses 0x6b (POISON_FREE) and the last byte is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) * 0xa5 (POISON_END)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) * object + s->object_size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * Padding to reach word boundary. This is also used for Redzoning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) * Padding is extended by another word if Redzoning is enabled and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) * object_size == inuse.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) * We fill with 0xbb (RED_INACTIVE) for inactive objects and with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) * 0xcc (RED_ACTIVE) for objects in use.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) * object + s->inuse
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) * Meta data starts here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) * A. Free pointer (if we cannot overwrite object on free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * B. Tracking data for SLAB_STORE_USER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * C. Padding to reach required alignment boundary or at mininum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * one word if debugging is on to be able to detect writes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * before the word boundary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) * Padding is done using 0x5a (POISON_INUSE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * object + s->size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * Nothing is used beyond s->size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) * If slabcaches are merged then the object_size and inuse boundaries are mostly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) * ignored. And therefore no slab options that rely on these boundaries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) * may be used with merged slabcaches.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) static int check_pad_bytes(struct kmem_cache *s, struct page *page, u8 *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) unsigned long off = get_info_end(s); /* The end of info */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) /* We also have user information there */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) off += 2 * sizeof(struct track);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) off += kasan_metadata_size(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) if (size_from_object(s) == off)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return check_bytes_and_report(s, page, p, "Object padding",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) p + off, POISON_INUSE, size_from_object(s) - off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) /* Check the pad bytes at the end of a slab page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) static int slab_pad_check(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) u8 *start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) u8 *fault;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) u8 *end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) u8 *pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) int length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) int remainder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) if (!(s->flags & SLAB_POISON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) start = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) length = page_size(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) end = start + length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) remainder = length % s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) if (!remainder)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) pad = end - remainder;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) fault = memchr_inv(kasan_reset_tag(pad), POISON_INUSE, remainder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (!fault)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) while (end > fault && end[-1] == POISON_INUSE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) end--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) slab_err(s, page, "Padding overwritten. 0x%p-0x%p @offset=%tu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) fault, end - 1, fault - start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) print_section(KERN_ERR, "Padding ", pad, remainder);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) static int check_object(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) void *object, u8 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) u8 *p = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) u8 *endobject = object + s->object_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) if (s->flags & SLAB_RED_ZONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) if (!check_bytes_and_report(s, page, object, "Left Redzone",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) object - s->red_left_pad, val, s->red_left_pad))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) if (!check_bytes_and_report(s, page, object, "Right Redzone",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) endobject, val, s->inuse - s->object_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) check_bytes_and_report(s, page, p, "Alignment padding",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) endobject, POISON_INUSE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) s->inuse - s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) if (s->flags & SLAB_POISON) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) (!check_bytes_and_report(s, page, p, "Poison", p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) POISON_FREE, s->object_size - 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) !check_bytes_and_report(s, page, p, "End Poison",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) p + s->object_size - 1, POISON_END, 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) * check_pad_bytes cleans up on its own.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) check_pad_bytes(s, page, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if (!freeptr_outside_object(s) && val == SLUB_RED_ACTIVE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) * Object and freepointer overlap. Cannot check
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) * freepointer while object is allocated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) /* Check free pointer validity */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (!check_valid_pointer(s, page, get_freepointer(s, p))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) object_err(s, page, p, "Freepointer corrupt");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) * No choice but to zap it and thus lose the remainder
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) * of the free objects in this slab. May cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) * another error because the object count is now wrong.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) set_freepointer(s, p, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) static int check_slab(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) int maxobj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) VM_BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) if (!PageSlab(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) slab_err(s, page, "Not a valid slab page");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) maxobj = order_objects(compound_order(page), s->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) if (page->objects > maxobj) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) slab_err(s, page, "objects %u > max %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) page->objects, maxobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (page->inuse > page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) slab_err(s, page, "inuse %u > max %u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) page->inuse, page->objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) /* Slab_pad_check fixes things up after itself */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) slab_pad_check(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) * Determine if a certain object on a page is on the freelist. Must hold the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) * slab lock to guarantee that the chains are in a consistent state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) static int on_freelist(struct kmem_cache *s, struct page *page, void *search)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) int nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) void *fp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) void *object = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) int max_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) fp = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) while (fp && nr <= page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) if (fp == search)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (!check_valid_pointer(s, page, fp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) if (object) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) object_err(s, page, object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) "Freechain corrupt");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) set_freepointer(s, object, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) slab_err(s, page, "Freepointer corrupt");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) page->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) page->inuse = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) slab_fix(s, "Freelist cleared");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) object = fp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) fp = get_freepointer(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) nr++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) max_objects = order_objects(compound_order(page), s->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) if (max_objects > MAX_OBJS_PER_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) max_objects = MAX_OBJS_PER_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) if (page->objects != max_objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) slab_err(s, page, "Wrong number of objects. Found %d but should be %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) page->objects, max_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) page->objects = max_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) slab_fix(s, "Number of objects adjusted.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) if (page->inuse != page->objects - nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) slab_err(s, page, "Wrong object count. Counter is %d but counted were %d",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) page->inuse, page->objects - nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) page->inuse = page->objects - nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) slab_fix(s, "Object count adjusted.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) return search == NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) static void trace(struct kmem_cache *s, struct page *page, void *object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) int alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (s->flags & SLAB_TRACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) s->name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) alloc ? "alloc" : "free",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) object, page->inuse,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) page->freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (!alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) print_section(KERN_INFO, "Object ", (void *)object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * Tracking of fully allocated slabs for debugging purposes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) static void add_full(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) struct kmem_cache_node *n, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) if (!(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) lockdep_assert_held(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) list_add(&page->slab_list, &n->full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) if (!(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) lockdep_assert_held(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) list_del(&page->slab_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) /* Tracking of the number of slabs for debugging purposes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) static inline unsigned long slabs_node(struct kmem_cache *s, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) struct kmem_cache_node *n = get_node(s, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return atomic_long_read(&n->nr_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return atomic_long_read(&n->nr_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) struct kmem_cache_node *n = get_node(s, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) * May be called early in order to allocate a slab for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * kmem_cache_node structure. Solve the chicken-egg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) * dilemma by deferring the increment of the count during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) * bootstrap (see early_kmem_cache_node_alloc).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) if (likely(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) atomic_long_inc(&n->nr_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) atomic_long_add(objects, &n->total_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) struct kmem_cache_node *n = get_node(s, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) atomic_long_dec(&n->nr_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) atomic_long_sub(objects, &n->total_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) /* Object debug checks for alloc/free paths */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) static void setup_object_debug(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) init_object(s, object, SLUB_RED_INACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) init_tracking(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) static
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) if (!kmem_cache_debug_flags(s, SLAB_POISON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) metadata_access_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) memset(kasan_reset_tag(addr), POISON_INUSE, page_size(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) metadata_access_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) static inline int alloc_consistency_checks(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) struct page *page, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) if (!check_slab(s, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if (!check_valid_pointer(s, page, object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) object_err(s, page, object, "Freelist Pointer check fails");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) if (!check_object(s, page, object, SLUB_RED_INACTIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) static noinline int alloc_debug_processing(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) void *object, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (s->flags & SLAB_CONSISTENCY_CHECKS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) if (!alloc_consistency_checks(s, page, object))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) goto bad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) /* Success perform special debug activities for allocs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) set_track(s, object, TRACK_ALLOC, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) trace(s, page, object, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) init_object(s, object, SLUB_RED_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) bad:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) if (PageSlab(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * If this is a slab page then lets do the best we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * to avoid issues in the future. Marking all objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) * as used avoids touching the remaining objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) slab_fix(s, "Marking all objects used");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) page->inuse = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) page->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) static inline int free_consistency_checks(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) struct page *page, void *object, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) if (!check_valid_pointer(s, page, object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) slab_err(s, page, "Invalid object pointer 0x%p", object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) if (on_freelist(s, page, object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) object_err(s, page, object, "Object already free");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (!check_object(s, page, object, SLUB_RED_ACTIVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) if (unlikely(s != page->slab_cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) if (!PageSlab(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) slab_err(s, page, "Attempt to free object(0x%p) outside of slab",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) } else if (!page->slab_cache) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) pr_err("SLUB <none>: no slab for object 0x%p.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) object_err(s, page, object,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) "page slab pointer corrupt.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) /* Supports checking bulk free of a constructed freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) static noinline int free_debug_processing(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) void *head, void *tail, int bulk_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) struct kmem_cache_node *n = get_node(s, page_to_nid(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) void *object = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) int cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) if (s->flags & SLAB_CONSISTENCY_CHECKS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) if (!check_slab(s, page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) next_object:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) if (s->flags & SLAB_CONSISTENCY_CHECKS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (!free_consistency_checks(s, page, object, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) set_track(s, object, TRACK_FREE, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) trace(s, page, object, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) /* Freepointer not overwritten by init_object(), SLAB_POISON moved it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) init_object(s, object, SLUB_RED_INACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /* Reached end of constructed freelist yet? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (object != tail) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) object = get_freepointer(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) goto next_object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) if (cnt != bulk_cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) slab_err(s, page, "Bulk freelist count(%d) invalid(%d)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) bulk_cnt, cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) if (!ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) slab_fix(s, "Object at 0x%p not freed", object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * Parse a block of slub_debug options. Blocks are delimited by ';'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * @str: start of block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) * @flags: returns parsed flags, or DEBUG_DEFAULT_FLAGS if none specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) * @slabs: return start of list of slabs, or NULL when there's no list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) * @init: assume this is initial parsing and not per-kmem-create parsing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) * returns the start of next block if there's any, or NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) static char *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) parse_slub_debug_flags(char *str, slab_flags_t *flags, char **slabs, bool init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) bool higher_order_disable = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) /* Skip any completely empty blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) while (*str && *str == ';')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) str++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) if (*str == ',') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) * No options but restriction on slabs. This means full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) * debugging for slabs matching a pattern.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) *flags = DEBUG_DEFAULT_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) goto check_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) *flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) /* Determine which debug features should be switched on */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) for (; *str && *str != ',' && *str != ';'; str++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) switch (tolower(*str)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) case '-':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) *flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) case 'f':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) *flags |= SLAB_CONSISTENCY_CHECKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) case 'z':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) *flags |= SLAB_RED_ZONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) case 'p':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) *flags |= SLAB_POISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) case 'u':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) *flags |= SLAB_STORE_USER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) case 't':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) *flags |= SLAB_TRACE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) case 'a':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) *flags |= SLAB_FAILSLAB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) case 'o':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * Avoid enabling debugging on caches if its minimum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * order would increase as a result.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) higher_order_disable = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) if (init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) pr_err("slub_debug option '%c' unknown. skipped\n", *str);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) check_slabs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) if (*str == ',')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) *slabs = ++str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) *slabs = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) /* Skip over the slab list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) while (*str && *str != ';')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) str++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) /* Skip any completely empty blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) while (*str && *str == ';')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) str++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) if (init && higher_order_disable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) disable_higher_order_debug = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) if (*str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) return str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) static int __init setup_slub_debug(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) slab_flags_t flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) slab_flags_t global_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) char *saved_str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) char *slab_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) bool global_slub_debug_changed = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) bool slab_list_specified = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) global_flags = DEBUG_DEFAULT_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) if (*str++ != '=' || !*str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) * No options specified. Switch on full debugging.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) saved_str = str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) while (str) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) str = parse_slub_debug_flags(str, &flags, &slab_list, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) if (!slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) global_flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) global_slub_debug_changed = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) slab_list_specified = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) * For backwards compatibility, a single list of flags with list of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) * slabs means debugging is only changed for those slabs, so the global
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) * slub_debug should be unchanged (0 or DEBUG_DEFAULT_FLAGS, depending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) * on CONFIG_SLUB_DEBUG_ON). We can extended that to multiple lists as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) * long as there is no option specifying flags without a slab list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) if (slab_list_specified) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) if (!global_slub_debug_changed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) global_flags = slub_debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) slub_debug_string = saved_str;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) slub_debug = global_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) if (slub_debug != 0 || slub_debug_string)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static_branch_enable(&slub_debug_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) if ((static_branch_unlikely(&init_on_alloc) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) static_branch_unlikely(&init_on_free)) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) (slub_debug & SLAB_POISON))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) __setup("slub_debug", setup_slub_debug);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) * kmem_cache_flags - apply debugging options to the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) * @object_size: the size of an object without meta data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) * @flags: flags to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) * @name: name of the cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) * Debug option(s) are applied to @flags. In addition to the debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) * option(s), if a slab name (or multiple) is specified i.e.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) * slub_debug=<Debug-Options>,<slab name1>,<slab name2> ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) * then only the select slabs will receive the debug option(s).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) slab_flags_t kmem_cache_flags(unsigned int object_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) slab_flags_t flags, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) char *iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) size_t len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) char *next_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) slab_flags_t block_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) len = strlen(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) next_block = slub_debug_string;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) /* Go through all blocks of debug options, see if any matches our slab's name */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) while (next_block) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) next_block = parse_slub_debug_flags(next_block, &block_flags, &iter, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (!iter)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) /* Found a block that has a slab list, search it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) while (*iter) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) char *end, *glob;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) size_t cmplen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) end = strchrnul(iter, ',');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) if (next_block && next_block < end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) end = next_block - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) glob = strnchr(iter, end - iter, '*');
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (glob)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) cmplen = glob - iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) cmplen = max_t(size_t, len, (end - iter));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) if (!strncmp(name, iter, cmplen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) flags |= block_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) return flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) if (!*end || *end == ';')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) iter = end + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) return flags | slub_debug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) #else /* !CONFIG_SLUB_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) static inline void setup_object_debug(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) struct page *page, void *object) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) void setup_page_debug(struct kmem_cache *s, struct page *page, void *addr) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) static inline int alloc_debug_processing(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) struct page *page, void *object, unsigned long addr) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) static inline int free_debug_processing(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) void *head, void *tail, int bulk_cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) unsigned long addr) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) static inline int slab_pad_check(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) { return 1; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) static inline int check_object(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) void *object, u8 val) { return 1; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) struct page *page) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) struct page *page) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) slab_flags_t kmem_cache_flags(unsigned int object_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) slab_flags_t flags, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) return flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) #define slub_debug 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) #define disable_higher_order_debug 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) static inline unsigned long slabs_node(struct kmem_cache *s, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) static inline unsigned long node_nr_slabs(struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) static inline void inc_slabs_node(struct kmem_cache *s, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) int objects) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) static inline void dec_slabs_node(struct kmem_cache *s, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) int objects) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) static bool freelist_corrupted(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) void **freelist, void *nextfree)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) #endif /* CONFIG_SLUB_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) * Hooks for other subsystems that check memory allocations. In a typical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) * production configuration these hooks all should produce no code at all.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) static inline void *kmalloc_large_node_hook(void *ptr, size_t size, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) ptr = kasan_kmalloc_large(ptr, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) /* As ptr might get tagged, call kmemleak hook after KASAN. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) kmemleak_alloc(ptr, size, 1, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) static __always_inline void kfree_hook(void *x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) kmemleak_free(x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) kasan_kfree_large(x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) static __always_inline bool slab_free_hook(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) void *x, bool init)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) kmemleak_free_recursive(x, s->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) * Trouble is that we may no longer disable interrupts in the fast path
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) * So in order to make the debug calls that expect irqs to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) * disabled we need to disable interrupts temporarily.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) #ifdef CONFIG_LOCKDEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) debug_check_no_locks_freed(x, s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) if (!(s->flags & SLAB_DEBUG_OBJECTS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) debug_check_no_obj_freed(x, s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) /* Use KCSAN to help debug racy use-after-free. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) if (!(s->flags & SLAB_TYPESAFE_BY_RCU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) __kcsan_check_access(x, s->object_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ASSERT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) * As memory initialization might be integrated into KASAN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) * kasan_slab_free and initialization memset's must be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) * kept together to avoid discrepancies in behavior.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) * The initialization memset's clear the object and the metadata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) * but don't touch the SLAB redzone.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) if (init) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) int rsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) if (!kasan_has_integrated_init())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) memset(kasan_reset_tag(x), 0, s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) memset((char *)kasan_reset_tag(x) + s->inuse, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) s->size - s->inuse - rsize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) /* KASAN might put x into memory quarantine, delaying its reuse. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return kasan_slab_free(s, x, init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) static inline bool slab_free_freelist_hook(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) void **head, void **tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) int *cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) void *object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) void *next = *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) void *old_tail = *tail ? *tail : *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) if (is_kfence_address(next)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) slab_free_hook(s, next, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) /* Head and tail of the reconstructed freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) *head = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) *tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) object = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) next = get_freepointer(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) /* If object's reuse doesn't have to be delayed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) if (!slab_free_hook(s, object, slab_want_init_on_free(s))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) /* Move object to the new freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) set_freepointer(s, object, *head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) *head = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) if (!*tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) *tail = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) * Adjust the reconstructed freelist depth
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) * accordingly if object's reuse is delayed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) --(*cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) } while (object != old_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) if (*head == *tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) *tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) return *head != NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) static void *setup_object(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) setup_object_debug(s, page, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) object = kasan_init_slab_obj(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) if (unlikely(s->ctor)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) kasan_unpoison_object_data(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) s->ctor(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) kasan_poison_object_data(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) return object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) * Slab allocation and freeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) static inline struct page *alloc_slab_page(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) gfp_t flags, int node, struct kmem_cache_order_objects oo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) unsigned int order = oo_order(oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) if (node == NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) page = alloc_pages(flags, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) page = __alloc_pages_node(node, flags, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) account_slab_page(page, order, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) #ifdef CONFIG_SLAB_FREELIST_RANDOM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) /* Pre-initialize the random sequence cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) static int init_cache_random_seq(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) unsigned int count = oo_objects(s->oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) /* Bailout if already initialised */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (s->random_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) err = cache_random_seq_create(s, count, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) pr_err("SLUB: Unable to initialize free list for %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) /* Transform to an offset on the set of pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) if (s->random_seq) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) for (i = 0; i < count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) s->random_seq[i] *= s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) /* Initialize each random sequence freelist per cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) static void __init init_freelist_randomization(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) list_for_each_entry(s, &slab_caches, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) init_cache_random_seq(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) /* Get the next entry on the pre-computed freelist randomized */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) static void *next_freelist_entry(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) unsigned long *pos, void *start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) unsigned long page_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) unsigned long freelist_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) unsigned int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) * If the target page allocation failed, the number of objects on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) * page might be smaller than the usual size defined by the cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) idx = s->random_seq[*pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) *pos += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) if (*pos >= freelist_count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) *pos = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) } while (unlikely(idx >= page_limit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) return (char *)start + idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) /* Shuffle the single linked freelist based on a random pre-computed sequence */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) static bool shuffle_freelist(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) void *start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) void *cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) void *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) unsigned long idx, pos, page_limit, freelist_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) if (page->objects < 2 || !s->random_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) freelist_count = oo_objects(s->oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) pos = get_random_int() % freelist_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) page_limit = page->objects * s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) start = fixup_red_left(s, page_address(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) /* First entry is used as the base of the freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) cur = next_freelist_entry(s, page, &pos, start, page_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) freelist_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) cur = setup_object(s, page, cur);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) page->freelist = cur;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) for (idx = 1; idx < page->objects; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) next = next_freelist_entry(s, page, &pos, start, page_limit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) freelist_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) next = setup_object(s, page, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) set_freepointer(s, cur, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) cur = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) set_freepointer(s, cur, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) static inline int init_cache_random_seq(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) static inline void init_freelist_randomization(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) static inline bool shuffle_freelist(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) #endif /* CONFIG_SLAB_FREELIST_RANDOM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) static struct page *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) struct kmem_cache_order_objects oo = s->oo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) gfp_t alloc_gfp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) void *start, *p, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) bool shuffle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) flags &= gfp_allowed_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) if (gfpflags_allow_blocking(flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) flags |= s->allocflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) * Let the initial higher-order allocation fail under memory pressure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * so we fall-back to the minimum order allocation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) alloc_gfp = (flags | __GFP_NOWARN | __GFP_NORETRY) & ~__GFP_NOFAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) alloc_gfp = (alloc_gfp | __GFP_NOMEMALLOC) & ~(__GFP_RECLAIM|__GFP_NOFAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) page = alloc_slab_page(s, alloc_gfp, node, oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) if (unlikely(!page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) oo = s->min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) alloc_gfp = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) * Allocation may have failed due to fragmentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) * Try a lower order alloc if possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) page = alloc_slab_page(s, alloc_gfp, node, oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826) if (unlikely(!page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) stat(s, ORDER_FALLBACK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) page->objects = oo_objects(oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) page->slab_cache = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) __SetPageSlab(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) if (page_is_pfmemalloc(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) SetPageSlabPfmemalloc(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) kasan_poison_slab(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) start = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) setup_page_debug(s, page, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) shuffle = shuffle_freelist(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) if (!shuffle) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) start = fixup_red_left(s, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) start = setup_object(s, page, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) page->freelist = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) for (idx = 0, p = start; idx < page->objects - 1; idx++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) next = p + s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) next = setup_object(s, page, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) set_freepointer(s, p, next);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) p = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) set_freepointer(s, p, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) page->inuse = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) page->frozen = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) if (gfpflags_allow_blocking(flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) inc_slabs_node(s, page_to_nid(page), page->objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) static struct page *new_slab(struct kmem_cache *s, gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) if (unlikely(flags & GFP_SLAB_BUG_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) flags = kmalloc_fix_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) return allocate_slab(s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) flags & (GFP_RECLAIM_MASK | GFP_CONSTRAINT_MASK), node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) static void __free_slab(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) int order = compound_order(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) int pages = 1 << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) slab_pad_check(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) for_each_object(p, s, page_address(page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) page->objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) check_object(s, page, p, SLUB_RED_INACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) __ClearPageSlabPfmemalloc(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) __ClearPageSlab(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) page->mapping = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) if (current->reclaim_state)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) current->reclaim_state->reclaimed_slab += pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) unaccount_slab_page(page, order, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) __free_pages(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) static void rcu_free_slab(struct rcu_head *h)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) struct page *page = container_of(h, struct page, rcu_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) __free_slab(page->slab_cache, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) static void free_slab(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) call_rcu(&page->rcu_head, rcu_free_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) __free_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) static void discard_slab(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) dec_slabs_node(s, page_to_nid(page), page->objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) free_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) * Management of partially allocated slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) __add_partial(struct kmem_cache_node *n, struct page *page, int tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) n->nr_partial++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) if (tail == DEACTIVATE_TO_TAIL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) list_add_tail(&page->slab_list, &n->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) list_add(&page->slab_list, &n->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) static inline void add_partial(struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) struct page *page, int tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) lockdep_assert_held(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) __add_partial(n, page, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) static inline void remove_partial(struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) lockdep_assert_held(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) list_del(&page->slab_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) n->nr_partial--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) * Remove slab from the partial list, freeze it and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) * return the pointer to the freelist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) * Returns a list of objects or NULL if it fails.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) static inline void *acquire_slab(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) struct kmem_cache_node *n, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) int mode, int *objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) void *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) unsigned long counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) struct page new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) lockdep_assert_held(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) * Zap the freelist and set the frozen bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) * The old freelist is the list of objects for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) * per cpu allocation list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) freelist = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) new.counters = counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) *objects = new.objects - new.inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) if (mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) new.inuse = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) new.freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) new.freelist = freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) VM_BUG_ON(new.frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) new.frozen = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) if (!__cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) freelist, counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) new.freelist, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) "acquire_slab"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) remove_partial(n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) WARN_ON(!freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) * Try to allocate a partial slab from a specific node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) static void *get_partial_node(struct kmem_cache *s, struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) struct kmem_cache_cpu *c, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) struct page *page, *page2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) void *object = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) unsigned int available = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) int objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) * Racy check. If we mistakenly see no partial slabs then we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) * just allocate an empty slab. If we mistakenly try to get a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) * partial slab and there is none available then get_partial()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) * will return NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) if (!n || !n->nr_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) spin_lock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) list_for_each_entry_safe(page, page2, &n->partial, slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) void *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) if (!pfmemalloc_match(page, flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) t = acquire_slab(s, n, page, object == NULL, &objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) available += objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) if (!object) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) c->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) stat(s, ALLOC_FROM_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) object = t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) put_cpu_partial(s, page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) stat(s, CPU_PARTIAL_NODE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) if (!kmem_cache_has_cpu_partial(s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) || available > slub_cpu_partial(s) / 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) spin_unlock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) return object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) * Get a page from somewhere. Search in increasing NUMA distances.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) static void *get_any_partial(struct kmem_cache *s, gfp_t flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) struct zonelist *zonelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) struct zoneref *z;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct zone *zone;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) enum zone_type highest_zoneidx = gfp_zone(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) void *object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) unsigned int cpuset_mems_cookie;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) * The defrag ratio allows a configuration of the tradeoffs between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) * inter node defragmentation and node local allocations. A lower
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) * defrag_ratio increases the tendency to do local allocations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) * instead of attempting to obtain partial slabs from other nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) * If the defrag_ratio is set to 0 then kmalloc() always
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) * returns node local objects. If the ratio is higher then kmalloc()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) * may return off node objects because partial slabs are obtained
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) * from other nodes and filled up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) * If /sys/kernel/slab/xx/remote_node_defrag_ratio is set to 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) * (which makes defrag_ratio = 1000) then every (well almost)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) * allocation will first attempt to defrag slab caches on other nodes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) * This means scanning over all nodes to look for partial slabs which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) * may be expensive if we do it every time we are trying to find a slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) * with available objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) if (!s->remote_node_defrag_ratio ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) get_cycles() % 1024 > s->remote_node_defrag_ratio)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) cpuset_mems_cookie = read_mems_allowed_begin();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) zonelist = node_zonelist(mempolicy_slab_node(), flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) for_each_zone_zonelist(zone, z, zonelist, highest_zoneidx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) n = get_node(s, zone_to_nid(zone));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) if (n && cpuset_zone_allowed(zone, flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) n->nr_partial > s->min_partial) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) object = get_partial_node(s, n, c, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) if (object) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) * Don't check read_mems_allowed_retry()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) * here - if mems_allowed was updated in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) * parallel, that was a harmless race
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) * between allocation and the cpuset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) * update
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) return object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) } while (read_mems_allowed_retry(cpuset_mems_cookie));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) #endif /* CONFIG_NUMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * Get a partial page, lock it and return it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) static void *get_partial(struct kmem_cache *s, gfp_t flags, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) void *object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) int searchnode = node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) if (node == NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) searchnode = numa_mem_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) object = get_partial_node(s, get_node(s, searchnode), c, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) if (object || node != NUMA_NO_NODE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) return object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) return get_any_partial(s, flags, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) #ifdef CONFIG_PREEMPTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) * Calculate the next globally unique transaction for disambiguation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) * during cmpxchg. The transactions start with the cpu number and are then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) * incremented by CONFIG_NR_CPUS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) #define TID_STEP roundup_pow_of_two(CONFIG_NR_CPUS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) * No preemption supported therefore also no need to check for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) * different cpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) #define TID_STEP 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) static inline unsigned long next_tid(unsigned long tid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) return tid + TID_STEP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) #ifdef SLUB_DEBUG_CMPXCHG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) static inline unsigned int tid_to_cpu(unsigned long tid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) return tid % TID_STEP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) static inline unsigned long tid_to_event(unsigned long tid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) return tid / TID_STEP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168) static inline unsigned int init_tid(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) return cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) static inline void note_cmpxchg_failure(const char *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) const struct kmem_cache *s, unsigned long tid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) #ifdef SLUB_DEBUG_CMPXCHG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) pr_info("%s %s: cmpxchg redo ", n, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) #ifdef CONFIG_PREEMPTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) if (tid_to_cpu(tid) != tid_to_cpu(actual_tid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) pr_warn("due to cpu change %d -> %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) tid_to_cpu(tid), tid_to_cpu(actual_tid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) if (tid_to_event(tid) != tid_to_event(actual_tid))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) pr_warn("due to cpu running other code. Event %ld->%ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) tid_to_event(tid), tid_to_event(actual_tid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) pr_warn("for unknown reason: actual=%lx was=%lx target=%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) actual_tid, tid, next_tid(tid));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) static void init_kmem_cache_cpus(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) per_cpu_ptr(s->cpu_slab, cpu)->tid = init_tid(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) * Remove the cpu slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) static void deactivate_slab(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) void *freelist, struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) enum slab_modes { M_NONE, M_PARTIAL, M_FULL, M_FREE };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) struct kmem_cache_node *n = get_node(s, page_to_nid(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) int lock = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) enum slab_modes l = M_NONE, m = M_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) void *nextfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) int tail = DEACTIVATE_TO_HEAD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) struct page new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) struct page old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) if (page->freelist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) stat(s, DEACTIVATE_REMOTE_FREES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) tail = DEACTIVATE_TO_TAIL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) * Stage one: Free all available per cpu objects back
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) * to the page freelist while it is still frozen. Leave the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) * last one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) * There is no need to take the list->lock because the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) * is still frozen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) while (freelist && (nextfree = get_freepointer(s, freelist))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) void *prior;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) unsigned long counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) * If 'nextfree' is invalid, it is possible that the object at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) * 'freelist' is already corrupted. So isolate all objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) * starting at 'freelist'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) if (freelist_corrupted(s, page, &freelist, nextfree))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) prior = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) set_freepointer(s, freelist, prior);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) new.counters = counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) new.inuse--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) VM_BUG_ON(!new.frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) } while (!__cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) prior, counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) freelist, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) "drain percpu freelist"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) freelist = nextfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) * Stage two: Ensure that the page is unfrozen while the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) * list presence reflects the actual number of objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) * during unfreeze.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) * We setup the list membership and then perform a cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) * with the count. If there is a mismatch then the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) * is not unfrozen but the page is on the wrong list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) * Then we restart the process which may have to remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) * the page from the list that we just put it on again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) * because the number of objects in the slab may have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) * changed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) redo:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) old.freelist = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) old.counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) VM_BUG_ON(!old.frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) /* Determine target state of the slab */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) new.counters = old.counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) if (freelist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) new.inuse--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) set_freepointer(s, freelist, old.freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) new.freelist = freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) new.freelist = old.freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) new.frozen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) if (!new.inuse && n->nr_partial >= s->min_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) m = M_FREE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) else if (new.freelist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) m = M_PARTIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) if (!lock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) lock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) * Taking the spinlock removes the possibility
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) * that acquire_slab() will see a slab page that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) * is frozen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) spin_lock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) m = M_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) if ((s->flags & SLAB_STORE_USER) && !lock) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) lock = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) * This also ensures that the scanning of full
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) * slabs from diagnostic functions will not see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) * any frozen slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) spin_lock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (l != m) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) if (l == M_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) remove_partial(n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) else if (l == M_FULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) remove_full(s, n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) if (m == M_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) add_partial(n, page, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) else if (m == M_FULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) add_full(s, n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) l = m;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) if (!__cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) old.freelist, old.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) new.freelist, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) "unfreezing slab"))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) goto redo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) if (lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340) spin_unlock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) if (m == M_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) stat(s, tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) else if (m == M_FULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) stat(s, DEACTIVATE_FULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) else if (m == M_FREE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) stat(s, DEACTIVATE_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) discard_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) stat(s, FREE_SLAB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) c->page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) c->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) * Unfreeze all the cpu partial slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) * This function must be called with interrupts disabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) * for the cpu using c (or some other guarantee must be there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) * to guarantee no concurrent accesses).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) static void unfreeze_partials(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) #ifdef CONFIG_SLUB_CPU_PARTIAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) struct kmem_cache_node *n = NULL, *n2 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) struct page *page, *discard_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) while ((page = slub_percpu_partial(c))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) struct page new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) struct page old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) slub_set_percpu_partial(c, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) n2 = get_node(s, page_to_nid(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) if (n != n2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) spin_unlock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) n = n2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) spin_lock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) old.freelist = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) old.counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) VM_BUG_ON(!old.frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) new.counters = old.counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) new.freelist = old.freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) new.frozen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) } while (!__cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) old.freelist, old.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) new.freelist, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) "unfreezing slab"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) page->next = discard_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) discard_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) add_partial(n, page, DEACTIVATE_TO_TAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) stat(s, FREE_ADD_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) if (n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) spin_unlock(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) while (discard_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) page = discard_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) discard_page = discard_page->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) stat(s, DEACTIVATE_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) discard_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) stat(s, FREE_SLAB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) #endif /* CONFIG_SLUB_CPU_PARTIAL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) * Put a page that was just frozen (in __slab_free|get_partial_node) into a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) * partial page slot if available.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) * If we did not find a slot then simply move all the partials to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * per node partial list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) static void put_cpu_partial(struct kmem_cache *s, struct page *page, int drain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) #ifdef CONFIG_SLUB_CPU_PARTIAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) struct page *oldpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) int pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) int pobjects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) pobjects = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) oldpage = this_cpu_read(s->cpu_slab->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) if (oldpage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) pobjects = oldpage->pobjects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) pages = oldpage->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) if (drain && pobjects > slub_cpu_partial(s)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) * partial array is full. Move the existing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) * set to the per node partial list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) oldpage = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) pobjects = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) stat(s, CPU_PARTIAL_DRAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) pobjects += page->objects - page->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) page->pages = pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) page->pobjects = pobjects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) page->next = oldpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) } while (this_cpu_cmpxchg(s->cpu_slab->partial, oldpage, page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) != oldpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) if (unlikely(!slub_cpu_partial(s))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) unfreeze_partials(s, this_cpu_ptr(s->cpu_slab));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) #endif /* CONFIG_SLUB_CPU_PARTIAL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) stat(s, CPUSLAB_FLUSH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) deactivate_slab(s, c->page, c->freelist, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488) c->tid = next_tid(c->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) * Flush cpu slab.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494) * Called from IPI handler with interrupts disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) if (c->page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) flush_slab(s, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) unfreeze_partials(s, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) static void flush_cpu_slab(void *d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) struct kmem_cache *s = d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) __flush_cpu_slab(s, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) static bool has_cpu_slab(int cpu, void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) struct kmem_cache *s = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) return c->page || slub_percpu_partial(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) static void flush_all(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) on_each_cpu_cond(has_cpu_slab, flush_cpu_slab, s, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) * Use the cpu notifier to insure that the cpu slabs are flushed when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) * necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530) static int slub_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) list_for_each_entry(s, &slab_caches, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) __flush_cpu_slab(s, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) * Check if the objects in a per cpu structure fit numa
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) * locality expectations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) static inline int node_match(struct page *page, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) if (node != NUMA_NO_NODE && page_to_nid(page) != node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) static int count_free(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) return page->objects - page->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) static inline unsigned long node_nr_objs(struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) return atomic_long_read(&n->total_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) #endif /* CONFIG_SLUB_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) #if defined(CONFIG_SLUB_DEBUG) || defined(CONFIG_SLUB_SYSFS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) static unsigned long count_partial(struct kmem_cache_node *n,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) int (*get_count)(struct page *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575) unsigned long x = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) list_for_each_entry(page, &n->partial, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) x += get_count(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) return x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) #endif /* CONFIG_SLUB_DEBUG || CONFIG_SLUB_SYSFS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586) static noinline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) static DEFINE_RATELIMIT_STATE(slub_oom_rs, DEFAULT_RATELIMIT_INTERVAL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) DEFAULT_RATELIMIT_BURST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) if ((gfpflags & __GFP_NOWARN) || !__ratelimit(&slub_oom_rs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598) pr_warn("SLUB: Unable to allocate memory on node %d, gfp=%#x(%pGg)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) nid, gfpflags, &gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) s->name, s->object_size, s->size, oo_order(s->oo),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) oo_order(s->min));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) if (oo_order(s->min) > get_order(s->object_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) pr_warn(" %s debugging increased min order, use slub_debug=O to disable.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606) s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) unsigned long nr_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) unsigned long nr_objs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) unsigned long nr_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) nr_free = count_partial(n, count_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) nr_slabs = node_nr_slabs(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) nr_objs = node_nr_objs(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) pr_warn(" node %d: slabs: %ld, objs: %ld, free: %ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) node, nr_slabs, nr_objs, nr_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623) static inline void *new_slab_objects(struct kmem_cache *s, gfp_t flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) int node, struct kmem_cache_cpu **pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) void *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) struct kmem_cache_cpu *c = *pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) freelist = get_partial(s, flags, node, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) if (freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637) page = new_slab(s, flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) c = raw_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) if (c->page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) flush_slab(s, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) * No other reference to the page yet so we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645) * muck around with it freely without cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) freelist = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) page->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) stat(s, ALLOC_SLAB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651) c->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) *pc = c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) static inline bool pfmemalloc_match(struct page *page, gfp_t gfpflags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) if (unlikely(PageSlabPfmemalloc(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661) return gfp_pfmemalloc_allowed(gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) * Check the page->freelist of a page and either transfer the freelist to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) * per cpu freelist or deactivate the page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) * The page is still frozen if the return value is not NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) * If this function returns NULL then the page has been unfrozen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) * This function must be called with interrupt disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) static inline void *get_freelist(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) struct page new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) unsigned long counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) void *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) freelist = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686) new.counters = counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) VM_BUG_ON(!new.frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) new.inuse = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690) new.frozen = freelist != NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) } while (!__cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) freelist, counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) NULL, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) "get_freelist"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) * Slow path. The lockless freelist is empty or we need to perform
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) * debugging duties.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) * Processing is still very fast if new objects have been freed to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) * regular freelist. In that case we simply take over the regular freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) * as the lockless freelist and zap the regular freelist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) * If that is not working then we fall back to the partial lists. We take the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) * first element of the freelist as the object to allocate now and move the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2710) * rest of the freelist to the lockless freelist.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2711) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2712) * And if we were unable to get a new slab from the partial slab lists then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2713) * we need to allocate a new slab. This is the slowest path since it involves
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2714) * a call to the page allocator and the setup of a new slab.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2715) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2716) * Version of __slab_alloc to use when we know that interrupts are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2717) * already disabled (which is the case for bulk allocation).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2718) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2719) static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2720) unsigned long addr, struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2721) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2722) void *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2723) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2725) stat(s, ALLOC_SLOWPATH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2727) page = c->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2728) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2729) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2730) * if the node is not online or has no normal memory, just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2731) * ignore the node constraint
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2732) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2733) if (unlikely(node != NUMA_NO_NODE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2734) !node_state(node, N_NORMAL_MEMORY)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2735) node = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2736) goto new_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2737) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2738) redo:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2740) if (unlikely(!node_match(page, node))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2741) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2742) * same as above but node_match() being false already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2743) * implies node != NUMA_NO_NODE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2745) if (!node_state(node, N_NORMAL_MEMORY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2746) node = NUMA_NO_NODE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2747) goto redo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2748) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2749) stat(s, ALLOC_NODE_MISMATCH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2750) deactivate_slab(s, page, c->freelist, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2751) goto new_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2753) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2755) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2756) * By rights, we should be searching for a slab page that was
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2757) * PFMEMALLOC but right now, we are losing the pfmemalloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2758) * information when the page leaves the per-cpu allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2759) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2760) if (unlikely(!pfmemalloc_match(page, gfpflags))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2761) deactivate_slab(s, page, c->freelist, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2762) goto new_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2763) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2764)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2765) /* must check again c->freelist in case of cpu migration or IRQ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2766) freelist = c->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2767) if (freelist)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2768) goto load_freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2770) freelist = get_freelist(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2772) if (!freelist) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2773) c->page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2774) stat(s, DEACTIVATE_BYPASS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2775) goto new_slab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2778) stat(s, ALLOC_REFILL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2780) load_freelist:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2781) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2782) * freelist is pointing to the list of objects to be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2783) * page is pointing to the page from which the objects are obtained.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2784) * That page must be frozen for per cpu allocations to work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2785) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2786) VM_BUG_ON(!c->page->frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2787) c->freelist = get_freepointer(s, freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2788) c->tid = next_tid(c->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2789) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2790)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2791) new_slab:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2793) if (slub_percpu_partial(c)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2794) page = c->page = slub_percpu_partial(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2795) slub_set_percpu_partial(c, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2796) stat(s, CPU_PARTIAL_ALLOC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2797) goto redo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2800) freelist = new_slab_objects(s, gfpflags, node, &c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2802) if (unlikely(!freelist)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2803) slab_out_of_memory(s, gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2804) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2805) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2807) page = c->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2808) if (likely(!kmem_cache_debug(s) && pfmemalloc_match(page, gfpflags)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2809) goto load_freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2811) /* Only entered in the debug case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2812) if (kmem_cache_debug(s) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2813) !alloc_debug_processing(s, page, freelist, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2814) goto new_slab; /* Slab failed checks. Next slab needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2816) deactivate_slab(s, page, get_freepointer(s, freelist), c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2817) return freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2820) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2821) * Another one that disabled interrupt and compensates for possible
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2822) * cpu changes by refetching the per cpu area pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2823) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2824) static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2825) unsigned long addr, struct kmem_cache_cpu *c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2826) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2827) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2828) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2830) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2831) #ifdef CONFIG_PREEMPTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2832) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2833) * We may have been preempted and rescheduled on a different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2834) * cpu before disabling interrupts. Need to reload cpu area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2835) * pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2836) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2837) c = this_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2838) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2840) p = ___slab_alloc(s, gfpflags, node, addr, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2841) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2842) return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2843) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2844)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2845) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2846) * If the object has been wiped upon free, make sure it's fully initialized by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2847) * zeroing out freelist pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2848) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2849) static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2850) void *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2852) if (unlikely(slab_want_init_on_free(s)) && obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2853) memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2854) 0, sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2855) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2857) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2858) * Inlined fastpath so that allocation functions (kmalloc, kmem_cache_alloc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2859) * have the fastpath folded into their functions. So no function call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2860) * overhead for requests that can be satisfied on the fastpath.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2861) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2862) * The fastpath works by first checking if the lockless freelist can be used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2863) * If not then __slab_alloc is called for slow processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2864) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2865) * Otherwise we can simply pick the next object from the lockless free list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2866) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2867) static __always_inline void *slab_alloc_node(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2868) gfp_t gfpflags, int node, unsigned long addr, size_t orig_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2870) void *object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2871) struct kmem_cache_cpu *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2872) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2873) unsigned long tid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2874) struct obj_cgroup *objcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2875) bool init = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2877) s = slab_pre_alloc_hook(s, &objcg, 1, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2878) if (!s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2879) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2881) object = kfence_alloc(s, orig_size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2882) if (unlikely(object))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2883) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2885) redo:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2886) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2887) * Must read kmem_cache cpu data via this cpu ptr. Preemption is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2888) * enabled. We may switch back and forth between cpus while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2889) * reading from one cpu area. That does not matter as long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2890) * as we end up on the original cpu again when doing the cmpxchg.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2891) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2892) * We should guarantee that tid and kmem_cache are retrieved on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2893) * the same cpu. It could be different if CONFIG_PREEMPTION so we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2894) * to check if it is matched or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2896) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2897) tid = this_cpu_read(s->cpu_slab->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2898) c = raw_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2899) } while (IS_ENABLED(CONFIG_PREEMPTION) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2900) unlikely(tid != READ_ONCE(c->tid)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2902) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2903) * Irqless object alloc/free algorithm used here depends on sequence
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2904) * of fetching cpu_slab's data. tid should be fetched before anything
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2905) * on c to guarantee that object and page associated with previous tid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2906) * won't be used with current tid. If we fetch tid first, object and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2907) * page could be one associated with next tid and our alloc/free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2908) * request will be failed. In this case, we will retry. So, no problem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2910) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2912) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2913) * The transaction ids are globally unique per cpu and per operation on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2914) * a per cpu queue. Thus they can be guarantee that the cmpxchg_double
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2915) * occurs on the right processor and that there was no operation on the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2916) * linked list in between.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2917) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2919) object = c->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2920) page = c->page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2921) if (unlikely(!object || !page || !node_match(page, node))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2922) object = __slab_alloc(s, gfpflags, node, addr, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2923) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2924) void *next_object = get_freepointer_safe(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2926) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2927) * The cmpxchg will only match if there was no additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2928) * operation and if we are on the right processor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2929) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2930) * The cmpxchg does the following atomically (without lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2931) * semantics!)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2932) * 1. Relocate first pointer to the current per cpu area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2933) * 2. Verify that tid and freelist have not been changed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2934) * 3. If they were not changed replace tid and freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2935) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2936) * Since this is without lock semantics the protection is only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2937) * against code executing on this cpu *not* from access by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2938) * other cpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2939) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2940) if (unlikely(!this_cpu_cmpxchg_double(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2941) s->cpu_slab->freelist, s->cpu_slab->tid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2942) object, tid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2943) next_object, next_tid(tid)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2945) note_cmpxchg_failure("slab_alloc", s, tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2946) goto redo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2947) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2948) prefetch_freepointer(s, next_object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2949) stat(s, ALLOC_FASTPATH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2952) maybe_wipe_obj_freeptr(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2953) init = slab_want_init_on_alloc(gfpflags, s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2955) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2956) slab_post_alloc_hook(s, objcg, gfpflags, 1, &object, init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2958) return object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2959) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2961) static __always_inline void *slab_alloc(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2962) gfp_t gfpflags, unsigned long addr, size_t orig_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2963) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2964) return slab_alloc_node(s, gfpflags, NUMA_NO_NODE, addr, orig_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2965) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2967) void *kmem_cache_alloc(struct kmem_cache *s, gfp_t gfpflags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2968) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2969) void *ret = slab_alloc(s, gfpflags, _RET_IP_, s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2971) trace_kmem_cache_alloc(_RET_IP_, ret, s->object_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2972) s->size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2974) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2976) EXPORT_SYMBOL(kmem_cache_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2978) #ifdef CONFIG_TRACING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2979) void *kmem_cache_alloc_trace(struct kmem_cache *s, gfp_t gfpflags, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2981) void *ret = slab_alloc(s, gfpflags, _RET_IP_, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2982) trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2983) ret = kasan_kmalloc(s, ret, size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2984) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2985) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2986) EXPORT_SYMBOL(kmem_cache_alloc_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2987) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2989) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2990) void *kmem_cache_alloc_node(struct kmem_cache *s, gfp_t gfpflags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2991) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2992) void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2993)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2994) trace_kmem_cache_alloc_node(_RET_IP_, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2995) s->object_size, s->size, gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2997) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2999) EXPORT_SYMBOL(kmem_cache_alloc_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3001) #ifdef CONFIG_TRACING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3002) void *kmem_cache_alloc_node_trace(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3003) gfp_t gfpflags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3004) int node, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3005) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3006) void *ret = slab_alloc_node(s, gfpflags, node, _RET_IP_, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3008) trace_kmalloc_node(_RET_IP_, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3009) size, s->size, gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3011) ret = kasan_kmalloc(s, ret, size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3012) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3014) EXPORT_SYMBOL(kmem_cache_alloc_node_trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3015) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3016) #endif /* CONFIG_NUMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3017)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3018) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3019) * Slow path handling. This may still be called frequently since objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3020) * have a longer lifetime than the cpu slabs in most processing loads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3021) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3022) * So we still attempt to reduce cache line usage. Just take the slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3023) * lock and free the item. If there is no additional partial page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3024) * handling required then we can return immediately.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3025) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3026) static void __slab_free(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3027) void *head, void *tail, int cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3028) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3030) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3031) void *prior;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3032) int was_frozen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3033) struct page new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3034) unsigned long counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3035) struct kmem_cache_node *n = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3036) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3038) stat(s, FREE_SLOWPATH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3040) if (kfence_free(head))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3041) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3043) if (kmem_cache_debug(s) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3044) !free_debug_processing(s, page, head, tail, cnt, addr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3045) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3046)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3047) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3048) if (unlikely(n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3049) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3050) n = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3052) prior = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3053) counters = page->counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3054) set_freepointer(s, tail, prior);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3055) new.counters = counters;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3056) was_frozen = new.frozen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3057) new.inuse -= cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3058) if ((!new.inuse || !prior) && !was_frozen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3060) if (kmem_cache_has_cpu_partial(s) && !prior) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3061)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3062) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3063) * Slab was on no list before and will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3064) * partially empty
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3065) * We can defer the list move and instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3066) * freeze it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3067) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3068) new.frozen = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3070) } else { /* Needs to be taken off a list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3071)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3072) n = get_node(s, page_to_nid(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3073) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3074) * Speculatively acquire the list_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3075) * If the cmpxchg does not succeed then we may
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3076) * drop the list_lock without any processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3077) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3078) * Otherwise the list_lock will synchronize with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3079) * other processors updating the list of slabs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3080) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3081) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3086) } while (!cmpxchg_double_slab(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3087) prior, counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3088) head, new.counters,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3089) "__slab_free"));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3091) if (likely(!n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3092)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3093) if (likely(was_frozen)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3094) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3095) * The list lock was not taken therefore no list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3096) * activity can be necessary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3097) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3098) stat(s, FREE_FROZEN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3099) } else if (new.frozen) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3100) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3101) * If we just froze the page then put it onto the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3102) * per cpu partial list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3104) put_cpu_partial(s, page, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3105) stat(s, CPU_PARTIAL_FREE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3106) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3108) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3111) if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3112) goto slab_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3115) * Objects left in the slab. If it was not on the partial list before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3116) * then add it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3118) if (!kmem_cache_has_cpu_partial(s) && unlikely(!prior)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3119) remove_full(s, n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3120) add_partial(n, page, DEACTIVATE_TO_TAIL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3121) stat(s, FREE_ADD_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3123) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3124) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3126) slab_empty:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3127) if (prior) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3129) * Slab on the partial list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3130) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3131) remove_partial(n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3132) stat(s, FREE_REMOVE_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3133) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3134) /* Slab must be on the full list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3135) remove_full(s, n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3138) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3139) stat(s, FREE_SLAB);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3140) discard_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3143) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3144) * Fastpath with forced inlining to produce a kfree and kmem_cache_free that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3145) * can perform fastpath freeing without additional function calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3146) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3147) * The fastpath is only possible if we are freeing to the current cpu slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3148) * of this processor. This typically the case if we have just allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3149) * the item before.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3150) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3151) * If fastpath is not possible then fall back to __slab_free where we deal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3152) * with all sorts of special processing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3153) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3154) * Bulk free of a freelist with several objects (all pointing to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3155) * same page) possible by specifying head and tail ptr, plus objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3156) * count (cnt). Bulk free indicated by tail pointer being set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3158) static __always_inline void do_slab_free(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3159) struct page *page, void *head, void *tail,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3160) int cnt, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3162) void *tail_obj = tail ? : head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3163) struct kmem_cache_cpu *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3164) unsigned long tid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3166) /* memcg_slab_free_hook() is already called for bulk free. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3167) if (!tail)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3168) memcg_slab_free_hook(s, &head, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3169) redo:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3170) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3171) * Determine the currently cpus per cpu slab.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3172) * The cpu may change afterward. However that does not matter since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3173) * data is retrieved via this pointer. If we are on the same cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3174) * during the cmpxchg then the free will succeed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3175) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3176) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3177) tid = this_cpu_read(s->cpu_slab->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3178) c = raw_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3179) } while (IS_ENABLED(CONFIG_PREEMPTION) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3180) unlikely(tid != READ_ONCE(c->tid)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3182) /* Same with comment on barrier() in slab_alloc_node() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3183) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3185) if (likely(page == c->page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3186) void **freelist = READ_ONCE(c->freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3188) set_freepointer(s, tail_obj, freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3190) if (unlikely(!this_cpu_cmpxchg_double(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3191) s->cpu_slab->freelist, s->cpu_slab->tid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3192) freelist, tid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3193) head, next_tid(tid)))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3195) note_cmpxchg_failure("slab_free", s, tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3196) goto redo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3198) stat(s, FREE_FASTPATH);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3199) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3200) __slab_free(s, page, head, tail_obj, cnt, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3204) static __always_inline void slab_free(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3205) void *head, void *tail, int cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3206) unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3208) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3209) * With KASAN enabled slab_free_freelist_hook modifies the freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3210) * to remove objects, whose reuse must be delayed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3211) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3212) if (slab_free_freelist_hook(s, &head, &tail, &cnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3213) do_slab_free(s, page, head, tail, cnt, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3216) #ifdef CONFIG_KASAN_GENERIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3217) void ___cache_free(struct kmem_cache *cache, void *x, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3219) do_slab_free(cache, virt_to_head_page(x), x, NULL, 1, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3221) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3223) void kmem_cache_free(struct kmem_cache *s, void *x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3225) s = cache_from_obj(s, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3226) if (!s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3227) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3228) slab_free(s, virt_to_head_page(x), x, NULL, 1, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3229) trace_kmem_cache_free(_RET_IP_, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3231) EXPORT_SYMBOL(kmem_cache_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3233) struct detached_freelist {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3234) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3235) void *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3236) void *freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3237) int cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3238) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3239) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3241) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3242) * This function progressively scans the array with free objects (with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3243) * a limited look ahead) and extract objects belonging to the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3244) * page. It builds a detached freelist directly within the given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3245) * page/objects. This can happen without any need for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3246) * synchronization, because the objects are owned by running process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3247) * The freelist is build up as a single linked list in the objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3248) * The idea is, that this detached freelist can then be bulk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3249) * transferred to the real freelist(s), but only requiring a single
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3250) * synchronization primitive. Look ahead in the array is limited due
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3251) * to performance reasons.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3253) static inline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3254) int build_detached_freelist(struct kmem_cache *s, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3255) void **p, struct detached_freelist *df)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3257) size_t first_skipped_index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3258) int lookahead = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3259) void *object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3260) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3262) /* Always re-init detached_freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3263) df->page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3265) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3266) object = p[--size];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3267) /* Do we need !ZERO_OR_NULL_PTR(object) here? (for kfree) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3268) } while (!object && size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3270) if (!object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3271) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3273) page = virt_to_head_page(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3274) if (!s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3275) /* Handle kalloc'ed objects */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3276) if (unlikely(!PageSlab(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3277) BUG_ON(!PageCompound(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3278) kfree_hook(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3279) __free_pages(page, compound_order(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3280) p[size] = NULL; /* mark object processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3281) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3283) /* Derive kmem_cache from object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3284) df->s = page->slab_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3285) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3286) df->s = cache_from_obj(s, object); /* Support for memcg */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3289) if (is_kfence_address(object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3290) slab_free_hook(df->s, object, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3291) __kfence_free(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3292) p[size] = NULL; /* mark object processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3293) return size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3296) /* Start new detached freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3297) df->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3298) set_freepointer(df->s, object, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3299) df->tail = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3300) df->freelist = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3301) p[size] = NULL; /* mark object processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3302) df->cnt = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3304) while (size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3305) object = p[--size];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3306) if (!object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3307) continue; /* Skip processed objects */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3309) /* df->page is always set at this point */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3310) if (df->page == virt_to_head_page(object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3311) /* Opportunity build freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3312) set_freepointer(df->s, object, df->freelist);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3313) df->freelist = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3314) df->cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3315) p[size] = NULL; /* mark object processed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3317) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3320) /* Limit look ahead search */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3321) if (!--lookahead)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3322) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3324) if (!first_skipped_index)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3325) first_skipped_index = size + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3328) return first_skipped_index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3331) /* Note that interrupts must be enabled when calling this function. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3332) void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3334) if (WARN_ON(!size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3335) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3337) memcg_slab_free_hook(s, p, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3338) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3339) struct detached_freelist df;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3341) size = build_detached_freelist(s, size, p, &df);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3342) if (!df.page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3343) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3345) slab_free(df.s, df.page, df.freelist, df.tail, df.cnt,_RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3346) } while (likely(size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3348) EXPORT_SYMBOL(kmem_cache_free_bulk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3350) /* Note that interrupts must be enabled when calling this function. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3351) int kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3352) void **p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3353) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3354) struct kmem_cache_cpu *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3355) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3356) struct obj_cgroup *objcg = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3358) /* memcg and kmem_cache debug support */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3359) s = slab_pre_alloc_hook(s, &objcg, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3360) if (unlikely(!s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3361) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3363) * Drain objects in the per cpu slab, while disabling local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3364) * IRQs, which protects against PREEMPT and interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3365) * handlers invoking normal fastpath.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3366) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3367) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3368) c = this_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3370) for (i = 0; i < size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3371) void *object = kfence_alloc(s, s->object_size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3373) if (unlikely(object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3374) p[i] = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3375) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3377)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3378) object = c->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3379) if (unlikely(!object)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3380) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3381) * We may have removed an object from c->freelist using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3382) * the fastpath in the previous iteration; in that case,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3383) * c->tid has not been bumped yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3384) * Since ___slab_alloc() may reenable interrupts while
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3385) * allocating memory, we should bump c->tid now.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3386) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3387) c->tid = next_tid(c->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3389) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3390) * Invoking slow path likely have side-effect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3391) * of re-populating per CPU c->freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3392) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3393) p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3394) _RET_IP_, c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3395) if (unlikely(!p[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3396) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3398) c = this_cpu_ptr(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3399) maybe_wipe_obj_freeptr(s, p[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3401) continue; /* goto for-loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3402) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3403) c->freelist = get_freepointer(s, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3404) p[i] = object;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3405) maybe_wipe_obj_freeptr(s, p[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3407) c->tid = next_tid(c->tid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3408) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3411) * memcg and kmem_cache debug support and memory initialization.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3412) * Done outside of the IRQ disabled fastpath loop.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3413) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3414) slab_post_alloc_hook(s, objcg, flags, size, p,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3415) slab_want_init_on_alloc(flags, s));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3416) return i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3417) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3418) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3419) slab_post_alloc_hook(s, objcg, flags, i, p, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3420) __kmem_cache_free_bulk(s, i, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3421) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3423) EXPORT_SYMBOL(kmem_cache_alloc_bulk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3424)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3426) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3427) * Object placement in a slab is made very easy because we always start at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3428) * offset 0. If we tune the size of the object to the alignment then we can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3429) * get the required alignment by putting one properly sized object after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3430) * another.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3431) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3432) * Notice that the allocation order determines the sizes of the per cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3433) * caches. Each processor has always one slab available for allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3434) * Increasing the allocation order reduces the number of times that slabs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3435) * must be moved on and off the partial lists and is therefore a factor in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3436) * locking overhead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3439) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3440) * Mininum / Maximum order of slab pages. This influences locking overhead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3441) * and slab fragmentation. A higher order reduces the number of partial slabs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3442) * and increases the number of allocations possible without having to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3443) * take the list_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3444) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3445) static unsigned int slub_min_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3446) static unsigned int slub_max_order = PAGE_ALLOC_COSTLY_ORDER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3447) static unsigned int slub_min_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3449) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3450) * Calculate the order of allocation given an slab object size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3451) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3452) * The order of allocation has significant impact on performance and other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3453) * system components. Generally order 0 allocations should be preferred since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3454) * order 0 does not cause fragmentation in the page allocator. Larger objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3455) * be problematic to put into order 0 slabs because there may be too much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3456) * unused space left. We go to a higher order if more than 1/16th of the slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3457) * would be wasted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3458) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3459) * In order to reach satisfactory performance we must ensure that a minimum
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3460) * number of objects is in one slab. Otherwise we may generate too much
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3461) * activity on the partial lists which requires taking the list_lock. This is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3462) * less a concern for large slabs though which are rarely used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3463) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3464) * slub_max_order specifies the order where we begin to stop considering the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3465) * number of objects in a slab as critical. If we reach slub_max_order then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3466) * we try to keep the page order as low as possible. So we accept more waste
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3467) * of space in favor of a small page order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3468) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3469) * Higher order allocations also allow the placement of more objects in a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3470) * slab and thereby reduce object handling overhead. If the user has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3471) * requested a higher mininum order then we start with that one instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3472) * the smallest order which will fit the object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3473) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3474) static inline unsigned int slab_order(unsigned int size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3475) unsigned int min_objects, unsigned int max_order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3476) unsigned int fract_leftover)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3478) unsigned int min_order = slub_min_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3479) unsigned int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3481) if (order_objects(min_order, size) > MAX_OBJS_PER_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3482) return get_order(size * MAX_OBJS_PER_PAGE) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3484) for (order = max(min_order, (unsigned int)get_order(min_objects * size));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3485) order <= max_order; order++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3487) unsigned int slab_size = (unsigned int)PAGE_SIZE << order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3488) unsigned int rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3490) rem = slab_size % size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3492) if (rem <= slab_size / fract_leftover)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3493) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3496) return order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3499) static inline int calculate_order(unsigned int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3500) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3501) unsigned int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3502) unsigned int min_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3503) unsigned int max_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3506) * Attempt to find best configuration for a slab. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3507) * works by first attempting to generate a layout with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3508) * the best configuration and backing off gradually.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3509) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3510) * First we increase the acceptable waste in a slab. Then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3511) * we reduce the minimum objects required in a slab.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3512) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3513) min_objects = slub_min_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3514) if (!min_objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3515) min_objects = 4 * (fls(nr_cpu_ids) + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3516) max_objects = order_objects(slub_max_order, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3517) min_objects = min(min_objects, max_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3518)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3519) while (min_objects > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3520) unsigned int fraction;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3522) fraction = 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3523) while (fraction >= 4) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3524) order = slab_order(size, min_objects,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3525) slub_max_order, fraction);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3526) if (order <= slub_max_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3527) return order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3528) fraction /= 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3530) min_objects--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3533) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3534) * We were unable to place multiple objects in a slab. Now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3535) * lets see if we can place a single object there.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3536) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3537) order = slab_order(size, 1, slub_max_order, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3538) if (order <= slub_max_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3539) return order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3541) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3542) * Doh this slab cannot be placed using slub_max_order.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3543) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3544) order = slab_order(size, 1, MAX_ORDER, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3545) if (order < MAX_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3546) return order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3547) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3550) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3551) init_kmem_cache_node(struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3553) n->nr_partial = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3554) spin_lock_init(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3555) INIT_LIST_HEAD(&n->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3556) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3557) atomic_long_set(&n->nr_slabs, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3558) atomic_long_set(&n->total_objects, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3559) INIT_LIST_HEAD(&n->full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3560) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3563) static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3565) BUILD_BUG_ON(PERCPU_DYNAMIC_EARLY_SIZE <
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3566) KMALLOC_SHIFT_HIGH * sizeof(struct kmem_cache_cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3568) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3569) * Must align to double word boundary for the double cmpxchg
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3570) * instructions to work; see __pcpu_double_call_return_bool().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3572) s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3573) 2 * sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3575) if (!s->cpu_slab)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3576) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3578) init_kmem_cache_cpus(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3580) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3583) static struct kmem_cache *kmem_cache_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3585) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3586) * No kmalloc_node yet so do it by hand. We know that this is the first
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3587) * slab on the node for this slabcache. There are no concurrent accesses
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3588) * possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3589) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3590) * Note that this function only works on the kmem_cache_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3591) * when allocating for the kmem_cache_node. This is used for bootstrapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3592) * memory on a fresh node that has no slab structures yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3594) static void early_kmem_cache_node_alloc(int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3596) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3597) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3599) BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3601) page = new_slab(kmem_cache_node, GFP_NOWAIT, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3603) BUG_ON(!page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3604) if (page_to_nid(page) != node) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3605) pr_err("SLUB: Unable to allocate memory from node %d\n", node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3606) pr_err("SLUB: Allocating a useless per node structure in order to be able to continue\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3609) n = page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3610) BUG_ON(!n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3611) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3612) init_object(kmem_cache_node, n, SLUB_RED_ACTIVE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3613) init_tracking(kmem_cache_node, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3614) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3615) n = kasan_slab_alloc(kmem_cache_node, n, GFP_KERNEL, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3616) page->freelist = get_freepointer(kmem_cache_node, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3617) page->inuse = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3618) page->frozen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3619) kmem_cache_node->node[node] = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3620) init_kmem_cache_node(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3621) inc_slabs_node(kmem_cache_node, node, page->objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3623) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3624) * No locks need to be taken here as it has just been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3625) * initialized and there is no concurrent access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3626) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3627) __add_partial(n, page, DEACTIVATE_TO_HEAD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3628) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3629)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3630) static void free_kmem_cache_nodes(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3631) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3632) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3633) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3635) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3636) s->node[node] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3637) kmem_cache_free(kmem_cache_node, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3641) void __kmem_cache_release(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3642) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3643) cache_random_seq_destroy(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3644) free_percpu(s->cpu_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3645) free_kmem_cache_nodes(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3646) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3648) static int init_kmem_cache_nodes(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3649) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3650) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3652) for_each_node_state(node, N_NORMAL_MEMORY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3653) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3654)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3655) if (slab_state == DOWN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3656) early_kmem_cache_node_alloc(node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3657) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3659) n = kmem_cache_alloc_node(kmem_cache_node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3660) GFP_KERNEL, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3662) if (!n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3663) free_kmem_cache_nodes(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3664) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3667) init_kmem_cache_node(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3668) s->node[node] = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3669) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3670) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3673) static void set_min_partial(struct kmem_cache *s, unsigned long min)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3674) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3675) if (min < MIN_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3676) min = MIN_PARTIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3677) else if (min > MAX_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3678) min = MAX_PARTIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3679) s->min_partial = min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3680) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3682) static void set_cpu_partial(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3683) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3684) #ifdef CONFIG_SLUB_CPU_PARTIAL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3685) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3686) * cpu_partial determined the maximum number of objects kept in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3687) * per cpu partial lists of a processor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3688) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3689) * Per cpu partial lists mainly contain slabs that just have one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3690) * object freed. If they are used for allocation then they can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3691) * filled up again with minimal effort. The slab will never hit the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3692) * per node partial lists and therefore no locking will be required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3693) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3694) * This setting also determines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3695) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3696) * A) The number of objects from per cpu partial slabs dumped to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3697) * per node list when we reach the limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3698) * B) The number of objects in cpu partial slabs to extract from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3699) * per node list when we run out of per cpu objects. We only fetch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3700) * 50% to keep some capacity around for frees.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3701) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3702) if (!kmem_cache_has_cpu_partial(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3703) slub_set_cpu_partial(s, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3704) else if (s->size >= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3705) slub_set_cpu_partial(s, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3706) else if (s->size >= 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3707) slub_set_cpu_partial(s, 6);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3708) else if (s->size >= 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3709) slub_set_cpu_partial(s, 13);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3710) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3711) slub_set_cpu_partial(s, 30);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3712) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3713) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3715) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3716) * calculate_sizes() determines the order and the distribution of data within
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3717) * a slab object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3718) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3719) static int calculate_sizes(struct kmem_cache *s, int forced_order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3720) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3721) slab_flags_t flags = s->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3722) unsigned int size = s->object_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3723) unsigned int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3725) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3726) * Round up object size to the next word boundary. We can only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3727) * place the free pointer at word boundaries and this determines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3728) * the possible location of the free pointer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3730) size = ALIGN(size, sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3732) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3733) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3734) * Determine if we can poison the object itself. If the user of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3735) * the slab may touch the object after free or before allocation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3736) * then we should never poison the object itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3737) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3738) if ((flags & SLAB_POISON) && !(flags & SLAB_TYPESAFE_BY_RCU) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3739) !s->ctor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3740) s->flags |= __OBJECT_POISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3741) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3742) s->flags &= ~__OBJECT_POISON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3745) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3746) * If we are Redzoning then check if there is some space between the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3747) * end of the object and the free pointer. If not then add an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3748) * additional word to have some bytes to store Redzone information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3749) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3750) if ((flags & SLAB_RED_ZONE) && size == s->object_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3751) size += sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3752) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3754) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3755) * With that we have determined the number of bytes in actual use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3756) * by the object and redzoning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3757) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3758) s->inuse = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3760) if ((flags & (SLAB_TYPESAFE_BY_RCU | SLAB_POISON)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3761) ((flags & SLAB_RED_ZONE) && s->object_size < sizeof(void *)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3762) s->ctor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3763) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3764) * Relocate free pointer after the object if it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3765) * permitted to overwrite the first word of the object on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3766) * kmem_cache_free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3767) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3768) * This is the case if we do RCU, have a constructor or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3769) * destructor, are poisoning the objects, or are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3770) * redzoning an object smaller than sizeof(void *).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3771) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3772) * The assumption that s->offset >= s->inuse means free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3773) * pointer is outside of the object is used in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3774) * freeptr_outside_object() function. If that is no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3775) * longer true, the function needs to be modified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3776) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3777) s->offset = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3778) size += sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3779) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3780) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3781) * Store freelist pointer near middle of object to keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3782) * it away from the edges of the object to avoid small
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3783) * sized over/underflows from neighboring allocations.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3784) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3785) s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3788) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3789) if (flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3790) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3791) * Need to store information about allocs and frees after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3792) * the object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3793) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3794) size += 2 * sizeof(struct track);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3795) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3797) kasan_cache_create(s, &size, &s->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3798) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3799) if (flags & SLAB_RED_ZONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3800) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3801) * Add some empty padding so that we can catch
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3802) * overwrites from earlier objects rather than let
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3803) * tracking information or the free pointer be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3804) * corrupted if a user writes before the start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3805) * of the object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3806) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3807) size += sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3809) s->red_left_pad = sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3810) s->red_left_pad = ALIGN(s->red_left_pad, s->align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3811) size += s->red_left_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3813) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3815) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3816) * SLUB stores one object immediately after another beginning from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3817) * offset 0. In order to align the objects we have to simply size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3818) * each object to conform to the alignment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3819) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3820) size = ALIGN(size, s->align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3821) s->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3822) s->reciprocal_size = reciprocal_value(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3823) if (forced_order >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3824) order = forced_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3825) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3826) order = calculate_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3828) if ((int)order < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3829) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3831) s->allocflags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3832) if (order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3833) s->allocflags |= __GFP_COMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3835) if (s->flags & SLAB_CACHE_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3836) s->allocflags |= GFP_DMA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3838) if (s->flags & SLAB_CACHE_DMA32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3839) s->allocflags |= GFP_DMA32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3841) if (s->flags & SLAB_RECLAIM_ACCOUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3842) s->allocflags |= __GFP_RECLAIMABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3844) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3845) * Determine the number of objects per slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3846) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3847) s->oo = oo_make(order, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3848) s->min = oo_make(get_order(size), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3849) if (oo_objects(s->oo) > oo_objects(s->max))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3850) s->max = s->oo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3852) return !!oo_objects(s->oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3855) static int kmem_cache_open(struct kmem_cache *s, slab_flags_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3856) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3857) s->flags = kmem_cache_flags(s->size, flags, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3858) #ifdef CONFIG_SLAB_FREELIST_HARDENED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3859) s->random = get_random_long();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3860) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3861)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3862) if (!calculate_sizes(s, -1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3863) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3864) if (disable_higher_order_debug) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3865) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3866) * Disable debugging flags that store metadata if the min slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3867) * order increased.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3868) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3869) if (get_order(s->size) > get_order(s->object_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3870) s->flags &= ~DEBUG_METADATA_FLAGS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3871) s->offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3872) if (!calculate_sizes(s, -1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3873) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3877) #if defined(CONFIG_HAVE_CMPXCHG_DOUBLE) && \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3878) defined(CONFIG_HAVE_ALIGNED_STRUCT_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3879) if (system_has_cmpxchg_double() && (s->flags & SLAB_NO_CMPXCHG) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3880) /* Enable fast mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3881) s->flags |= __CMPXCHG_DOUBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3882) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3884) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3885) * The larger the object size is, the more pages we want on the partial
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3886) * list to avoid pounding the page allocator excessively.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3887) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3888) set_min_partial(s, ilog2(s->size) / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3890) set_cpu_partial(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3892) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3893) s->remote_node_defrag_ratio = 1000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3894) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3895)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3896) /* Initialize the pre-computed randomized freelist if slab is up */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3897) if (slab_state >= UP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3898) if (init_cache_random_seq(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3899) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3902) if (!init_kmem_cache_nodes(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3903) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3904)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3905) if (alloc_kmem_cache_cpus(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3906) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3907)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3908) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3909) __kmem_cache_release(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3910) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3911) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3912)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3913) static void list_slab_objects(struct kmem_cache *s, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3914) const char *text)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3915) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3916) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3917) void *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3918) unsigned long *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3919) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3920)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3921) slab_err(s, page, text, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3922) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3924) map = get_map(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3925) for_each_object(p, s, addr, page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3926)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3927) if (!test_bit(__obj_to_index(s, addr, p), map)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3928) pr_err("INFO: Object 0x%p @offset=%tu\n", p, p - addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3929) print_tracking(s, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3930) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3931) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3932) put_map(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3933) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3934) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3937) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3938) * Attempt to free all partial slabs on a node.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3939) * This is called from __kmem_cache_shutdown(). We must take list_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3940) * because sysfs file might still access partial list after the shutdowning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3941) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3942) static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3943) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3944) LIST_HEAD(discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3945) struct page *page, *h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3946)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3947) BUG_ON(irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3948) spin_lock_irq(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3949) list_for_each_entry_safe(page, h, &n->partial, slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3950) if (!page->inuse) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3951) remove_partial(n, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3952) list_add(&page->slab_list, &discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3953) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3954) list_slab_objects(s, page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3955) "Objects remaining in %s on __kmem_cache_shutdown()");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3958) spin_unlock_irq(&n->list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3959)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3960) list_for_each_entry_safe(page, h, &discard, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3961) discard_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3964) bool __kmem_cache_empty(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3966) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3967) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3969) for_each_kmem_cache_node(s, node, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3970) if (n->nr_partial || slabs_node(s, node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3971) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3972) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3973) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3975) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3976) * Release all resources used by a slab cache.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3977) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3978) int __kmem_cache_shutdown(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3980) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3981) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3983) flush_all(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3984) /* Attempt to free all objects */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3985) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3986) free_partial(s, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3987) if (n->nr_partial || slabs_node(s, node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3988) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3990) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3991) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3992)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3993) /********************************************************************
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3994) * Kmalloc subsystem
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3995) *******************************************************************/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3996)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3997) static int __init setup_slub_min_order(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3998) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3999) get_option(&str, (int *)&slub_min_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4001) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4002) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4003)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4004) __setup("slub_min_order=", setup_slub_min_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4005)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4006) static int __init setup_slub_max_order(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4008) get_option(&str, (int *)&slub_max_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4009) slub_max_order = min(slub_max_order, (unsigned int)MAX_ORDER - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4011) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4012) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4014) __setup("slub_max_order=", setup_slub_max_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4016) static int __init setup_slub_min_objects(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4018) get_option(&str, (int *)&slub_min_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4020) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4023) __setup("slub_min_objects=", setup_slub_min_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4025) void *__kmalloc(size_t size, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4027) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4028) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4030) if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4031) return kmalloc_large(size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4032)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4033) s = kmalloc_slab(size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4035) if (unlikely(ZERO_OR_NULL_PTR(s)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4036) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4038) ret = slab_alloc(s, flags, _RET_IP_, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4040) trace_kmalloc(_RET_IP_, ret, size, s->size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4042) ret = kasan_kmalloc(s, ret, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4044) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4046) EXPORT_SYMBOL(__kmalloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4048) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4049) static void *kmalloc_large_node(size_t size, gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4050) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4051) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4052) void *ptr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4053) unsigned int order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4055) flags |= __GFP_COMP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4056) page = alloc_pages_node(node, flags, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4057) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4058) ptr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4059) mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4060) PAGE_SIZE << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4063) return kmalloc_large_node_hook(ptr, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4064) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4066) void *__kmalloc_node(size_t size, gfp_t flags, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4067) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4068) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4069) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4071) if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4072) ret = kmalloc_large_node(size, flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4074) trace_kmalloc_node(_RET_IP_, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4075) size, PAGE_SIZE << get_order(size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4076) flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4078) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4079) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4081) s = kmalloc_slab(size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4083) if (unlikely(ZERO_OR_NULL_PTR(s)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4084) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4086) ret = slab_alloc_node(s, flags, node, _RET_IP_, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4087)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4088) trace_kmalloc_node(_RET_IP_, ret, size, s->size, flags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4090) ret = kasan_kmalloc(s, ret, size, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4092) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4094) EXPORT_SYMBOL(__kmalloc_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4095) #endif /* CONFIG_NUMA */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4097) #ifdef CONFIG_HARDENED_USERCOPY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4098) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4099) * Rejects incorrectly sized objects and objects that are to be copied
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4100) * to/from userspace but do not fall entirely within the containing slab
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4101) * cache's usercopy region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4102) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4103) * Returns NULL if check passes, otherwise const char * to name of cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4104) * to indicate an error.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4106) void __check_heap_object(const void *ptr, unsigned long n, struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4107) bool to_user)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4109) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4110) unsigned int offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4111) size_t object_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4112) bool is_kfence = is_kfence_address(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4114) ptr = kasan_reset_tag(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4116) /* Find object and usable object size. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4117) s = page->slab_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4119) /* Reject impossible pointers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4120) if (ptr < page_address(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4121) usercopy_abort("SLUB object not in SLUB page?!", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4122) to_user, 0, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4124) /* Find offset within object. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4125) if (is_kfence)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4126) offset = ptr - kfence_object_start(ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4127) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4128) offset = (ptr - page_address(page)) % s->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4130) /* Adjust for redzone and reject if within the redzone. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4131) if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4132) if (offset < s->red_left_pad)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4133) usercopy_abort("SLUB object in left red zone",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4134) s->name, to_user, offset, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4135) offset -= s->red_left_pad;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4138) /* Allow address range falling entirely within usercopy region. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4139) if (offset >= s->useroffset &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4140) offset - s->useroffset <= s->usersize &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4141) n <= s->useroffset - offset + s->usersize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4142) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4144) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4145) * If the copy is still within the allocated object, produce
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4146) * a warning instead of rejecting the copy. This is intended
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4147) * to be a temporary method to find any missing usercopy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4148) * whitelists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4149) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4150) object_size = slab_ksize(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4151) if (usercopy_fallback &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4152) offset <= object_size && n <= object_size - offset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4153) usercopy_warn("SLUB object", s->name, to_user, offset, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4154) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4155) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4157) usercopy_abort("SLUB object", s->name, to_user, offset, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4159) #endif /* CONFIG_HARDENED_USERCOPY */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4161) size_t __ksize(const void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4163) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4165) if (unlikely(object == ZERO_SIZE_PTR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4166) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4168) page = virt_to_head_page(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4170) if (unlikely(!PageSlab(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4171) WARN_ON(!PageCompound(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4172) return page_size(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4175) return slab_ksize(page->slab_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4176) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4177) EXPORT_SYMBOL(__ksize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4179) void kfree(const void *x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4181) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4182) void *object = (void *)x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4184) trace_kfree(_RET_IP_, x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4186) if (unlikely(ZERO_OR_NULL_PTR(x)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4187) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4189) page = virt_to_head_page(x);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4190) if (unlikely(!PageSlab(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4191) unsigned int order = compound_order(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4193) BUG_ON(!PageCompound(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4194) kfree_hook(object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4195) mod_lruvec_page_state(page, NR_SLAB_UNRECLAIMABLE_B,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4196) -(PAGE_SIZE << order));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4197) __free_pages(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4198) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4200) slab_free(page->slab_cache, page, object, NULL, 1, _RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4202) EXPORT_SYMBOL(kfree);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4204) #define SHRINK_PROMOTE_MAX 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4206) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4207) * kmem_cache_shrink discards empty slabs and promotes the slabs filled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4208) * up most to the head of the partial lists. New allocations will then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4209) * fill those up and thus they can be removed from the partial lists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4210) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4211) * The slabs with the least items are placed last. This results in them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4212) * being allocated from last increasing the chance that the last objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4213) * are freed in them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4214) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4215) int __kmem_cache_shrink(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4217) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4218) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4219) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4220) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4221) struct page *t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4222) struct list_head discard;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4223) struct list_head promote[SHRINK_PROMOTE_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4224) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4225) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4227) flush_all(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4228) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4229) INIT_LIST_HEAD(&discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4230) for (i = 0; i < SHRINK_PROMOTE_MAX; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4231) INIT_LIST_HEAD(promote + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4233) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4235) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4236) * Build lists of slabs to discard or promote.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4237) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4238) * Note that concurrent frees may occur while we hold the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4239) * list_lock. page->inuse here is the upper limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4241) list_for_each_entry_safe(page, t, &n->partial, slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4242) int free = page->objects - page->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4244) /* Do not reread page->inuse */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4245) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4247) /* We do not keep full slabs on the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4248) BUG_ON(free <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4250) if (free == page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4251) list_move(&page->slab_list, &discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4252) n->nr_partial--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4253) } else if (free <= SHRINK_PROMOTE_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4254) list_move(&page->slab_list, promote + free - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4257) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4258) * Promote the slabs filled up most to the head of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4259) * partial list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4261) for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4262) list_splice(promote + i, &n->partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4264) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4266) /* Release empty slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4267) list_for_each_entry_safe(page, t, &discard, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4268) discard_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4270) if (slabs_node(s, node))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4271) ret = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4272) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4274) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4277) static int slab_mem_going_offline_callback(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4279) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4281) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4282) list_for_each_entry(s, &slab_caches, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4283) __kmem_cache_shrink(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4284) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4286) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4287) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4289) static void slab_mem_offline_callback(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4291) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4292) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4293) struct memory_notify *marg = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4294) int offline_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4296) offline_node = marg->status_change_nid_normal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4298) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4299) * If the node still has available memory. we need kmem_cache_node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4300) * for it yet.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4302) if (offline_node < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4303) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4305) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4306) list_for_each_entry(s, &slab_caches, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4307) n = get_node(s, offline_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4308) if (n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4309) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4310) * if n->nr_slabs > 0, slabs still exist on the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4311) * that is going down. We were unable to free them,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4312) * and offline_pages() function shouldn't call this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4313) * callback. So, we must fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4314) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4315) BUG_ON(slabs_node(s, offline_node));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4317) s->node[offline_node] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4318) kmem_cache_free(kmem_cache_node, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4319) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4321) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4322) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4324) static int slab_mem_going_online_callback(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4326) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4327) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4328) struct memory_notify *marg = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4329) int nid = marg->status_change_nid_normal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4330) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4332) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4333) * If the node's memory is already available, then kmem_cache_node is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4334) * already created. Nothing to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4336) if (nid < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4337) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4339) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4340) * We are bringing a node online. No memory is available yet. We must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4341) * allocate a kmem_cache_node structure in order to bring the node
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4342) * online.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4343) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4344) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4345) list_for_each_entry(s, &slab_caches, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4346) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4347) * XXX: kmem_cache_alloc_node will fallback to other nodes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4348) * since memory is not yet available from the node that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4349) * is brought up.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4350) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4351) n = kmem_cache_alloc(kmem_cache_node, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4352) if (!n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4353) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4354) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4356) init_kmem_cache_node(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4357) s->node[nid] = n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4359) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4360) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4361) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4364) static int slab_memory_callback(struct notifier_block *self,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4365) unsigned long action, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4366) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4367) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4369) switch (action) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4370) case MEM_GOING_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4371) ret = slab_mem_going_online_callback(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4372) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4373) case MEM_GOING_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4374) ret = slab_mem_going_offline_callback(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4375) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4376) case MEM_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4377) case MEM_CANCEL_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4378) slab_mem_offline_callback(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4380) case MEM_ONLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4381) case MEM_CANCEL_OFFLINE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4382) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4384) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4385) ret = notifier_from_errno(ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4386) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4387) ret = NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4388) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4391) static struct notifier_block slab_memory_callback_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4392) .notifier_call = slab_memory_callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4393) .priority = SLAB_CALLBACK_PRI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4394) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4396) /********************************************************************
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4397) * Basic setup of slabs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4398) *******************************************************************/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4400) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4401) * Used for early kmem_cache structures that were allocated using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4402) * the page allocator. Allocate them properly then fix up the pointers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4403) * that may be pointing to the wrong kmem_cache structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4404) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4406) static struct kmem_cache * __init bootstrap(struct kmem_cache *static_cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4408) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4409) struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4410) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4412) memcpy(s, static_cache, kmem_cache->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4414) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4415) * This runs very early, and only the boot processor is supposed to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4416) * up. Even if it weren't true, IRQs are not up so we couldn't fire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4417) * IPIs around.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4419) __flush_cpu_slab(s, smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4420) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4421) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4423) list_for_each_entry(p, &n->partial, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4424) p->slab_cache = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4426) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4427) list_for_each_entry(p, &n->full, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4428) p->slab_cache = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4429) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4431) list_add(&s->list, &slab_caches);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4432) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4435) void __init kmem_cache_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4437) static __initdata struct kmem_cache boot_kmem_cache,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4438) boot_kmem_cache_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4440) if (debug_guardpage_minorder())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4441) slub_max_order = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4443) kmem_cache_node = &boot_kmem_cache_node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4444) kmem_cache = &boot_kmem_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4446) create_boot_cache(kmem_cache_node, "kmem_cache_node",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4447) sizeof(struct kmem_cache_node), SLAB_HWCACHE_ALIGN, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4449) register_hotmemory_notifier(&slab_memory_callback_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4451) /* Able to allocate the per node structures */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4452) slab_state = PARTIAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4454) create_boot_cache(kmem_cache, "kmem_cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4455) offsetof(struct kmem_cache, node) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4456) nr_node_ids * sizeof(struct kmem_cache_node *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4457) SLAB_HWCACHE_ALIGN, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4459) kmem_cache = bootstrap(&boot_kmem_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4460) kmem_cache_node = bootstrap(&boot_kmem_cache_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4462) /* Now we can use the kmem_cache to allocate kmalloc slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4463) setup_kmalloc_cache_index_table();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4464) create_kmalloc_caches(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4466) /* Setup random freelists for each cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4467) init_freelist_randomization();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4469) cpuhp_setup_state_nocalls(CPUHP_SLUB_DEAD, "slub:dead", NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4470) slub_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4472) pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4473) cache_line_size(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4474) slub_min_order, slub_max_order, slub_min_objects,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4475) nr_cpu_ids, nr_node_ids);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4478) void __init kmem_cache_init_late(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4482) struct kmem_cache *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4483) __kmem_cache_alias(const char *name, unsigned int size, unsigned int align,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4484) slab_flags_t flags, void (*ctor)(void *))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4486) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4488) s = find_mergeable(size, align, flags, name, ctor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4489) if (s) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4490) s->refcount++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4492) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4493) * Adjust the object sizes so that we clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4494) * the complete object on kzalloc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4496) s->object_size = max(s->object_size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4497) s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4499) if (sysfs_slab_alias(s, name)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4500) s->refcount--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4501) s = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4505) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4508) int __kmem_cache_create(struct kmem_cache *s, slab_flags_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4510) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4512) err = kmem_cache_open(s, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4513) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4514) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4516) /* Mutex is not taken during early boot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4517) if (slab_state <= UP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4518) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4520) err = sysfs_slab_add(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4521) if (err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4522) __kmem_cache_release(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4523) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4526) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4527) debugfs_slab_add(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4529) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4530) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4532) void *__kmalloc_track_caller(size_t size, gfp_t gfpflags, unsigned long caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4533) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4534) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4535) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4537) if (unlikely(size > KMALLOC_MAX_CACHE_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4538) return kmalloc_large(size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4540) s = kmalloc_slab(size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4542) if (unlikely(ZERO_OR_NULL_PTR(s)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4543) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4545) ret = slab_alloc(s, gfpflags, caller, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4547) /* Honor the call site pointer we received. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4548) trace_kmalloc(caller, ret, size, s->size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4550) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4552) EXPORT_SYMBOL(__kmalloc_track_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4554) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4555) void *__kmalloc_node_track_caller(size_t size, gfp_t gfpflags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4556) int node, unsigned long caller)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4557) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4558) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4559) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4561) if (unlikely(size > KMALLOC_MAX_CACHE_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4562) ret = kmalloc_large_node(size, gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4563)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4564) trace_kmalloc_node(caller, ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4565) size, PAGE_SIZE << get_order(size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4566) gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4568) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4571) s = kmalloc_slab(size, gfpflags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4573) if (unlikely(ZERO_OR_NULL_PTR(s)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4574) return s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4576) ret = slab_alloc_node(s, gfpflags, node, caller, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4578) /* Honor the call site pointer we received. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4579) trace_kmalloc_node(caller, ret, size, s->size, gfpflags, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4581) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4583) EXPORT_SYMBOL(__kmalloc_node_track_caller);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4584) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4586) #ifdef CONFIG_SLUB_SYSFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4587) static int count_inuse(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4588) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4589) return page->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4590) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4592) static int count_total(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4594) return page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4596) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4598) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4599) static void validate_slab(struct kmem_cache *s, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4601) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4602) void *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4603) unsigned long *map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4605) slab_lock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4607) if (!check_slab(s, page) || !on_freelist(s, page, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4608) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4610) /* Now we know that a valid freelist exists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4611) map = get_map(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4612) for_each_object(p, s, addr, page->objects) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4613) u8 val = test_bit(__obj_to_index(s, addr, p), map) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4614) SLUB_RED_INACTIVE : SLUB_RED_ACTIVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4616) if (!check_object(s, page, p, val))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4617) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4619) put_map(map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4620) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4621) slab_unlock(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4624) static int validate_slab_node(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4625) struct kmem_cache_node *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4626) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4627) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4628) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4629) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4631) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4633) list_for_each_entry(page, &n->partial, slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4634) validate_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4635) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4637) if (count != n->nr_partial)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4638) pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4639) s->name, count, n->nr_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4641) if (!(s->flags & SLAB_STORE_USER))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4642) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4644) list_for_each_entry(page, &n->full, slab_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4645) validate_slab(s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4646) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4648) if (count != atomic_long_read(&n->nr_slabs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4649) pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4650) s->name, count, atomic_long_read(&n->nr_slabs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4652) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4653) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4654) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4657) static long validate_slab_cache(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4658) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4659) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4660) unsigned long count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4661) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4663) flush_all(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4664) for_each_kmem_cache_node(s, node, n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4665) count += validate_slab_node(s, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4667) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4669)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4670) #ifdef CONFIG_DEBUG_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4671) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4672) * Generate lists of code addresses where slabcache objects are allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4673) * and freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4674) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4676) struct location {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4677) unsigned long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4678) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4679) long long sum_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4680) long min_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4681) long max_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4682) long min_pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4683) long max_pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4684) DECLARE_BITMAP(cpus, NR_CPUS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4685) nodemask_t nodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4686) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4688) struct loc_track {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4689) unsigned long max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4690) unsigned long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4691) struct location *loc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4692) loff_t idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4693) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4695) static struct dentry *slab_debugfs_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4697) static void free_loc_track(struct loc_track *t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4698) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4699) if (t->max)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4700) free_pages((unsigned long)t->loc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4701) get_order(sizeof(struct location) * t->max));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4702) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4704) static int alloc_loc_track(struct loc_track *t, unsigned long max, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4705) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4706) struct location *l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4707) int order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4709) order = get_order(sizeof(struct location) * max);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4710)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4711) l = (void *)__get_free_pages(flags, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4712) if (!l)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4713) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4714)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4715) if (t->count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4716) memcpy(l, t->loc, sizeof(struct location) * t->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4717) free_loc_track(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4719) t->max = max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4720) t->loc = l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4721) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4723)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4724) static int add_location(struct loc_track *t, struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4725) const struct track *track)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4726) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4727) long start, end, pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4728) struct location *l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4729) unsigned long caddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4730) unsigned long age = jiffies - track->when;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4732) start = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4733) end = t->count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4735) for ( ; ; ) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4736) pos = start + (end - start + 1) / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4738) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4739) * There is nothing at "end". If we end up there
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4740) * we need to add something to before end.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4741) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4742) if (pos == end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4743) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4745) caddr = t->loc[pos].addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4746) if (track->addr == caddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4748) l = &t->loc[pos];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4749) l->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4750) if (track->when) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4751) l->sum_time += age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4752) if (age < l->min_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4753) l->min_time = age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4754) if (age > l->max_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4755) l->max_time = age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4756)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4757) if (track->pid < l->min_pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4758) l->min_pid = track->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4759) if (track->pid > l->max_pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4760) l->max_pid = track->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4761)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4762) cpumask_set_cpu(track->cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4763) to_cpumask(l->cpus));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4764) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4765) node_set(page_to_nid(virt_to_page(track)), l->nodes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4766) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4769) if (track->addr < caddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4770) end = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4771) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4772) start = pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4775) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4776) * Not found. Insert new tracking element.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4777) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4778) if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4779) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4781) l = t->loc + pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4782) if (pos < t->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4783) memmove(l + 1, l,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4784) (t->count - pos) * sizeof(struct location));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4785) t->count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4786) l->count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4787) l->addr = track->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4788) l->sum_time = age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4789) l->min_time = age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4790) l->max_time = age;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4791) l->min_pid = track->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4792) l->max_pid = track->pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4793) cpumask_clear(to_cpumask(l->cpus));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4794) cpumask_set_cpu(track->cpu, to_cpumask(l->cpus));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4795) nodes_clear(l->nodes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4796) node_set(page_to_nid(virt_to_page(track)), l->nodes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4797) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4800) static void process_slab(struct loc_track *t, struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4801) struct page *page, enum track_item alloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4802) unsigned long *obj_map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4803) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4804) void *addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4805) void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4807) __fill_map(obj_map, s, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4809) for_each_object(p, s, addr, page->objects)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4810) if (!test_bit(__obj_to_index(s, addr, p), obj_map))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4811) add_location(t, s, get_track(s, p, alloc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4812) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4813) #endif /* CONFIG_DEBUG_FS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4814) #endif /* CONFIG_SLUB_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4816) #ifdef SLUB_RESILIENCY_TEST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4817) static void __init resiliency_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4818) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4819) u8 *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4820) int type = KMALLOC_NORMAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4821)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4822) BUILD_BUG_ON(KMALLOC_MIN_SIZE > 16 || KMALLOC_SHIFT_HIGH < 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4824) pr_err("SLUB resiliency testing\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4825) pr_err("-----------------------\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4826) pr_err("A. Corruption after allocation\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4828) p = kzalloc(16, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4829) p[16] = 0x12;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4830) pr_err("\n1. kmalloc-16: Clobber Redzone/next pointer 0x12->0x%p\n\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4831) p + 16);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4833) validate_slab_cache(kmalloc_caches[type][4]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4834)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4835) /* Hmmm... The next two are dangerous */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4836) p = kzalloc(32, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4837) p[32 + sizeof(void *)] = 0x34;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4838) pr_err("\n2. kmalloc-32: Clobber next pointer/next slab 0x34 -> -0x%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4839) p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4840) pr_err("If allocated object is overwritten then not detectable\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4842) validate_slab_cache(kmalloc_caches[type][5]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4843) p = kzalloc(64, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4844) p += 64 + (get_cycles() & 0xff) * sizeof(void *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4845) *p = 0x56;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4846) pr_err("\n3. kmalloc-64: corrupting random byte 0x56->0x%p\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4847) p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4848) pr_err("If allocated object is overwritten then not detectable\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4849) validate_slab_cache(kmalloc_caches[type][6]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4850)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4851) pr_err("\nB. Corruption after free\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4852) p = kzalloc(128, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4853) kfree(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4854) *p = 0x78;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4855) pr_err("1. kmalloc-128: Clobber first word 0x78->0x%p\n\n", p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4856) validate_slab_cache(kmalloc_caches[type][7]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4858) p = kzalloc(256, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4859) kfree(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4860) p[50] = 0x9a;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4861) pr_err("\n2. kmalloc-256: Clobber 50th byte 0x9a->0x%p\n\n", p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4862) validate_slab_cache(kmalloc_caches[type][8]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4864) p = kzalloc(512, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4865) kfree(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4866) p[512] = 0xab;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4867) pr_err("\n3. kmalloc-512: Clobber redzone 0xab->0x%p\n\n", p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4868) validate_slab_cache(kmalloc_caches[type][9]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4870) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4871) #ifdef CONFIG_SLUB_SYSFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4872) static void resiliency_test(void) {};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4873) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4874) #endif /* SLUB_RESILIENCY_TEST */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4876) #ifdef CONFIG_SLUB_SYSFS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4877) enum slab_stat_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4878) SL_ALL, /* All slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4879) SL_PARTIAL, /* Only partially allocated slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4880) SL_CPU, /* Only slabs used for cpu caches */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4881) SL_OBJECTS, /* Determine allocated objects not slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4882) SL_TOTAL /* Determine object capacity not slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4883) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4885) #define SO_ALL (1 << SL_ALL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4886) #define SO_PARTIAL (1 << SL_PARTIAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4887) #define SO_CPU (1 << SL_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4888) #define SO_OBJECTS (1 << SL_OBJECTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4889) #define SO_TOTAL (1 << SL_TOTAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4890)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4891) #ifdef CONFIG_MEMCG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4892) static bool memcg_sysfs_enabled = IS_ENABLED(CONFIG_SLUB_MEMCG_SYSFS_ON);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4893)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4894) static int __init setup_slub_memcg_sysfs(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4895) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4896) int v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4898) if (get_option(&str, &v) > 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4899) memcg_sysfs_enabled = v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4901) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4904) __setup("slub_memcg_sysfs=", setup_slub_memcg_sysfs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4905) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4907) static ssize_t show_slab_objects(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4908) char *buf, unsigned long flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4909) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4910) unsigned long total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4911) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4912) int x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4913) unsigned long *nodes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4915) nodes = kcalloc(nr_node_ids, sizeof(unsigned long), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4916) if (!nodes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4917) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4919) if (flags & SO_CPU) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4920) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4922) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4923) struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4924) cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4925) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4926) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4928) page = READ_ONCE(c->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4929) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4930) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4931)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4932) node = page_to_nid(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4933) if (flags & SO_TOTAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4934) x = page->objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4935) else if (flags & SO_OBJECTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4936) x = page->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4937) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4938) x = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4940) total += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4941) nodes[node] += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4942)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4943) page = slub_percpu_partial_read_once(c);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4944) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4945) node = page_to_nid(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4946) if (flags & SO_TOTAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4947) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4948) else if (flags & SO_OBJECTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4949) WARN_ON_ONCE(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4950) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4951) x = page->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4952) total += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4953) nodes[node] += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4954) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4955) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4958) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4959) * It is impossible to take "mem_hotplug_lock" here with "kernfs_mutex"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4960) * already held which will conflict with an existing lock order:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4961) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4962) * mem_hotplug_lock->slab_mutex->kernfs_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4963) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4964) * We don't really need mem_hotplug_lock (to hold off
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4965) * slab_mem_going_offline_callback) here because slab's memory hot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4966) * unplug code doesn't destroy the kmem_cache->node[] data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4967) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4969) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4970) if (flags & SO_ALL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4971) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4973) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4974)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4975) if (flags & SO_TOTAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4976) x = atomic_long_read(&n->total_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4977) else if (flags & SO_OBJECTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4978) x = atomic_long_read(&n->total_objects) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4979) count_partial(n, count_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4980) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4981) x = atomic_long_read(&n->nr_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4982) total += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4983) nodes[node] += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4985)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4986) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4987) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4988) if (flags & SO_PARTIAL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4989) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4991) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4992) if (flags & SO_TOTAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4993) x = count_partial(n, count_total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4994) else if (flags & SO_OBJECTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4995) x = count_partial(n, count_inuse);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4996) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4997) x = n->nr_partial;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4998) total += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4999) nodes[node] += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5001) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5002) x = sprintf(buf, "%lu", total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5003) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5004) for (node = 0; node < nr_node_ids; node++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5005) if (nodes[node])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5006) x += sprintf(buf + x, " N%d=%lu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5007) node, nodes[node]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5008) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5009) kfree(nodes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5010) return x + sprintf(buf + x, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5013) #define to_slab_attr(n) container_of(n, struct slab_attribute, attr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5014) #define to_slab(n) container_of(n, struct kmem_cache, kobj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5016) struct slab_attribute {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5017) struct attribute attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5018) ssize_t (*show)(struct kmem_cache *s, char *buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5019) ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5020) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5021)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5022) #define SLAB_ATTR_RO(_name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5023) static struct slab_attribute _name##_attr = \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5024) __ATTR(_name, 0400, _name##_show, NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5026) #define SLAB_ATTR(_name) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5027) static struct slab_attribute _name##_attr = \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5028) __ATTR(_name, 0600, _name##_show, _name##_store)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5030) static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5031) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5032) return sprintf(buf, "%u\n", s->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5033) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5034) SLAB_ATTR_RO(slab_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5036) static ssize_t align_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5037) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5038) return sprintf(buf, "%u\n", s->align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5039) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5040) SLAB_ATTR_RO(align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5041)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5042) static ssize_t object_size_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5044) return sprintf(buf, "%u\n", s->object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5046) SLAB_ATTR_RO(object_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5048) static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5050) return sprintf(buf, "%u\n", oo_objects(s->oo));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5052) SLAB_ATTR_RO(objs_per_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5054) static ssize_t order_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5055) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5056) return sprintf(buf, "%u\n", oo_order(s->oo));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5058) SLAB_ATTR_RO(order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5060) static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5061) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5062) return sprintf(buf, "%lu\n", s->min_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5063) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5064)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5065) static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5066) size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5067) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5068) unsigned long min;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5069) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5071) err = kstrtoul(buf, 10, &min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5072) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5073) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5075) set_min_partial(s, min);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5076) return length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5077) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5078) SLAB_ATTR(min_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5080) static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5081) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5082) return sprintf(buf, "%u\n", slub_cpu_partial(s));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5084)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5085) static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5086) size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5087) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5088) unsigned int objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5089) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5091) err = kstrtouint(buf, 10, &objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5092) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5093) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5094) if (objects && !kmem_cache_has_cpu_partial(s))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5095) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5097) slub_set_cpu_partial(s, objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5098) flush_all(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5099) return length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5101) SLAB_ATTR(cpu_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5103) static ssize_t ctor_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5105) if (!s->ctor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5106) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5107) return sprintf(buf, "%pS\n", s->ctor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5109) SLAB_ATTR_RO(ctor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5111) static ssize_t aliases_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5113) return sprintf(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5115) SLAB_ATTR_RO(aliases);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5117) static ssize_t partial_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5119) return show_slab_objects(s, buf, SO_PARTIAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5121) SLAB_ATTR_RO(partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5123) static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5125) return show_slab_objects(s, buf, SO_CPU);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5126) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5127) SLAB_ATTR_RO(cpu_slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5129) static ssize_t objects_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5130) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5131) return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5133) SLAB_ATTR_RO(objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5135) static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5137) return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5139) SLAB_ATTR_RO(objects_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5141) static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5143) int objects = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5144) int pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5145) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5146) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5148) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5149) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5150)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5151) page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5153) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5154) pages += page->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5155) objects += page->pobjects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5159) len = sprintf(buf, "%d(%d)", objects, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5161) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5162) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5163) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5165) page = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5167) if (page && len < PAGE_SIZE - 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5168) len += sprintf(buf + len, " C%d=%d(%d)", cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5169) page->pobjects, page->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5171) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5172) return len + sprintf(buf + len, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5174) SLAB_ATTR_RO(slabs_cpu_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5176) static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5178) return sprintf(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5179) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5180) SLAB_ATTR_RO(reclaim_account);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5182) static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5183) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5184) return sprintf(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5186) SLAB_ATTR_RO(hwcache_align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5188) #ifdef CONFIG_ZONE_DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5189) static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5191) return sprintf(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5193) SLAB_ATTR_RO(cache_dma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5194) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5196) static ssize_t usersize_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5198) return sprintf(buf, "%u\n", s->usersize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5200) SLAB_ATTR_RO(usersize);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5202) static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5204) return sprintf(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5206) SLAB_ATTR_RO(destroy_by_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5208) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5209) static ssize_t slabs_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5211) return show_slab_objects(s, buf, SO_ALL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5212) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5213) SLAB_ATTR_RO(slabs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5215) static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5217) return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5219) SLAB_ATTR_RO(total_objects);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5221) static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5223) return sprintf(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5225) SLAB_ATTR_RO(sanity_checks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5227) static ssize_t trace_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5229) return sprintf(buf, "%d\n", !!(s->flags & SLAB_TRACE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5231) SLAB_ATTR_RO(trace);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5233) static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5235) return sprintf(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5238) SLAB_ATTR_RO(red_zone);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5240) static ssize_t poison_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5241) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5242) return sprintf(buf, "%d\n", !!(s->flags & SLAB_POISON));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5245) SLAB_ATTR_RO(poison);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5247) static ssize_t store_user_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5248) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5249) return sprintf(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5252) SLAB_ATTR_RO(store_user);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5254) static ssize_t validate_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5255) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5256) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5259) static ssize_t validate_store(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5260) const char *buf, size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5262) int ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5264) if (buf[0] == '1') {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5265) ret = validate_slab_cache(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5266) if (ret >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5267) ret = length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5269) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5271) SLAB_ATTR(validate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5273) #endif /* CONFIG_SLUB_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5275) #ifdef CONFIG_FAILSLAB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5276) static ssize_t failslab_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5278) return sprintf(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5280) SLAB_ATTR_RO(failslab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5281) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5283) static ssize_t shrink_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5285) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5288) static ssize_t shrink_store(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5289) const char *buf, size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5291) if (buf[0] == '1')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5292) kmem_cache_shrink(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5293) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5294) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5295) return length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5297) SLAB_ATTR(shrink);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5299) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5300) static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5302) return sprintf(buf, "%u\n", s->remote_node_defrag_ratio / 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5304)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5305) static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5306) const char *buf, size_t length)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5308) unsigned int ratio;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5309) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5311) err = kstrtouint(buf, 10, &ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5312) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5313) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5314) if (ratio > 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5315) return -ERANGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5317) s->remote_node_defrag_ratio = ratio * 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5319) return length;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5321) SLAB_ATTR(remote_node_defrag_ratio);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5322) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5324) #ifdef CONFIG_SLUB_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5325) static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5326) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5327) unsigned long sum = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5328) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5329) int len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5330) int *data = kmalloc_array(nr_cpu_ids, sizeof(int), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5332) if (!data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5333) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5335) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5336) unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5338) data[cpu] = x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5339) sum += x;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5342) len = sprintf(buf, "%lu", sum);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5344) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5345) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5346) if (data[cpu] && len < PAGE_SIZE - 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5347) len += sprintf(buf + len, " C%d=%u", cpu, data[cpu]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5349) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5350) kfree(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5351) return len + sprintf(buf + len, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5354) static void clear_stat(struct kmem_cache *s, enum stat_item si)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5356) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5358) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5359) per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5360) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5362) #define STAT_ATTR(si, text) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5363) static ssize_t text##_show(struct kmem_cache *s, char *buf) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5364) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5365) return show_stat(s, buf, si); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5366) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5367) static ssize_t text##_store(struct kmem_cache *s, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5368) const char *buf, size_t length) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5369) { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5370) if (buf[0] != '0') \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5371) return -EINVAL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5372) clear_stat(s, si); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5373) return length; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5374) } \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5375) SLAB_ATTR(text); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5377) STAT_ATTR(ALLOC_FASTPATH, alloc_fastpath);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5378) STAT_ATTR(ALLOC_SLOWPATH, alloc_slowpath);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5379) STAT_ATTR(FREE_FASTPATH, free_fastpath);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5380) STAT_ATTR(FREE_SLOWPATH, free_slowpath);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5381) STAT_ATTR(FREE_FROZEN, free_frozen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5382) STAT_ATTR(FREE_ADD_PARTIAL, free_add_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5383) STAT_ATTR(FREE_REMOVE_PARTIAL, free_remove_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5384) STAT_ATTR(ALLOC_FROM_PARTIAL, alloc_from_partial);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5385) STAT_ATTR(ALLOC_SLAB, alloc_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5386) STAT_ATTR(ALLOC_REFILL, alloc_refill);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5387) STAT_ATTR(ALLOC_NODE_MISMATCH, alloc_node_mismatch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5388) STAT_ATTR(FREE_SLAB, free_slab);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5389) STAT_ATTR(CPUSLAB_FLUSH, cpuslab_flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5390) STAT_ATTR(DEACTIVATE_FULL, deactivate_full);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5391) STAT_ATTR(DEACTIVATE_EMPTY, deactivate_empty);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5392) STAT_ATTR(DEACTIVATE_TO_HEAD, deactivate_to_head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5393) STAT_ATTR(DEACTIVATE_TO_TAIL, deactivate_to_tail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5394) STAT_ATTR(DEACTIVATE_REMOTE_FREES, deactivate_remote_frees);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5395) STAT_ATTR(DEACTIVATE_BYPASS, deactivate_bypass);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5396) STAT_ATTR(ORDER_FALLBACK, order_fallback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5397) STAT_ATTR(CMPXCHG_DOUBLE_CPU_FAIL, cmpxchg_double_cpu_fail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5398) STAT_ATTR(CMPXCHG_DOUBLE_FAIL, cmpxchg_double_fail);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5399) STAT_ATTR(CPU_PARTIAL_ALLOC, cpu_partial_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5400) STAT_ATTR(CPU_PARTIAL_FREE, cpu_partial_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5401) STAT_ATTR(CPU_PARTIAL_NODE, cpu_partial_node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5402) STAT_ATTR(CPU_PARTIAL_DRAIN, cpu_partial_drain);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5403) #endif /* CONFIG_SLUB_STATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5405) static struct attribute *slab_attrs[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5406) &slab_size_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5407) &object_size_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5408) &objs_per_slab_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5409) &order_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5410) &min_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5411) &cpu_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5412) &objects_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5413) &objects_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5414) &partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5415) &cpu_slabs_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5416) &ctor_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5417) &aliases_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5418) &align_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5419) &hwcache_align_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5420) &reclaim_account_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5421) &destroy_by_rcu_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5422) &shrink_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5423) &slabs_cpu_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5424) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5425) &total_objects_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5426) &slabs_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5427) &sanity_checks_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5428) &trace_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5429) &red_zone_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5430) &poison_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5431) &store_user_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5432) &validate_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5433) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5434) #ifdef CONFIG_ZONE_DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5435) &cache_dma_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5436) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5437) #ifdef CONFIG_NUMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5438) &remote_node_defrag_ratio_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5439) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5440) #ifdef CONFIG_SLUB_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5441) &alloc_fastpath_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5442) &alloc_slowpath_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5443) &free_fastpath_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5444) &free_slowpath_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5445) &free_frozen_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5446) &free_add_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5447) &free_remove_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5448) &alloc_from_partial_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5449) &alloc_slab_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5450) &alloc_refill_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5451) &alloc_node_mismatch_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5452) &free_slab_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5453) &cpuslab_flush_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5454) &deactivate_full_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5455) &deactivate_empty_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5456) &deactivate_to_head_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5457) &deactivate_to_tail_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5458) &deactivate_remote_frees_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5459) &deactivate_bypass_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5460) &order_fallback_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5461) &cmpxchg_double_fail_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5462) &cmpxchg_double_cpu_fail_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5463) &cpu_partial_alloc_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5464) &cpu_partial_free_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5465) &cpu_partial_node_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5466) &cpu_partial_drain_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5467) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5468) #ifdef CONFIG_FAILSLAB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5469) &failslab_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5470) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5471) &usersize_attr.attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5473) NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5474) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5476) static const struct attribute_group slab_attr_group = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5477) .attrs = slab_attrs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5478) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5480) static ssize_t slab_attr_show(struct kobject *kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5481) struct attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5482) char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5484) struct slab_attribute *attribute;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5485) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5486) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5488) attribute = to_slab_attr(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5489) s = to_slab(kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5491) if (!attribute->show)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5492) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5494) err = attribute->show(s, buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5496) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5499) static ssize_t slab_attr_store(struct kobject *kobj,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5500) struct attribute *attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5501) const char *buf, size_t len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5503) struct slab_attribute *attribute;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5504) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5505) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5507) attribute = to_slab_attr(attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5508) s = to_slab(kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5510) if (!attribute->store)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5511) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5513) err = attribute->store(s, buf, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5514) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5517) static void kmem_cache_release(struct kobject *k)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5518) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5519) slab_kmem_cache_release(to_slab(k));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5522) static const struct sysfs_ops slab_sysfs_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5523) .show = slab_attr_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5524) .store = slab_attr_store,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5525) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5527) static struct kobj_type slab_ktype = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5528) .sysfs_ops = &slab_sysfs_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5529) .release = kmem_cache_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5530) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5531)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5532) static struct kset *slab_kset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5534) static inline struct kset *cache_kset(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5535) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5536) return slab_kset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5537) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5539) #define ID_STR_LENGTH 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5541) /* Create a unique string id for a slab cache:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5542) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5543) * Format :[flags-]size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5544) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5545) static char *create_unique_id(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5547) char *name = kmalloc(ID_STR_LENGTH, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5548) char *p = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5550) BUG_ON(!name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5552) *p++ = ':';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5553) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5554) * First flags affecting slabcache operations. We will only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5555) * get here for aliasable slabs so we do not need to support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5556) * too many flags. The flags here must cover all flags that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5557) * are matched during merging to guarantee that the id is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5558) * unique.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5559) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5560) if (s->flags & SLAB_CACHE_DMA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5561) *p++ = 'd';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5562) if (s->flags & SLAB_CACHE_DMA32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5563) *p++ = 'D';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5564) if (s->flags & SLAB_RECLAIM_ACCOUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5565) *p++ = 'a';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5566) if (s->flags & SLAB_CONSISTENCY_CHECKS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5567) *p++ = 'F';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5568) if (s->flags & SLAB_ACCOUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5569) *p++ = 'A';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5570) if (p != name + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5571) *p++ = '-';
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5572) p += sprintf(p, "%07u", s->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5574) BUG_ON(p > name + ID_STR_LENGTH - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5575) return name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5578) static int sysfs_slab_add(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5579) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5580) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5581) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5582) struct kset *kset = cache_kset(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5583) int unmergeable = slab_unmergeable(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5585) if (!kset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5586) kobject_init(&s->kobj, &slab_ktype);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5587) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5588) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5590) if (!unmergeable && disable_higher_order_debug &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5591) (slub_debug & DEBUG_METADATA_FLAGS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5592) unmergeable = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5594) if (unmergeable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5595) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5596) * Slabcache can never be merged so we can use the name proper.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5597) * This is typically the case for debug situations. In that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5598) * case we can catch duplicate names easily.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5599) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5600) sysfs_remove_link(&slab_kset->kobj, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5601) name = s->name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5602) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5603) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5604) * Create a unique name for the slab as a target
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5605) * for the symlinks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5606) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5607) name = create_unique_id(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5610) s->kobj.kset = kset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5611) err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5612) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5613) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5615) err = sysfs_create_group(&s->kobj, &slab_attr_group);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5616) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5617) goto out_del_kobj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5619) if (!unmergeable) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5620) /* Setup first alias */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5621) sysfs_slab_alias(s, s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5623) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5624) if (!unmergeable)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5625) kfree(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5626) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5627) out_del_kobj:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5628) kobject_del(&s->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5629) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5630) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5632) void sysfs_slab_unlink(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5633) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5634) if (slab_state >= FULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5635) kobject_del(&s->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5638) void sysfs_slab_release(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5639) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5640) if (slab_state >= FULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5641) kobject_put(&s->kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5644) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5645) * Need to buffer aliases during bootup until sysfs becomes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5646) * available lest we lose that information.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5647) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5648) struct saved_alias {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5649) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5650) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5651) struct saved_alias *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5652) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5654) static struct saved_alias *alias_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5656) static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5658) struct saved_alias *al;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5660) if (slab_state == FULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5661) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5662) * If we have a leftover link then remove it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5664) sysfs_remove_link(&slab_kset->kobj, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5665) return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5668) al = kmalloc(sizeof(struct saved_alias), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5669) if (!al)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5670) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5672) al->s = s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5673) al->name = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5674) al->next = alias_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5675) alias_list = al;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5676) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5677) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5678)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5679) static int __init slab_sysfs_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5680) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5681) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5682) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5684) mutex_lock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5686) slab_kset = kset_create_and_add("slab", NULL, kernel_kobj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5687) if (!slab_kset) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5688) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5689) pr_err("Cannot register slab subsystem.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5690) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5692)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5693) slab_state = FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5695) list_for_each_entry(s, &slab_caches, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5696) err = sysfs_slab_add(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5697) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5698) pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5699) s->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5702) while (alias_list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5703) struct saved_alias *al = alias_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5705) alias_list = alias_list->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5706) err = sysfs_slab_alias(al->s, al->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5707) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5708) pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5709) al->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5710) kfree(al);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5712)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5713) mutex_unlock(&slab_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5714) resiliency_test();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5715) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5718) __initcall(slab_sysfs_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5719) #endif /* CONFIG_SLUB_SYSFS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5721) #if defined(CONFIG_SLUB_DEBUG) && defined(CONFIG_DEBUG_FS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5722) static int slab_debugfs_show(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5723) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5724) struct loc_track *t = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5725) struct location *l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5726) unsigned long idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5727)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5728) idx = (unsigned long) t->idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5729) if (idx < t->count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5730) l = &t->loc[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5732) seq_printf(seq, "%7ld ", l->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5734) if (l->addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5735) seq_printf(seq, "%pS", (void *)l->addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5736) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5737) seq_puts(seq, "<not-available>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5738)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5739) if (l->sum_time != l->min_time) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5740) seq_printf(seq, " age=%ld/%llu/%ld",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5741) l->min_time, div_u64(l->sum_time, l->count),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5742) l->max_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5743) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5744) seq_printf(seq, " age=%ld", l->min_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5746) if (l->min_pid != l->max_pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5747) seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5748) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5749) seq_printf(seq, " pid=%ld",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5750) l->min_pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5752) if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5753) seq_printf(seq, " cpus=%*pbl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5754) cpumask_pr_args(to_cpumask(l->cpus)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5756) if (nr_online_nodes > 1 && !nodes_empty(l->nodes))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5757) seq_printf(seq, " nodes=%*pbl",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5758) nodemask_pr_args(&l->nodes));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5760) seq_puts(seq, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5763) if (!idx && !t->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5764) seq_puts(seq, "No data\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5766) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5767) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5769) static void slab_debugfs_stop(struct seq_file *seq, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5773) static void *slab_debugfs_next(struct seq_file *seq, void *v, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5774) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5775) struct loc_track *t = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5777) t->idx = ++(*ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5778) if (*ppos <= t->count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5779) return ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5781) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5784) static void *slab_debugfs_start(struct seq_file *seq, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5785) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5786) struct loc_track *t = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5788) t->idx = *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5789) return ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5790) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5791)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5792) static const struct seq_operations slab_debugfs_sops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5793) .start = slab_debugfs_start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5794) .next = slab_debugfs_next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5795) .stop = slab_debugfs_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5796) .show = slab_debugfs_show,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5797) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5799) static int slab_debug_trace_open(struct inode *inode, struct file *filep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5802) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5803) enum track_item alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5804) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5805) struct loc_track *t = __seq_open_private(filep, &slab_debugfs_sops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5806) sizeof(struct loc_track));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5807) struct kmem_cache *s = file_inode(filep)->i_private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5808) unsigned long *obj_map;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5810) if (!t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5811) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5812)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5813) obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5814) if (!obj_map) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5815) seq_release_private(inode, filep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5816) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5817) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5819) if (strcmp(filep->f_path.dentry->d_name.name, "alloc_traces") == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5820) alloc = TRACK_ALLOC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5821) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5822) alloc = TRACK_FREE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5824) if (!alloc_loc_track(t, PAGE_SIZE / sizeof(struct location), GFP_KERNEL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5825) bitmap_free(obj_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5826) seq_release_private(inode, filep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5827) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5830) /* Push back cpu slabs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5831) flush_all(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5833) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5834) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5835) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5836)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5837) if (!atomic_long_read(&n->nr_slabs))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5838) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5840) spin_lock_irqsave(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5841) list_for_each_entry(page, &n->partial, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5842) process_slab(t, s, page, alloc, obj_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5843) list_for_each_entry(page, &n->full, slab_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5844) process_slab(t, s, page, alloc, obj_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5845) spin_unlock_irqrestore(&n->list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5848) bitmap_free(obj_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5849) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5850) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5852) static int slab_debug_trace_release(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5854) struct seq_file *seq = file->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5855) struct loc_track *t = seq->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5857) free_loc_track(t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5858) return seq_release_private(inode, file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5859) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5860)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5861) static const struct file_operations slab_debugfs_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5862) .open = slab_debug_trace_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5863) .read = seq_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5864) .llseek = seq_lseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5865) .release = slab_debug_trace_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5866) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5868) static void debugfs_slab_add(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5870) struct dentry *slab_cache_dir;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5871)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5872) if (unlikely(!slab_debugfs_root))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5873) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5874)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5875) slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5877) debugfs_create_file("alloc_traces", 0400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5878) slab_cache_dir, s, &slab_debugfs_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5880) debugfs_create_file("free_traces", 0400,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5881) slab_cache_dir, s, &slab_debugfs_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5882) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5884) void debugfs_slab_release(struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5885) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5886) debugfs_remove_recursive(debugfs_lookup(s->name, slab_debugfs_root));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5888)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5889) static int __init slab_debugfs_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5890) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5891) struct kmem_cache *s;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5893) slab_debugfs_root = debugfs_create_dir("slab", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5895) list_for_each_entry(s, &slab_caches, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5896) if (s->flags & SLAB_STORE_USER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5897) debugfs_slab_add(s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5898)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5899) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5902) __initcall(slab_debugfs_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5903) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5904) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5905) * The /proc/slabinfo ABI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5906) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5907) #ifdef CONFIG_SLUB_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5908) void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5909) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5910) unsigned long nr_slabs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5911) unsigned long nr_objs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5912) unsigned long nr_free = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5913) int node;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5914) struct kmem_cache_node *n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5916) for_each_kmem_cache_node(s, node, n) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5917) nr_slabs += node_nr_slabs(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5918) nr_objs += node_nr_objs(n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5919) nr_free += count_partial(n, count_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5922) sinfo->active_objs = nr_objs - nr_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5923) sinfo->num_objs = nr_objs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5924) sinfo->active_slabs = nr_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5925) sinfo->num_slabs = nr_slabs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5926) sinfo->objects_per_slab = oo_objects(s->oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5927) sinfo->cache_order = oo_order(s->oo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5928) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5929) EXPORT_SYMBOL_GPL(get_slabinfo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5931) void slabinfo_show_stats(struct seq_file *m, struct kmem_cache *s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5932) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5935) ssize_t slabinfo_write(struct file *file, const char __user *buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5936) size_t count, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5937) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5938) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5939) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5940) #endif /* CONFIG_SLUB_DEBUG */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags