Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #ifndef _MM_PERCPU_INTERNAL_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #define _MM_PERCPU_INTERNAL_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * There are two chunk types: root and memcg-aware.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * Chunks of each type have separate slots list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * Memcg-aware chunks have an attached vector of obj_cgroup pointers, which is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * used to store memcg membership data of a percpu object.  Obj_cgroups are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * ref-counted pointers to a memory cgroup with an ability to switch dynamically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * to the parent memory cgroup.  This allows to reclaim a deleted memory cgroup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * without reclaiming of all outstanding objects, which hold a reference at it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) enum pcpu_chunk_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 	PCPU_CHUNK_ROOT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	PCPU_CHUNK_MEMCG,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	PCPU_NR_CHUNK_TYPES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	PCPU_FAIL_ALLOC = PCPU_NR_CHUNK_TYPES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * pcpu_block_md is the metadata block struct.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * Each chunk's bitmap is split into a number of full blocks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  * All units are in terms of bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  * The scan hint is the largest known contiguous area before the contig hint.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  * It is not necessarily the actual largest contig hint though.  There is an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  * invariant that the scan_hint_start > contig_hint_start iff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * scan_hint == contig_hint.  This is necessary because when scanning forward,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  * we don't know if a new contig hint would be better than the current one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) struct pcpu_block_md {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	int			scan_hint;	/* scan hint for block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 	int			scan_hint_start; /* block relative starting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 						    position of the scan hint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	int                     contig_hint;    /* contig hint for block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	int                     contig_hint_start; /* block relative starting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 						      position of the contig hint */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 	int                     left_free;      /* size of free space along
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 						   the left side of the block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	int                     right_free;     /* size of free space along
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 						   the right side of the block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 	int                     first_free;     /* block position of first free */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	int			nr_bits;	/* total bits responsible for */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) struct pcpu_chunk {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) #ifdef CONFIG_PERCPU_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	int			nr_alloc;	/* # of allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	size_t			max_alloc_size; /* largest allocation size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	struct list_head	list;		/* linked to pcpu_slot lists */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 	int			free_bytes;	/* free bytes in the chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	struct pcpu_block_md	chunk_md;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	void			*base_addr;	/* base address of this chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	unsigned long		*alloc_map;	/* allocation map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	unsigned long		*bound_map;	/* boundary map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	struct pcpu_block_md	*md_blocks;	/* metadata blocks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	void			*data;		/* chunk data */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	bool			immutable;	/* no [de]population allowed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	int			start_offset;	/* the overlap with the previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 						   region to have a page aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 						   base_addr */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	int			end_offset;	/* additional area required to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 						   have the region end page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 						   aligned */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	struct obj_cgroup	**obj_cgroups;	/* vector of object cgroups */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	int			nr_pages;	/* # of pages served by this chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	int			nr_populated;	/* # of populated pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	int                     nr_empty_pop_pages; /* # of empty populated pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	unsigned long		populated[];	/* populated bitmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) extern spinlock_t pcpu_lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) extern struct list_head *pcpu_chunk_lists;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) extern int pcpu_nr_slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) extern int pcpu_nr_empty_pop_pages[];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) extern struct pcpu_chunk *pcpu_first_chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) extern struct pcpu_chunk *pcpu_reserved_chunk;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96)  * pcpu_chunk_nr_blocks - converts nr_pages to # of md_blocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97)  * @chunk: chunk of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99)  * This conversion is from the number of physical pages that the chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)  * serves to the number of bitmap blocks used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static inline int pcpu_chunk_nr_blocks(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	return chunk->nr_pages * PAGE_SIZE / PCPU_BITMAP_BLOCK_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  * pcpu_nr_pages_to_map_bits - converts the pages to size of bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  * @pages: number of physical pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * This conversion is from physical pages to the number of bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  * required in the bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) static inline int pcpu_nr_pages_to_map_bits(int pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	return pages * PAGE_SIZE / PCPU_MIN_ALLOC_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  * pcpu_chunk_map_bits - helper to convert nr_pages to size of bitmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  * @chunk: chunk of interest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  * This conversion is from the number of physical pages that the chunk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  * serves to the number of bits in the bitmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static inline int pcpu_chunk_map_bits(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	return pcpu_nr_pages_to_map_bits(chunk->nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #ifdef CONFIG_MEMCG_KMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	if (chunk->obj_cgroups)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 		return PCPU_CHUNK_MEMCG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	return PCPU_CHUNK_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	return chunk_type == PCPU_CHUNK_MEMCG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static inline enum pcpu_chunk_type pcpu_chunk_type(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	return PCPU_CHUNK_ROOT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static inline bool pcpu_is_memcg_chunk(enum pcpu_chunk_type chunk_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static inline struct list_head *pcpu_chunk_list(enum pcpu_chunk_type chunk_type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	return &pcpu_chunk_lists[pcpu_nr_slots *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 				 pcpu_is_memcg_chunk(chunk_type)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #ifdef CONFIG_PERCPU_STATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) struct percpu_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	u64 nr_alloc;		/* lifetime # of allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	u64 nr_dealloc;		/* lifetime # of deallocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	u64 nr_cur_alloc;	/* current # of allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	u64 nr_max_alloc;	/* max # of live allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	u32 nr_chunks;		/* current # of live chunks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	u32 nr_max_chunks;	/* max # of live chunks */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	size_t min_alloc_size;	/* min allocaiton size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	size_t max_alloc_size;	/* max allocation size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) extern struct percpu_stats pcpu_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) extern struct pcpu_alloc_info pcpu_stats_ai;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)  * For debug purposes. We don't care about the flexible array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	memcpy(&pcpu_stats_ai, ai, sizeof(struct pcpu_alloc_info));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	/* initialize min_alloc_size to unit_size */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	pcpu_stats.min_alloc_size = pcpu_stats_ai.unit_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)  * pcpu_stats_area_alloc - increment area allocation stats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)  * @chunk: the location of the area being allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)  * @size: size of area to allocate in bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)  * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  * pcpu_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	lockdep_assert_held(&pcpu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	pcpu_stats.nr_alloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	pcpu_stats.nr_cur_alloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	pcpu_stats.nr_max_alloc =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 		max(pcpu_stats.nr_max_alloc, pcpu_stats.nr_cur_alloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	pcpu_stats.min_alloc_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 		min(pcpu_stats.min_alloc_size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	pcpu_stats.max_alloc_size =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		max(pcpu_stats.max_alloc_size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	chunk->nr_alloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 	chunk->max_alloc_size = max(chunk->max_alloc_size, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  * pcpu_stats_area_dealloc - decrement allocation stats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  * @chunk: the location of the area being deallocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)  * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  * pcpu_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	lockdep_assert_held(&pcpu_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	pcpu_stats.nr_dealloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	pcpu_stats.nr_cur_alloc--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	chunk->nr_alloc--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)  * pcpu_stats_chunk_alloc - increment chunk stats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static inline void pcpu_stats_chunk_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 	spin_lock_irqsave(&pcpu_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	pcpu_stats.nr_chunks++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	pcpu_stats.nr_max_chunks =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		max(pcpu_stats.nr_max_chunks, pcpu_stats.nr_chunks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	spin_unlock_irqrestore(&pcpu_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)  * pcpu_stats_chunk_dealloc - decrement chunk stats
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static inline void pcpu_stats_chunk_dealloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	spin_lock_irqsave(&pcpu_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	pcpu_stats.nr_chunks--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	spin_unlock_irqrestore(&pcpu_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static inline void pcpu_stats_save_ai(const struct pcpu_alloc_info *ai)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static inline void pcpu_stats_area_alloc(struct pcpu_chunk *chunk, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static inline void pcpu_stats_area_dealloc(struct pcpu_chunk *chunk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static inline void pcpu_stats_chunk_alloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) static inline void pcpu_stats_chunk_dealloc(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #endif /* !CONFIG_PERCPU_STATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) #endif