Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/stacktrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/page_pinner.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/migrate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/stackdepot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include "internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define PAGE_PINNER_STACK_DEPTH 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #define LONGTERM_PIN_BUCKETS	4096
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) struct page_pinner {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	depot_stack_handle_t handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 	s64 ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 	atomic_t count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) struct captured_pinner {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 	depot_stack_handle_t handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 		s64 ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 		s64 elapsed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 	};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	/* struct page fields */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	int mapcount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 	struct address_space *mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) struct longterm_pinner {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 	spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	unsigned int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	struct captured_pinner pinner[LONGTERM_PIN_BUCKETS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) static struct longterm_pinner lt_pinner = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	.lock = __SPIN_LOCK_UNLOCKED(lt_pinner.lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static s64 threshold_usec = 300000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) /* alloc_contig failed pinner */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) static struct longterm_pinner acf_pinner = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 	.lock = __SPIN_LOCK_UNLOCKED(acf_pinner.lock),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) static bool page_pinner_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) DEFINE_STATIC_KEY_FALSE(page_pinner_inited);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) DEFINE_STATIC_KEY_TRUE(failure_tracking);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) EXPORT_SYMBOL(failure_tracking);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) static depot_stack_handle_t failure_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) static int __init early_page_pinner_param(char *buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	page_pinner_enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) early_param("page_pinner", early_page_pinner_param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) static bool need_page_pinner(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	return page_pinner_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) static noinline void register_failure_stack(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	unsigned long entries[4];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	failure_handle = stack_depot_save(entries, nr_entries, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) static void init_page_pinner(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	if (!page_pinner_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	register_failure_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	static_branch_enable(&page_pinner_inited);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) struct page_ext_operations page_pinner_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 	.size = sizeof(struct page_pinner),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	.need = need_page_pinner,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	.init = init_page_pinner,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static inline struct page_pinner *get_page_pinner(struct page_ext *page_ext)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	return (void *)page_ext + page_pinner_ops.offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) static noinline depot_stack_handle_t save_stack(gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	unsigned long entries[PAGE_PINNER_STACK_DEPTH];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	depot_stack_handle_t handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	handle = stack_depot_save(entries, nr_entries, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	if (!handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		handle = failure_handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	return handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void capture_page_state(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 			       struct captured_pinner *record)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	record->flags = page->flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	record->mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	record->pfn = page_to_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	record->count = page_count(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	record->mapcount = page_mapcount(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) static void check_longterm_pin(struct page_pinner *page_pinner,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 			      struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	s64 now, delta = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	unsigned int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	struct captured_pinner record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	now = ktime_to_us(ktime_get_boottime());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	/* get/put_page can be raced. Ignore that case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	if (page_pinner->ts_usec < now)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 		delta = now - page_pinner->ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	if (delta <= threshold_usec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	record.handle = page_pinner->handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	record.elapsed = delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	capture_page_state(page, &record);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	spin_lock_irqsave(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	idx = lt_pinner.index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	lt_pinner.index %= LONGTERM_PIN_BUCKETS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	lt_pinner.pinner[idx] = record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	spin_unlock_irqrestore(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) void __reset_page_pinner(struct page *page, unsigned int order, bool free)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	struct page_pinner *page_pinner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	struct page_ext *page_ext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	page_ext = lookup_page_ext(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	if (unlikely(!page_ext))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	for (i = 0; i < (1 << order); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		if (!test_bit(PAGE_EXT_GET, &page_ext->flags) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 			!test_bit(PAGE_EXT_PINNER_MIGRATION_FAILED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 				  &page_ext->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		page_pinner = get_page_pinner(page_ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		if (free) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 			/* record page free call path */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 			__page_pinner_migration_failed(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 			atomic_set(&page_pinner->count, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 			__clear_bit(PAGE_EXT_PINNER_MIGRATION_FAILED, &page_ext->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 			check_longterm_pin(page_pinner, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		clear_bit(PAGE_EXT_GET, &page_ext->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		page_ext = page_ext_next(page_ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) static inline void __set_page_pinner_handle(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	struct page_ext *page_ext, depot_stack_handle_t handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	unsigned int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	struct page_pinner *page_pinner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	s64 usec = ktime_to_us(ktime_get_boottime());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	for (i = 0; i < (1 << order); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 		page_pinner = get_page_pinner(page_ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 		page_pinner->handle = handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 		page_pinner->ts_usec = usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		set_bit(PAGE_EXT_GET, &page_ext->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		atomic_inc(&page_pinner->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		page_ext = page_ext_next(page_ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) noinline void __set_page_pinner(struct page *page, unsigned int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	struct page_ext *page_ext = lookup_page_ext(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	depot_stack_handle_t handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	if (unlikely(!page_ext))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	handle = save_stack(GFP_NOWAIT|__GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	__set_page_pinner_handle(page, page_ext, handle, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) print_page_pinner(bool longterm, char __user *buf, size_t count, struct captured_pinner *record)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	unsigned long *entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	char *kbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	count = min_t(size_t, count, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	kbuf = kmalloc(count, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	if (!kbuf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	if (longterm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		ret = snprintf(kbuf, count, "Page pinned for %lld us\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 			       record->elapsed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		s64 ts_usec = record->ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		unsigned long rem_usec = do_div(ts_usec, 1000000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		ret = snprintf(kbuf, count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 			       "Page pinned ts [%5lu.%06lu]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			       (unsigned long)ts_usec, rem_usec);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	if (ret >= count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	/* Print information relevant to grouping pages by mobility */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	ret += snprintf(kbuf + ret, count - ret,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 			"PFN 0x%lx Block %lu count %d mapcount %d mapping %pS Flags %#lx(%pGp)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 			record->pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 			record->pfn >> pageblock_order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 			record->count, record->mapcount,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 			record->mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 			record->flags, &record->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 	if (ret >= count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	nr_entries = stack_depot_fetch(record->handle, &entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	ret += stack_trace_snprint(kbuf + ret, count - ret, entries,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 				   nr_entries, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	if (ret >= count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 	ret += snprintf(kbuf + ret, count - ret, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	if (ret >= count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	if (copy_to_user(buf, kbuf, ret))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 		ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	kfree(kbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	kfree(kbuf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) void __dump_page_pinner(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	struct page_ext *page_ext = lookup_page_ext(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	struct page_pinner *page_pinner;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	depot_stack_handle_t handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	unsigned long *entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	unsigned int nr_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	int pageblock_mt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	unsigned long rem_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	s64 ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	if (unlikely(!page_ext)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		pr_alert("There is not page extension available.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	page_pinner = get_page_pinner(page_ext);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	count = atomic_read(&page_pinner->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	if (!count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		pr_alert("page_pinner info is not present (never set?)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	pfn = page_to_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	ts_usec = page_pinner->ts_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	rem_usec = do_div(ts_usec, 1000000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	pr_alert("page last pinned %5lu.%06lu] count %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 		 (unsigned long)ts_usec, rem_usec, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	pageblock_mt = get_pageblock_migratetype(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	pr_alert("PFN %lu Block %lu type %s Flags %#lx(%pGp)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 			pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 			pfn >> pageblock_order,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 			migratetype_names[pageblock_mt],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 			page->flags, &page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	handle = READ_ONCE(page_pinner->handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	if (!handle) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		pr_alert("page_pinner allocation stack trace missing\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 		nr_entries = stack_depot_fetch(handle, &entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		stack_trace_print(entries, nr_entries, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) void __page_pinner_migration_failed(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	struct page_ext *page_ext = lookup_page_ext(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	struct captured_pinner record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	unsigned int idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 	if (unlikely(!page_ext))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	if (!test_bit(PAGE_EXT_PINNER_MIGRATION_FAILED, &page_ext->flags))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	record.handle = save_stack(GFP_NOWAIT|__GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 	record.ts_usec = ktime_to_us(ktime_get_boottime());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	capture_page_state(page, &record);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	spin_lock_irqsave(&acf_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 	idx = acf_pinner.index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	acf_pinner.index %= LONGTERM_PIN_BUCKETS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	acf_pinner.pinner[idx] = record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 	spin_unlock_irqrestore(&acf_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) EXPORT_SYMBOL(__page_pinner_migration_failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) void __page_pinner_mark_migration_failed_pages(struct list_head *page_list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 	struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 	struct page_ext *page_ext;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 	list_for_each_entry(page, page_list, lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 		/* The page will be freed by putback_movable_pages soon */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 		if (page_count(page) == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 		page_ext = lookup_page_ext(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 		if (unlikely(!page_ext))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 		__set_bit(PAGE_EXT_PINNER_MIGRATION_FAILED, &page_ext->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 		__page_pinner_migration_failed(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) read_longterm_page_pinner(struct file *file, char __user *buf, size_t count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 			  loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	loff_t i, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	struct captured_pinner record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) 	if (!static_branch_unlikely(&page_pinner_inited))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 	if (*ppos >= LONGTERM_PIN_BUCKETS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 	i = *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 	*ppos = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 	 * reading the records in the reverse order with newest one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 	 * being read first followed by older ones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 	idx = (lt_pinner.index - 1 - i + LONGTERM_PIN_BUCKETS) %
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	       LONGTERM_PIN_BUCKETS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 	spin_lock_irqsave(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) 	record = lt_pinner.pinner[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	spin_unlock_irqrestore(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	if (!record.handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) 	return print_page_pinner(true, buf, count, &record);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) static const struct file_operations proc_longterm_pinner_operations = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	.read		= read_longterm_page_pinner,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static ssize_t read_alloc_contig_failed(struct file *file, char __user *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) 					size_t count, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	loff_t i, idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) 	struct captured_pinner record;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	if (!static_branch_unlikely(&failure_tracking))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	if (*ppos >= LONGTERM_PIN_BUCKETS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	i = *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	*ppos = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 	 * reading the records in the reverse order with newest one
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 	 * being read first followed by older ones
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) 	idx = (acf_pinner.index - 1 - i + LONGTERM_PIN_BUCKETS) %
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 	       LONGTERM_PIN_BUCKETS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) 	spin_lock_irqsave(&acf_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	record = acf_pinner.pinner[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	spin_unlock_irqrestore(&acf_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 	if (!record.handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 	return print_page_pinner(false, buf, count, &record);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) static const struct file_operations proc_alloc_contig_failed_operations = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	.read		= read_alloc_contig_failed,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) static int pp_threshold_set(void *data, unsigned long long val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) 	threshold_usec = (s64)val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) 	spin_lock_irqsave(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) 	memset(lt_pinner.pinner, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) 	       sizeof(struct captured_pinner) * LONGTERM_PIN_BUCKETS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) 	lt_pinner.index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) 	spin_unlock_irqrestore(&lt_pinner.lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) static int pp_threshold_get(void *data, unsigned long long *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) 	*val = (unsigned long long)threshold_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) DEFINE_DEBUGFS_ATTRIBUTE(pp_threshold_fops, pp_threshold_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) 			 pp_threshold_set, "%lld\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static int failure_tracking_set(void *data, u64 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) 	bool on;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) 	on = (bool)val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 	if (on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) 		static_branch_enable(&failure_tracking);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) 		static_branch_disable(&failure_tracking);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static int failure_tracking_get(void *data, u64 *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) 	*val = static_branch_unlikely(&failure_tracking);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) DEFINE_DEBUGFS_ATTRIBUTE(failure_tracking_fops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) 			 failure_tracking_get,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) 			 failure_tracking_set, "%llu\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static int __init page_pinner_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) 	struct dentry *pp_debugfs_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) 	if (!static_branch_unlikely(&page_pinner_inited))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) 	pr_info("page_pinner enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) 	pp_debugfs_root = debugfs_create_dir("page_pinner", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) 	debugfs_create_file("longterm_pinner", 0444, pp_debugfs_root, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) 			    &proc_longterm_pinner_operations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) 	debugfs_create_file("threshold", 0644, pp_debugfs_root, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) 			    &pp_threshold_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) 	debugfs_create_file("alloc_contig_failed", 0444,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) 			    pp_debugfs_root, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) 			    &proc_alloc_contig_failed_operations);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) 	debugfs_create_file("failure_tracking", 0644,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) 			    pp_debugfs_root, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) 			    &failure_tracking_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) late_initcall(page_pinner_init)