^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Manage cache of swap slots to be used for and returned from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * swap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright(c) 2016 Intel Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Author: Tim Chen <tim.c.chen@linux.intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * We allocate the swap slots from the global pool and put
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * it into local per cpu caches. This has the advantage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * of no needing to acquire the swap_info lock every time
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * we need a new slot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * There is also opportunity to simply return the slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * to local caches without needing to acquire swap_info
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * lock. We do not reuse the returned slots directly but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * move them back to the global pool in a batch. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * allows the slots to coaellesce and reduce fragmentation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * The swap entry allocated is marked with SWAP_HAS_CACHE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * flag in map_count that prevents it from being allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * again from the global pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * The swap slots cache is protected by a mutex instead of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * a spin lock as when we search for slots with scan_swap_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * we can possibly sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/swap_slots.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <trace/hooks/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static DEFINE_PER_CPU(struct swap_slots_cache, swp_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static bool swap_slot_cache_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) bool swap_slot_cache_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) static bool swap_slot_cache_initialized;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static DEFINE_MUTEX(swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) /* Serialize swap slots cache enable/disable operations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static DEFINE_MUTEX(swap_slots_cache_enable_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void __drain_swap_slots_cache(unsigned int type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static void deactivate_swap_slots_cache(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static void reactivate_swap_slots_cache(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #define use_swap_slot_cache (swap_slot_cache_active && swap_slot_cache_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define SLOTS_CACHE 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define SLOTS_CACHE_RET 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static void deactivate_swap_slots_cache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) mutex_lock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) swap_slot_cache_active = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) trace_android_vh_swap_slot_cache_active(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) mutex_unlock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) static void reactivate_swap_slots_cache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) mutex_lock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) swap_slot_cache_active = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) trace_android_vh_swap_slot_cache_active(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) mutex_unlock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /* Must not be called with cpu hot plug lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) void disable_swap_slots_cache_lock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) mutex_lock(&swap_slots_cache_enable_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) swap_slot_cache_enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) if (swap_slot_cache_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* serialize with cpu hotplug operations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) get_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) __drain_swap_slots_cache(SLOTS_CACHE|SLOTS_CACHE_RET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) put_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static void __reenable_swap_slots_cache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) swap_slot_cache_enabled = has_usable_swap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) void reenable_swap_slots_cache_unlock(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __reenable_swap_slots_cache();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) mutex_unlock(&swap_slots_cache_enable_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) bool is_swap_slot_cache_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) return swap_slot_cache_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) EXPORT_SYMBOL_GPL(is_swap_slot_cache_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) bool check_cache_active(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) long pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) if (!swap_slot_cache_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) pages = get_nr_swap_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) if (!swap_slot_cache_active) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (pages > num_online_cpus() *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) THRESHOLD_ACTIVATE_SWAP_SLOTS_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) reactivate_swap_slots_cache();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) /* if global pool of slot caches too low, deactivate cache */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (pages < num_online_cpus() * THRESHOLD_DEACTIVATE_SWAP_SLOTS_CACHE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) deactivate_swap_slots_cache();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return swap_slot_cache_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) EXPORT_SYMBOL_GPL(check_cache_active);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static int alloc_swap_slot_cache(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) struct swap_slots_cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) swp_entry_t *slots, *slots_ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) bool skip = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * Do allocation outside swap_slots_cache_mutex
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * as kvzalloc could trigger reclaim and get_swap_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * which can lock swap_slots_cache_mutex.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) trace_android_vh_alloc_swap_slot_cache(&per_cpu(swp_slots, cpu),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) &ret, &skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) if (skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) slots = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (!slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) slots_ret = kvcalloc(SWAP_SLOTS_CACHE_SIZE, sizeof(swp_entry_t),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (!slots_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) mutex_lock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) cache = &per_cpu(swp_slots, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) if (cache->slots || cache->slots_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* cache already allocated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) mutex_unlock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) kvfree(slots_ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) if (!cache->lock_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) mutex_init(&cache->alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) spin_lock_init(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) cache->lock_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) cache->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) cache->cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) cache->n_ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * We initialized alloc_lock and free_lock earlier. We use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * !cache->slots or !cache->slots_ret to know if it is safe to acquire
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * the corresponding lock and use the cache. Memory barrier below
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * ensures the assumption.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) cache->slots = slots;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) cache->slots_ret = slots_ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) mutex_unlock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) static void drain_slots_cache_cpu(unsigned int cpu, unsigned int type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) bool free_slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) struct swap_slots_cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) swp_entry_t *slots = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) bool skip = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) cache = &per_cpu(swp_slots, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) trace_android_vh_drain_slots_cache_cpu(cache, type,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) free_slots, &skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if ((type & SLOTS_CACHE) && cache->slots) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mutex_lock(&cache->alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) swapcache_free_entries(cache->slots + cache->cur, cache->nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) cache->cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) cache->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) if (free_slots && cache->slots) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) kvfree(cache->slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) cache->slots = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) mutex_unlock(&cache->alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if ((type & SLOTS_CACHE_RET) && cache->slots_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) spin_lock_irq(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) swapcache_free_entries(cache->slots_ret, cache->n_ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) cache->n_ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (free_slots && cache->slots_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) slots = cache->slots_ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) cache->slots_ret = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) spin_unlock_irq(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (slots)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) kvfree(slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static void __drain_swap_slots_cache(unsigned int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * This function is called during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * 1) swapoff, when we have to make sure no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * left over slots are in cache when we remove
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * a swap device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * 2) disabling of swap slot cache, when we run low
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * on swap slots when allocating memory and need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * to return swap slots to global pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * We cannot acquire cpu hot plug lock here as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * this function can be invoked in the cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * hot plug path:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * cpu_up -> lock cpu_hotplug -> cpu hotplug state callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * -> memory allocation -> direct reclaim -> get_swap_page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * -> drain_swap_slots_cache
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * Hence the loop over current online cpu below could miss cpu that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * is being brought online but not yet marked as online.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) * That is okay as we do not schedule and run anything on a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * cpu before it has been marked online. Hence, we will not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * fill any swap slots in slots cache of such cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * There are no slots on such cpu that need to be drained.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) drain_slots_cache_cpu(cpu, type, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static int free_slot_cache(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) mutex_lock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) drain_slots_cache_cpu(cpu, SLOTS_CACHE | SLOTS_CACHE_RET, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) mutex_unlock(&swap_slots_cache_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) void enable_swap_slots_cache(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) mutex_lock(&swap_slots_cache_enable_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) if (!swap_slot_cache_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "swap_slots_cache",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) alloc_swap_slot_cache, free_slot_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) if (WARN_ONCE(ret < 0, "Cache allocation failed (%s), operating "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) "without swap slots cache.\n", __func__))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) goto out_unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) swap_slot_cache_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) __reenable_swap_slots_cache();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) out_unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) mutex_unlock(&swap_slots_cache_enable_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* called with swap slot cache's alloc lock held */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static int refill_swap_slots_cache(struct swap_slots_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (!use_swap_slot_cache || cache->nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) cache->cur = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) if (swap_slot_cache_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) cache->nr = get_swap_pages(SWAP_SLOTS_CACHE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) cache->slots, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) return cache->nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) int free_swap_slot(swp_entry_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) struct swap_slots_cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) bool skip = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) cache = raw_cpu_ptr(&swp_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) trace_android_vh_free_swap_slot(entry, cache, &skip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (skip)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (likely(use_swap_slot_cache && cache->slots_ret)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) spin_lock_irq(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /* Swap slots cache may be deactivated before acquiring lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (!use_swap_slot_cache || !cache->slots_ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) spin_unlock_irq(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) goto direct_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (cache->n_ret >= SWAP_SLOTS_CACHE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * Return slots to global pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * The current swap_map value is SWAP_HAS_CACHE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * Set it to 0 to indicate it is available for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * allocation in global pool
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) swapcache_free_entries(cache->slots_ret, cache->n_ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) cache->n_ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) cache->slots_ret[cache->n_ret++] = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) spin_unlock_irq(&cache->free_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) direct_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) swapcache_free_entries(&entry, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) swp_entry_t get_swap_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) swp_entry_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) struct swap_slots_cache *cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) bool found = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) entry.val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) trace_android_vh_get_swap_page(page, &entry, raw_cpu_ptr(&swp_slots), &found);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (found)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) if (PageTransHuge(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (IS_ENABLED(CONFIG_THP_SWAP))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) get_swap_pages(1, &entry, HPAGE_PMD_NR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * Preemption is allowed here, because we may sleep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * in refill_swap_slots_cache(). But it is safe, because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * accesses to the per-CPU data structure are protected by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * mutex cache->alloc_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * The alloc path here does not touch cache->slots_ret
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * so cache->free_lock is not taken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) cache = raw_cpu_ptr(&swp_slots);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (likely(check_cache_active() && cache->slots)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) mutex_lock(&cache->alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (cache->slots) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (cache->nr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) entry = cache->slots[cache->cur];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) cache->slots[cache->cur++].val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) cache->nr--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) } else if (refill_swap_slots_cache(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) mutex_unlock(&cache->alloc_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (entry.val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) get_swap_pages(1, &entry, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (mem_cgroup_try_charge_swap(page, entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) put_swap_page(page, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) entry.val = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) return entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags