^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * KASAN quarantine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Author: Alexander Potapenko <glider@google.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2016 Google, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Based on code by Dmitry Chernenkov.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/printk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/shrinker.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/srcu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/cpuhotplug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include "../slab.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include "kasan.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* Data structure and operations for quarantine queues. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * Each queue is a signle-linked list, which also stores the total size of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * objects inside of it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct qlist_head {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) struct qlist_node *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct qlist_node *tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) size_t bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) bool offline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define QLIST_INIT { NULL, NULL, 0 }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static bool qlist_empty(struct qlist_head *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return !q->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static void qlist_init(struct qlist_head *q)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) q->head = q->tail = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) q->bytes = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static void qlist_put(struct qlist_head *q, struct qlist_node *qlink,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (unlikely(qlist_empty(q)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) q->head = qlink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) q->tail->next = qlink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) q->tail = qlink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) qlink->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) q->bytes += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) static void qlist_move_all(struct qlist_head *from, struct qlist_head *to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) if (unlikely(qlist_empty(from)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (qlist_empty(to)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) *to = *from;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) qlist_init(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) to->tail->next = from->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) to->tail = from->tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) to->bytes += from->bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) qlist_init(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) #define QUARANTINE_PERCPU_SIZE (1 << 20)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #define QUARANTINE_BATCHES \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) (1024 > 4 * CONFIG_NR_CPUS ? 1024 : 4 * CONFIG_NR_CPUS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) * The object quarantine consists of per-cpu queues and a global queue,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) * guarded by quarantine_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static DEFINE_PER_CPU(struct qlist_head, cpu_quarantine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /* Round-robin FIFO array of batches. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static struct qlist_head global_quarantine[QUARANTINE_BATCHES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static int quarantine_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static int quarantine_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* Total size of all objects in global_quarantine across all batches. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static unsigned long quarantine_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static DEFINE_RAW_SPINLOCK(quarantine_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) DEFINE_STATIC_SRCU(remove_cache_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /* Maximum size of the global queue. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) static unsigned long quarantine_max_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Target size of a batch in global_quarantine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * Usually equal to QUARANTINE_PERCPU_SIZE unless we have too much RAM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static unsigned long quarantine_batch_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * The fraction of physical memory the quarantine is allowed to occupy.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * Quarantine doesn't support memory shrinker with SLAB allocator, so we keep
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * the ratio low to avoid OOM.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) #define QUARANTINE_FRACTION 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static struct kmem_cache *qlink_to_cache(struct qlist_node *qlink)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) return virt_to_head_page(qlink)->slab_cache;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) static void *qlink_to_object(struct qlist_node *qlink, struct kmem_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) struct kasan_free_meta *free_info =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) container_of(qlink, struct kasan_free_meta,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) quarantine_link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) return ((void *)free_info) - cache->kasan_info.free_meta_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) void *object = qlink_to_object(qlink, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (IS_ENABLED(CONFIG_SLAB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * As the object now gets freed from the quaratine, assume that its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * free track is no longer valid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) *(u8 *)kasan_mem_to_shadow(object) = KASAN_KMALLOC_FREE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) ___cache_free(cache, object, _THIS_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) if (IS_ENABLED(CONFIG_SLAB))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static void qlist_free_all(struct qlist_head *q, struct kmem_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) struct qlist_node *qlink;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) if (unlikely(qlist_empty(q)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) qlink = q->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) while (qlink) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) struct kmem_cache *obj_cache =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) cache ? cache : qlink_to_cache(qlink);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) struct qlist_node *next = qlink->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) qlink_free(qlink, obj_cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) qlink = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) qlist_init(q);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) bool kasan_quarantine_put(struct kmem_cache *cache, void *object)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) struct qlist_head *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) struct qlist_head temp = QLIST_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) struct kasan_free_meta *meta = kasan_get_free_meta(cache, object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * If there's no metadata for this object, don't put it into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * quarantine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (!meta)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * Note: irq must be disabled until after we move the batch to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * global quarantine. Otherwise kasan_quarantine_remove_cache() can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * miss some objects belonging to the cache if they are in our local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * temp list. kasan_quarantine_remove_cache() executes on_each_cpu()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * at the beginning which ensures that it either sees the objects in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * per-cpu lists or in the global quarantine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) q = this_cpu_ptr(&cpu_quarantine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) if (q->offline) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) qlist_put(q, &meta->quarantine_link, cache->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) qlist_move_all(q, &temp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) raw_spin_lock(&quarantine_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) WRITE_ONCE(quarantine_size, quarantine_size + temp.bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) qlist_move_all(&temp, &global_quarantine[quarantine_tail]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (global_quarantine[quarantine_tail].bytes >=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) READ_ONCE(quarantine_batch_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) int new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) new_tail = quarantine_tail + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) if (new_tail == QUARANTINE_BATCHES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) new_tail = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (new_tail != quarantine_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) quarantine_tail = new_tail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) raw_spin_unlock(&quarantine_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) void kasan_quarantine_reduce(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) size_t total_size, new_quarantine_size, percpu_quarantines;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) int srcu_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) struct qlist_head to_free = QLIST_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (likely(READ_ONCE(quarantine_size) <=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) READ_ONCE(quarantine_max_size)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * srcu critical section ensures that kasan_quarantine_remove_cache()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * will not miss objects belonging to the cache while they are in our
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * local to_free list. srcu is chosen because (1) it gives us private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * grace period domain that does not interfere with anything else,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * and (2) it allows synchronize_srcu() to return without waiting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * if there are no pending read critical sections (which is the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) * expected case).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) srcu_idx = srcu_read_lock(&remove_cache_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) raw_spin_lock_irqsave(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * Update quarantine size in case of hotplug. Allocate a fraction of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * the installed memory to quarantine minus per-cpu queue limits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) total_size = (totalram_pages() << PAGE_SHIFT) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) QUARANTINE_FRACTION;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) percpu_quarantines = QUARANTINE_PERCPU_SIZE * num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) new_quarantine_size = (total_size < percpu_quarantines) ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 0 : total_size - percpu_quarantines;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) WRITE_ONCE(quarantine_max_size, new_quarantine_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /* Aim at consuming at most 1/2 of slots in quarantine. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) WRITE_ONCE(quarantine_batch_size, max((size_t)QUARANTINE_PERCPU_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 2 * total_size / QUARANTINE_BATCHES));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) if (likely(quarantine_size > quarantine_max_size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) qlist_move_all(&global_quarantine[quarantine_head], &to_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) WRITE_ONCE(quarantine_size, quarantine_size - to_free.bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) quarantine_head++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) if (quarantine_head == QUARANTINE_BATCHES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) quarantine_head = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) raw_spin_unlock_irqrestore(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) qlist_free_all(&to_free, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) srcu_read_unlock(&remove_cache_srcu, srcu_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void qlist_move_cache(struct qlist_head *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) struct qlist_head *to,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) struct kmem_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct qlist_node *curr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (unlikely(qlist_empty(from)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) curr = from->head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) qlist_init(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) while (curr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct qlist_node *next = curr->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct kmem_cache *obj_cache = qlink_to_cache(curr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (obj_cache == cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) qlist_put(to, curr, obj_cache->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) qlist_put(from, curr, obj_cache->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) curr = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static void per_cpu_remove_cache(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct kmem_cache *cache = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) struct qlist_head to_free = QLIST_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) struct qlist_head *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) q = this_cpu_ptr(&cpu_quarantine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) qlist_move_cache(q, &to_free, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) qlist_free_all(&to_free, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /* Free all quarantined objects belonging to cache. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) void kasan_quarantine_remove_cache(struct kmem_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) unsigned long flags, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) struct qlist_head to_free = QLIST_INIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * Must be careful to not miss any objects that are being moved from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) * per-cpu list to the global quarantine in kasan_quarantine_put(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * nor objects being freed in kasan_quarantine_reduce(). on_each_cpu()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * achieves the first goal, while synchronize_srcu() achieves the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * second.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) on_each_cpu(per_cpu_remove_cache, cache, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) raw_spin_lock_irqsave(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) for (i = 0; i < QUARANTINE_BATCHES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (qlist_empty(&global_quarantine[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) qlist_move_cache(&global_quarantine[i], &to_free, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /* Scanning whole quarantine can take a while. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) raw_spin_unlock_irqrestore(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) raw_spin_lock_irqsave(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) raw_spin_unlock_irqrestore(&quarantine_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) qlist_free_all(&to_free, cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) synchronize_srcu(&remove_cache_srcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) static int kasan_cpu_online(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) this_cpu_ptr(&cpu_quarantine)->offline = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static int kasan_cpu_offline(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) struct qlist_head *q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) q = this_cpu_ptr(&cpu_quarantine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) /* Ensure the ordering between the writing to q->offline and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * qlist_free_all. Otherwise, cpu_quarantine may be corrupted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * by interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) WRITE_ONCE(q->offline, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) qlist_free_all(q, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static int __init kasan_cpu_quarantine_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) ret = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "mm/kasan:online",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) kasan_cpu_online, kasan_cpu_offline);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) pr_err("kasan cpu quarantine register failed [%d]\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) late_initcall(kasan_cpu_quarantine_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags