^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * zsmalloc memory allocator
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 2011 Nitin Gupta
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2012, 2013 Minchan Kim
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This code is released using a dual license strategy: BSD/GPL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * You can choose the license that better fits your requirements.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Released under the terms of 3-clause BSD License
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Released under the terms of GNU General Public License Version 2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Following is how we use various fields and flags of underlying
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * struct page(s) to form a zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * Usage of struct page fields:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * page->private: points to zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * page->freelist(index): links together all component pages of a zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * For the huge page, this is always 0, so we use this field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * to store handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * page->units: first object offset in a subpage of zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * Usage of struct page flags:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * PG_private: identifies the first component page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * PG_owner_priv_1: identifies the huge component page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/magic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/shrinker.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/zsmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/zpool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/mount.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/pseudo_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/migrate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #define ZSPAGE_MAGIC 0x58
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * This must be power of 2 and greater than of equal to sizeof(link_free).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * These two conditions ensure that any 'struct link_free' itself doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * span more than 1 page which avoids complex case of mapping 2 pages simply
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * to restore link_free pointer values.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define ZS_ALIGN 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * A single 'zspage' is composed of up to 2^N discontiguous 0-order (single)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) * pages. ZS_MAX_ZSPAGE_ORDER defines upper limit on N.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define ZS_MAX_ZSPAGE_ORDER 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #define ZS_MAX_PAGES_PER_ZSPAGE (_AC(1, UL) << ZS_MAX_ZSPAGE_ORDER)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define ZS_HANDLE_SIZE (sizeof(unsigned long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) * Object location (<PFN>, <obj_idx>) is encoded as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) * a single (unsigned long) handle value.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) * Note that object index <obj_idx> starts from 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) * This is made more complicated by various memory models and PAE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #ifndef MAX_POSSIBLE_PHYSMEM_BITS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) #ifdef MAX_PHYSMEM_BITS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #define MAX_POSSIBLE_PHYSMEM_BITS MAX_PHYSMEM_BITS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * If this definition of MAX_PHYSMEM_BITS is used, OBJ_INDEX_BITS will just
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * be PAGE_SHIFT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) #define MAX_POSSIBLE_PHYSMEM_BITS BITS_PER_LONG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define _PFN_BITS (MAX_POSSIBLE_PHYSMEM_BITS - PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * Memory for allocating for handle keeps object position by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * encoding <page, obj_idx> and the encoded value has a room
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * in least bit(ie, look at obj_to_location).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * We use the bit to synchronize between object access by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * user and migration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) #define HANDLE_PIN_BIT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * Head in allocated object should have OBJ_ALLOCATED_TAG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * to identify the object was allocated or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * It's okay to add the status bit in the least bit because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) * header keeps handle which is 4byte-aligned address so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * have room for two bit at least.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) #define OBJ_ALLOCATED_TAG 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define OBJ_TAG_BITS 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define OBJ_INDEX_BITS (BITS_PER_LONG - _PFN_BITS - OBJ_TAG_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #define OBJ_INDEX_MASK ((_AC(1, UL) << OBJ_INDEX_BITS) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define FULLNESS_BITS 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #define CLASS_BITS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define ISOLATED_BITS 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) #define MAGIC_VAL_BITS 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #define MAX(a, b) ((a) >= (b) ? (a) : (b))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /* ZS_MIN_ALLOC_SIZE must be multiple of ZS_ALIGN */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define ZS_MIN_ALLOC_SIZE \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) MAX(32, (ZS_MAX_PAGES_PER_ZSPAGE << PAGE_SHIFT >> OBJ_INDEX_BITS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* each chunk includes extra space to keep handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define ZS_MAX_ALLOC_SIZE PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * On systems with 4K page size, this gives 255 size classes! There is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * trader-off here:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * - Large number of size classes is potentially wasteful as free page are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) * spread across these classes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * - Small number of size classes causes large internal fragmentation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * - Probably its better to use specific size classes (empirically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * determined). NOTE: all those class sizes must be set as multiple of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * ZS_ALIGN to make sure link_free itself never has to span 2 pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) * ZS_MIN_ALLOC_SIZE and ZS_SIZE_CLASS_DELTA must be multiple of ZS_ALIGN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * (reason above)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) #define ZS_SIZE_CLASS_DELTA (PAGE_SIZE >> CLASS_BITS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #define ZS_SIZE_CLASSES (DIV_ROUND_UP(ZS_MAX_ALLOC_SIZE - ZS_MIN_ALLOC_SIZE, \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) ZS_SIZE_CLASS_DELTA) + 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) enum fullness_group {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) ZS_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) ZS_ALMOST_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) ZS_ALMOST_FULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) ZS_FULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) NR_ZS_FULLNESS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) enum zs_stat_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) CLASS_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) CLASS_ALMOST_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) CLASS_ALMOST_FULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) CLASS_FULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) OBJ_ALLOCATED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) OBJ_USED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) NR_ZS_STAT_TYPE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) struct zs_size_stat {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned long objs[NR_ZS_STAT_TYPE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #ifdef CONFIG_ZSMALLOC_STAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static struct dentry *zs_stat_root;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static struct vfsmount *zsmalloc_mnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * We assign a page to ZS_ALMOST_EMPTY fullness group when:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * n <= N / f, where
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * n = number of allocated objects
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * N = total number of objects zspage can store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * f = fullness_threshold_frac
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * Similarly, we assign zspage to:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) * ZS_ALMOST_FULL when n > N / f
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * ZS_EMPTY when n == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * ZS_FULL when n == N
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) * (see: fix_fullness_group())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static const int fullness_threshold_frac = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) static size_t huge_class_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) struct size_class {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) struct list_head fullness_list[NR_ZS_FULLNESS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * Size of objects stored in this class. Must be multiple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * of ZS_ALIGN.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) int objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /* Number of PAGE_SIZE sized pages to combine to form a 'zspage' */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) int pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) unsigned int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct zs_size_stat stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /* huge object: pages_per_zspage == 1 && maxobj_per_zspage == 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static void SetPageHugeObject(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) SetPageOwnerPriv1(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static void ClearPageHugeObject(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) ClearPageOwnerPriv1(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static int PageHugeObject(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return PageOwnerPriv1(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * Placed within free objects to form a singly linked list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * For every zspage, zspage->freeobj gives head of this list.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * This must be power of 2 and less than or equal to ZS_ALIGN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct link_free {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) * Free object index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) * It's valid for non-allocated object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * Handle of allocated object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) unsigned long handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) struct zs_pool {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) const char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) struct size_class *size_class[ZS_SIZE_CLASSES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) struct kmem_cache *handle_cachep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct kmem_cache *zspage_cachep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) atomic_long_t pages_allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) struct zs_pool_stats stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) /* Compact classes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) struct shrinker shrinker;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) #ifdef CONFIG_ZSMALLOC_STAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) struct dentry *stat_dentry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) struct inode *inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) struct work_struct free_work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) /* A wait queue for when migration races with async_free_zspage() */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) struct wait_queue_head migration_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) atomic_long_t isolated_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) bool destroying;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct zspage {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) unsigned int fullness:FULLNESS_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) unsigned int class:CLASS_BITS + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) unsigned int isolated:ISOLATED_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned int magic:MAGIC_VAL_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) unsigned int inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) unsigned int freeobj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) struct page *first_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) struct list_head list; /* fullness list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) rwlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct mapping_area {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) char *vm_buf; /* copy buffer for objects that span pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) char *vm_addr; /* address of kmap_atomic()'ed pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) enum zs_mapmode vm_mm; /* mapping mode */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) static int zs_register_migration(struct zs_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) static void zs_unregister_migration(struct zs_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void migrate_lock_init(struct zspage *zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static void migrate_read_lock(struct zspage *zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) static void migrate_read_unlock(struct zspage *zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) static void kick_deferred_free(struct zs_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) static void init_deferred_free(struct zs_pool *pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) static int zsmalloc_mount(void) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) static void zsmalloc_unmount(void) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static int zs_register_migration(struct zs_pool *pool) { return 0; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) static void zs_unregister_migration(struct zs_pool *pool) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) static void migrate_lock_init(struct zspage *zspage) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) static void migrate_read_lock(struct zspage *zspage) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static void migrate_read_unlock(struct zspage *zspage) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) static void kick_deferred_free(struct zs_pool *pool) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void init_deferred_free(struct zs_pool *pool) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) static int create_cache(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) pool->handle_cachep = kmem_cache_create("zs_handle", ZS_HANDLE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!pool->handle_cachep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) pool->zspage_cachep = kmem_cache_create("zspage", sizeof(struct zspage),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 0, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) if (!pool->zspage_cachep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) kmem_cache_destroy(pool->handle_cachep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) pool->handle_cachep = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) static void destroy_cache(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) kmem_cache_destroy(pool->handle_cachep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) kmem_cache_destroy(pool->zspage_cachep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) static unsigned long cache_alloc_handle(struct zs_pool *pool, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) return (unsigned long)kmem_cache_alloc(pool->handle_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) gfp & ~(__GFP_HIGHMEM|__GFP_MOVABLE|__GFP_CMA));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static void cache_free_handle(struct zs_pool *pool, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) kmem_cache_free(pool->handle_cachep, (void *)handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static struct zspage *cache_alloc_zspage(struct zs_pool *pool, gfp_t flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) return kmem_cache_alloc(pool->zspage_cachep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) flags & ~(__GFP_HIGHMEM|__GFP_MOVABLE|__GFP_CMA));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) static void cache_free_zspage(struct zs_pool *pool, struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) kmem_cache_free(pool->zspage_cachep, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) static void record_obj(unsigned long handle, unsigned long obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * lsb of @obj represents handle lock while other bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * represent object value the handle is pointing so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * updating shouldn't do store tearing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) WRITE_ONCE(*(unsigned long *)handle, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) /* zpool driver */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) #ifdef CONFIG_ZPOOL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) static void *zs_zpool_create(const char *name, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) const struct zpool_ops *zpool_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) struct zpool *zpool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * Ignore global gfp flags: zs_malloc() may be invoked from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * different contexts and its caller must provide a valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * gfp mask.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) return zs_create_pool(name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) static void zs_zpool_destroy(void *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) zs_destroy_pool(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static int zs_zpool_malloc(void *pool, size_t size, gfp_t gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) unsigned long *handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) *handle = zs_malloc(pool, size, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) return *handle ? 0 : -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static void zs_zpool_free(void *pool, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) zs_free(pool, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static void *zs_zpool_map(void *pool, unsigned long handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) enum zpool_mapmode mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) enum zs_mapmode zs_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) switch (mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) case ZPOOL_MM_RO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) zs_mm = ZS_MM_RO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) case ZPOOL_MM_WO:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) zs_mm = ZS_MM_WO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) case ZPOOL_MM_RW:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) zs_mm = ZS_MM_RW;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) return zs_map_object(pool, handle, zs_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) static void zs_zpool_unmap(void *pool, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) zs_unmap_object(pool, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static u64 zs_zpool_total_size(void *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return zs_get_total_pages(pool) << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static struct zpool_driver zs_zpool_driver = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) .type = "zsmalloc",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) .create = zs_zpool_create,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) .destroy = zs_zpool_destroy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) .malloc_support_movable = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) .malloc = zs_zpool_malloc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) .free = zs_zpool_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) .map = zs_zpool_map,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .unmap = zs_zpool_unmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) .total_size = zs_zpool_total_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) MODULE_ALIAS("zpool-zsmalloc");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) #endif /* CONFIG_ZPOOL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /* per-cpu VM mapping areas for zspage accesses that cross page boundaries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) static DEFINE_PER_CPU(struct mapping_area, zs_map_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static bool is_zspage_isolated(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) return zspage->isolated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static __maybe_unused int is_first_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return PagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) /* Protected by class->lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) static inline int get_zspage_inuse(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) return zspage->inuse;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) static inline void mod_zspage_inuse(struct zspage *zspage, int val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) zspage->inuse += val;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) static inline struct page *get_first_page(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) struct page *first_page = zspage->first_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) VM_BUG_ON_PAGE(!is_first_page(first_page), first_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) return first_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) static inline int get_first_obj_offset(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return page->units;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) static inline void set_first_obj_offset(struct page *page, int offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) page->units = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static inline unsigned int get_freeobj(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) return zspage->freeobj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) static inline void set_freeobj(struct zspage *zspage, unsigned int obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) zspage->freeobj = obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) static void get_zspage_mapping(struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) unsigned int *class_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) enum fullness_group *fullness)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) BUG_ON(zspage->magic != ZSPAGE_MAGIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) *fullness = zspage->fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) *class_idx = zspage->class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static void set_zspage_mapping(struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) unsigned int class_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) enum fullness_group fullness)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) zspage->class = class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) zspage->fullness = fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * zsmalloc divides the pool into various size classes where each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * class maintains a list of zspages where each zspage is divided
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * into equal sized chunks. Each allocation falls into one of these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * classes depending on its size. This function returns index of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * size class which has chunk size big enough to hold the give size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static int get_size_class_index(int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) int idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) if (likely(size > ZS_MIN_ALLOC_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) idx = DIV_ROUND_UP(size - ZS_MIN_ALLOC_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) ZS_SIZE_CLASS_DELTA);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) return min_t(int, ZS_SIZE_CLASSES - 1, idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) /* type can be of enum type zs_stat_type or fullness_group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) static inline void zs_stat_inc(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) int type, unsigned long cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) class->stats.objs[type] += cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* type can be of enum type zs_stat_type or fullness_group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) static inline void zs_stat_dec(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) int type, unsigned long cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) class->stats.objs[type] -= cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) /* type can be of enum type zs_stat_type or fullness_group */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) static inline unsigned long zs_stat_get(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return class->stats.objs[type];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) #ifdef CONFIG_ZSMALLOC_STAT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) static void __init zs_stat_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (!debugfs_initialized()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) pr_warn("debugfs not available, stat dir not created\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) zs_stat_root = debugfs_create_dir("zsmalloc", NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) static void __exit zs_stat_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) debugfs_remove_recursive(zs_stat_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) static unsigned long zs_can_compact(struct size_class *class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static int zs_stats_size_show(struct seq_file *s, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) struct zs_pool *pool = s->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) int objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) unsigned long class_almost_full, class_almost_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) unsigned long obj_allocated, obj_used, pages_used, freeable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) unsigned long total_class_almost_full = 0, total_class_almost_empty = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) unsigned long total_objs = 0, total_used_objs = 0, total_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) unsigned long total_freeable = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) seq_printf(s, " %5s %5s %11s %12s %13s %10s %10s %16s %8s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) "class", "size", "almost_full", "almost_empty",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) "obj_allocated", "obj_used", "pages_used",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) "pages_per_zspage", "freeable");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) for (i = 0; i < ZS_SIZE_CLASSES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) class = pool->size_class[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (class->index != i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) class_almost_full = zs_stat_get(class, CLASS_ALMOST_FULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) class_almost_empty = zs_stat_get(class, CLASS_ALMOST_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) obj_used = zs_stat_get(class, OBJ_USED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) freeable = zs_can_compact(class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) objs_per_zspage = class->objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) pages_used = obj_allocated / objs_per_zspage *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) class->pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) seq_printf(s, " %5u %5u %11lu %12lu %13lu"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) " %10lu %10lu %16d %8lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) i, class->size, class_almost_full, class_almost_empty,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) obj_allocated, obj_used, pages_used,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) class->pages_per_zspage, freeable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) total_class_almost_full += class_almost_full;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) total_class_almost_empty += class_almost_empty;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) total_objs += obj_allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) total_used_objs += obj_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) total_pages += pages_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) total_freeable += freeable;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) seq_puts(s, "\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) seq_printf(s, " %5s %5s %11lu %12lu %13lu %10lu %10lu %16s %8lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) "Total", "", total_class_almost_full,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) total_class_almost_empty, total_objs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) total_used_objs, total_pages, "", total_freeable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) DEFINE_SHOW_ATTRIBUTE(zs_stats_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) static void zs_pool_stat_create(struct zs_pool *pool, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) if (!zs_stat_root) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) pr_warn("no root stat dir, not creating <%s> stat dir\n", name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) pool->stat_dentry = debugfs_create_dir(name, zs_stat_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) debugfs_create_file("classes", S_IFREG | 0444, pool->stat_dentry, pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) &zs_stats_size_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) static void zs_pool_stat_destroy(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) debugfs_remove_recursive(pool->stat_dentry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) #else /* CONFIG_ZSMALLOC_STAT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) static void __init zs_stat_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) static void __exit zs_stat_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) static inline void zs_pool_stat_create(struct zs_pool *pool, const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) static inline void zs_pool_stat_destroy(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) * For each size class, zspages are divided into different groups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) * depending on how "full" they are. This was done so that we could
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * easily find empty or nearly empty zspages when we try to shrink
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * the pool (not yet implemented). This function returns fullness
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) * status of the given page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static enum fullness_group get_fullness_group(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) int inuse, objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) inuse = get_zspage_inuse(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) objs_per_zspage = class->objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) if (inuse == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) fg = ZS_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) else if (inuse == objs_per_zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) fg = ZS_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) else if (inuse <= 3 * objs_per_zspage / fullness_threshold_frac)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) fg = ZS_ALMOST_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) fg = ZS_ALMOST_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) return fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) * Each size class maintains various freelists and zspages are assigned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) * to one of these freelists based on the number of live objects they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) * have. This functions inserts the given zspage into the freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * identified by <class, fullness_group>.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) static void insert_zspage(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) enum fullness_group fullness)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) struct zspage *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) zs_stat_inc(class, fullness, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) head = list_first_entry_or_null(&class->fullness_list[fullness],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) struct zspage, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) * We want to see more ZS_FULL pages and less almost empty/full.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * Put pages with higher ->inuse first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (get_zspage_inuse(zspage) < get_zspage_inuse(head)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) list_add(&zspage->list, &head->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) list_add(&zspage->list, &class->fullness_list[fullness]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) * This function removes the given zspage from the freelist identified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) * by <class, fullness_group>.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) static void remove_zspage(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) enum fullness_group fullness)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) VM_BUG_ON(list_empty(&class->fullness_list[fullness]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) VM_BUG_ON(is_zspage_isolated(zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) list_del_init(&zspage->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) zs_stat_dec(class, fullness, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) * Each size class maintains zspages in different fullness groups depending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * on the number of live objects they contain. When allocating or freeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * objects, the fullness status of the page can change, say, from ALMOST_FULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * to ALMOST_EMPTY when freeing an object. This function checks if such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * a status change has occurred for the given page and accordingly moves the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * page from the freelist of the old fullness group to that of the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) * fullness group.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) static enum fullness_group fix_fullness_group(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) enum fullness_group currfg, newfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) get_zspage_mapping(zspage, &class_idx, &currfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) newfg = get_fullness_group(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) if (newfg == currfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (!is_zspage_isolated(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) remove_zspage(class, zspage, currfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) insert_zspage(class, zspage, newfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) set_zspage_mapping(zspage, class_idx, newfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) return newfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) * We have to decide on how many pages to link together
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) * to form a zspage for each size class. This is important
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) * to reduce wastage due to unusable space left at end of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) * each zspage which is given as:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * wastage = Zp % class_size
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * usage = Zp - wastage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * where Zp = zspage size = k * PAGE_SIZE where k = 1, 2, ...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) * For example, for size class of 3/8 * PAGE_SIZE, we should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) * link together 3 PAGE_SIZE sized pages to form a zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * since then we can perfectly fit in 8 such objects.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) static int get_pages_per_zspage(int class_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) int i, max_usedpc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) /* zspage order which gives maximum used size per KB */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) int max_usedpc_order = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) for (i = 1; i <= ZS_MAX_PAGES_PER_ZSPAGE; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) int zspage_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) int waste, usedpc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) zspage_size = i * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) waste = zspage_size % class_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) usedpc = (zspage_size - waste) * 100 / zspage_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (usedpc > max_usedpc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) max_usedpc = usedpc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) max_usedpc_order = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) return max_usedpc_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) static struct zspage *get_zspage(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) struct zspage *zspage = (struct zspage *)page->private;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) BUG_ON(zspage->magic != ZSPAGE_MAGIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) static struct page *get_next_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) if (unlikely(PageHugeObject(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) return page->freelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * obj_to_location - get (<page>, <obj_idx>) from encoded object value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * @obj: the encoded object value
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) * @page: page object resides in zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) * @obj_idx: object index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) static void obj_to_location(unsigned long obj, struct page **page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) unsigned int *obj_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) obj >>= OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) *page = pfn_to_page(obj >> OBJ_INDEX_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) *obj_idx = (obj & OBJ_INDEX_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) * location_to_obj - get obj value encoded from (<page>, <obj_idx>)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) * @page: page object resides in zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) * @obj_idx: object index
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) static unsigned long location_to_obj(struct page *page, unsigned int obj_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) unsigned long obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) obj = page_to_pfn(page) << OBJ_INDEX_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) obj |= obj_idx & OBJ_INDEX_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) obj <<= OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) return obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) static unsigned long handle_to_obj(unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) return *(unsigned long *)handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) static unsigned long obj_to_head(struct page *page, void *obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) if (unlikely(PageHugeObject(page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) VM_BUG_ON_PAGE(!is_first_page(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) return page->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) return *(unsigned long *)obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) static inline int testpin_tag(unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) return bit_spin_is_locked(HANDLE_PIN_BIT, (unsigned long *)handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) static inline int trypin_tag(unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) return bit_spin_trylock(HANDLE_PIN_BIT, (unsigned long *)handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) static void pin_tag(unsigned long handle) __acquires(bitlock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) bit_spin_lock(HANDLE_PIN_BIT, (unsigned long *)handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) static void unpin_tag(unsigned long handle) __releases(bitlock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) bit_spin_unlock(HANDLE_PIN_BIT, (unsigned long *)handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) static void reset_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) __ClearPageMovable(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) ClearPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) set_page_private(page, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) page_mapcount_reset(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) ClearPageHugeObject(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) page->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) static int trylock_zspage(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) struct page *cursor, *fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) for (cursor = get_first_page(zspage); cursor != NULL; cursor =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) get_next_page(cursor)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (!trylock_page(cursor)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) fail = cursor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) for (cursor = get_first_page(zspage); cursor != fail; cursor =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) get_next_page(cursor))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) unlock_page(cursor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) static void __free_zspage(struct zs_pool *pool, struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) struct page *page, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) unsigned int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) get_zspage_mapping(zspage, &class_idx, &fg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) assert_spin_locked(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) VM_BUG_ON(get_zspage_inuse(zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) VM_BUG_ON(fg != ZS_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) next = page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) VM_BUG_ON_PAGE(!PageLocked(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) next = get_next_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) reset_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) dec_zone_page_state(page, NR_ZSPAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) page = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) } while (page != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) cache_free_zspage(pool, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) zs_stat_dec(class, OBJ_ALLOCATED, class->objs_per_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) atomic_long_sub(class->pages_per_zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) &pool->pages_allocated);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) static void free_zspage(struct zs_pool *pool, struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) VM_BUG_ON(get_zspage_inuse(zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) VM_BUG_ON(list_empty(&zspage->list));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) if (!trylock_zspage(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) kick_deferred_free(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) remove_zspage(class, zspage, ZS_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) __free_zspage(pool, class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) /* Initialize a newly allocated zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) static void init_zspage(struct size_class *class, struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) unsigned int freeobj = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) unsigned long off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) struct page *page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) while (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) struct page *next_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) struct link_free *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) set_first_obj_offset(page, off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) vaddr = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) link = (struct link_free *)vaddr + off / sizeof(*link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) while ((off += class->size) < PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) link->next = freeobj++ << OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) link += class->size / sizeof(*link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * We now come to the last (full or partial) object on this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) * page, which must point to the first object on the next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * page (if present)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) next_page = get_next_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (next_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) link->next = freeobj++ << OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) * Reset OBJ_TAG_BITS bit to last link to tell
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) * whether it's allocated object or not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) link->next = -1UL << OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) kunmap_atomic(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) page = next_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) off %= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) set_freeobj(zspage, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) static void create_page_chain(struct size_class *class, struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) struct page *pages[])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) struct page *prev_page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) int nr_pages = class->pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * Allocate individual pages and link them together as:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * 1. all pages are linked together using page->freelist
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * 2. each sub-page point to zspage using page->private
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * we set PG_private to identify the first page (i.e. no other sub-page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * has this flag set).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) page = pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) set_page_private(page, (unsigned long)zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) page->freelist = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (i == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) zspage->first_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) SetPagePrivate(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) if (unlikely(class->objs_per_zspage == 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) class->pages_per_zspage == 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) SetPageHugeObject(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) prev_page->freelist = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) prev_page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) * Allocate a zspage for the given size class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) static struct zspage *alloc_zspage(struct zs_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) struct zspage *zspage = cache_alloc_zspage(pool, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) if (!zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) memset(zspage, 0, sizeof(struct zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) zspage->magic = ZSPAGE_MAGIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) migrate_lock_init(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) for (i = 0; i < class->pages_per_zspage; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) page = alloc_page(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) while (--i >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) dec_zone_page_state(pages[i], NR_ZSPAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) __free_page(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) cache_free_zspage(pool, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) inc_zone_page_state(page, NR_ZSPAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) pages[i] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) create_page_chain(class, zspage, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) init_zspage(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) return zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) static struct zspage *find_get_zspage(struct size_class *class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) for (i = ZS_ALMOST_FULL; i >= ZS_EMPTY; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) zspage = list_first_entry_or_null(&class->fullness_list[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) struct zspage, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) if (zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) static inline int __zs_cpu_up(struct mapping_area *area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) * Make sure we don't leak memory if a cpu UP notification
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) * and zs_init() race and both call zs_cpu_up() on the same cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) if (area->vm_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) area->vm_buf = kmalloc(ZS_MAX_ALLOC_SIZE, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (!area->vm_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) static inline void __zs_cpu_down(struct mapping_area *area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) kfree(area->vm_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) area->vm_buf = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) static void *__zs_map_object(struct mapping_area *area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) struct page *pages[2], int off, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) int sizes[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) char *buf = area->vm_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) /* disable page faults to match kmap_atomic() return conditions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) /* no read fastpath */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (area->vm_mm == ZS_MM_WO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) sizes[0] = PAGE_SIZE - off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) sizes[1] = size - sizes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) /* copy object to per-cpu buffer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) addr = kmap_atomic(pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) memcpy(buf, addr + off, sizes[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) addr = kmap_atomic(pages[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) memcpy(buf + sizes[0], addr, sizes[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) return area->vm_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) static void __zs_unmap_object(struct mapping_area *area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct page *pages[2], int off, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) int sizes[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) char *buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) /* no write fastpath */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) if (area->vm_mm == ZS_MM_RO)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) buf = area->vm_buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) buf = buf + ZS_HANDLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) size -= ZS_HANDLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) off += ZS_HANDLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) sizes[0] = PAGE_SIZE - off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) sizes[1] = size - sizes[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) /* copy per-cpu buffer to object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) addr = kmap_atomic(pages[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) memcpy(addr + off, buf, sizes[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) addr = kmap_atomic(pages[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) memcpy(addr, buf + sizes[0], sizes[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) /* enable page faults to match kunmap_atomic() return conditions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) static int zs_cpu_prepare(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) struct mapping_area *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) area = &per_cpu(zs_map_area, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return __zs_cpu_up(area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) static int zs_cpu_dead(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) struct mapping_area *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) area = &per_cpu(zs_map_area, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) __zs_cpu_down(area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) static bool can_merge(struct size_class *prev, int pages_per_zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) int objs_per_zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) if (prev->pages_per_zspage == pages_per_zspage &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) prev->objs_per_zspage == objs_per_zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) static bool zspage_full(struct size_class *class, struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) return get_zspage_inuse(zspage) == class->objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) unsigned long zs_get_total_pages(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) return atomic_long_read(&pool->pages_allocated);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) EXPORT_SYMBOL_GPL(zs_get_total_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) * zs_map_object - get address of allocated object from handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) * @pool: pool from which the object was allocated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) * @handle: handle returned from zs_malloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) * @mm: maping mode to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) * Before using an object allocated from zs_malloc, it must be mapped using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) * this function. When done with the object, it must be unmapped using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) * zs_unmap_object.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) * Only one object can be mapped per cpu at a time. There is no protection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) * against nested mappings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) * This function returns with preemption and page faults disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) void *zs_map_object(struct zs_pool *pool, unsigned long handle,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) enum zs_mapmode mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) unsigned long obj, off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) unsigned int obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) unsigned int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) struct mapping_area *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) struct page *pages[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) void *ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) * Because we use per-cpu mapping areas shared among the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) * pools/users, we can't allow mapping in interrupt context
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) * because it can corrupt another users mappings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) BUG_ON(in_interrupt());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) /* From now on, migration cannot move the object */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) pin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) obj = handle_to_obj(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) obj_to_location(obj, &page, &obj_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) zspage = get_zspage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) /* migration cannot move any subpage in this zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) migrate_read_lock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) get_zspage_mapping(zspage, &class_idx, &fg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) off = (class->size * obj_idx) & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) area = &get_cpu_var(zs_map_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) area->vm_mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (off + class->size <= PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) /* this object is contained entirely within a page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) area->vm_addr = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) ret = area->vm_addr + off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) /* this object spans two pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) pages[0] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) pages[1] = get_next_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) BUG_ON(!pages[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) ret = __zs_map_object(area, pages, off, class->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) if (likely(!PageHugeObject(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) ret += ZS_HANDLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) EXPORT_SYMBOL_GPL(zs_map_object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) void zs_unmap_object(struct zs_pool *pool, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) unsigned long obj, off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) unsigned int obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) unsigned int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) struct mapping_area *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) obj = handle_to_obj(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) obj_to_location(obj, &page, &obj_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) zspage = get_zspage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) get_zspage_mapping(zspage, &class_idx, &fg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) off = (class->size * obj_idx) & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) area = this_cpu_ptr(&zs_map_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (off + class->size <= PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) kunmap_atomic(area->vm_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) struct page *pages[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) pages[0] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) pages[1] = get_next_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) BUG_ON(!pages[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) __zs_unmap_object(area, pages, off, class->size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) put_cpu_var(zs_map_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) migrate_read_unlock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) unpin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) EXPORT_SYMBOL_GPL(zs_unmap_object);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) * zs_huge_class_size() - Returns the size (in bytes) of the first huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) * zsmalloc &size_class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) * @pool: zsmalloc pool to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) * The function returns the size of the first huge class - any object of equal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) * or bigger size will be stored in zspage consisting of a single physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) * Context: Any context.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * Return: the size (in bytes) of the first huge zsmalloc &size_class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) size_t zs_huge_class_size(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) return huge_class_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) EXPORT_SYMBOL_GPL(zs_huge_class_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) static unsigned long obj_malloc(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) struct zspage *zspage, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) int i, nr_page, offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) unsigned long obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) struct link_free *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) struct page *m_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) unsigned long m_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) handle |= OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) obj = get_freeobj(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) offset = obj * class->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) nr_page = offset >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) m_offset = offset & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) m_page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) for (i = 0; i < nr_page; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) m_page = get_next_page(m_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) vaddr = kmap_atomic(m_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) link = (struct link_free *)vaddr + m_offset / sizeof(*link);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) set_freeobj(zspage, link->next >> OBJ_TAG_BITS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) if (likely(!PageHugeObject(m_page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) /* record handle in the header of allocated chunk */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) link->handle = handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) /* record handle to page->index */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) zspage->first_page->index = handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) kunmap_atomic(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) mod_zspage_inuse(zspage, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) zs_stat_inc(class, OBJ_USED, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) obj = location_to_obj(m_page, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) return obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) * zs_malloc - Allocate block of given size from pool.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * @pool: pool to allocate from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * @size: size of block to allocate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) * @gfp: gfp flags when allocating object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) * On success, handle to the allocated object is returned,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) * otherwise 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) * Allocation requests with size > ZS_MAX_ALLOC_SIZE will fail.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) unsigned long zs_malloc(struct zs_pool *pool, size_t size, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) unsigned long handle, obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) enum fullness_group newfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) if (unlikely(!size || size > ZS_MAX_ALLOC_SIZE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) handle = cache_alloc_handle(pool, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) if (!handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) /* extra space in chunk to keep the handle */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) size += ZS_HANDLE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) class = pool->size_class[get_size_class_index(size)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) zspage = find_get_zspage(class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) if (likely(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) obj = obj_malloc(class, zspage, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) /* Now move the zspage to another fullness group, if required */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) fix_fullness_group(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) record_obj(handle, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) return handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) zspage = alloc_zspage(pool, class, gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) if (!zspage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) cache_free_handle(pool, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) obj = obj_malloc(class, zspage, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) newfg = get_fullness_group(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) insert_zspage(class, zspage, newfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) set_zspage_mapping(zspage, class->index, newfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) record_obj(handle, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) atomic_long_add(class->pages_per_zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) &pool->pages_allocated);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) zs_stat_inc(class, OBJ_ALLOCATED, class->objs_per_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) /* We completely set up zspage so mark them as movable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) SetZsPageMovable(pool, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) return handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) EXPORT_SYMBOL_GPL(zs_malloc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) static void obj_free(struct size_class *class, unsigned long obj)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) struct link_free *link;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) struct page *f_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) unsigned long f_offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) unsigned int f_objidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) obj &= ~OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) obj_to_location(obj, &f_page, &f_objidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) f_offset = (class->size * f_objidx) & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) zspage = get_zspage(f_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) vaddr = kmap_atomic(f_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) /* Insert this object in containing zspage's freelist */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) link = (struct link_free *)(vaddr + f_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) link->next = get_freeobj(zspage) << OBJ_TAG_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) kunmap_atomic(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) set_freeobj(zspage, f_objidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) mod_zspage_inuse(zspage, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) zs_stat_dec(class, OBJ_USED, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) void zs_free(struct zs_pool *pool, unsigned long handle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) struct page *f_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) unsigned long obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) unsigned int f_objidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) enum fullness_group fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) bool isolated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) if (unlikely(!handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) pin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) obj = handle_to_obj(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) obj_to_location(obj, &f_page, &f_objidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) zspage = get_zspage(f_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) migrate_read_lock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) get_zspage_mapping(zspage, &class_idx, &fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) obj_free(class, obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) fullness = fix_fullness_group(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) if (fullness != ZS_EMPTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) migrate_read_unlock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) isolated = is_zspage_isolated(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) migrate_read_unlock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) /* If zspage is isolated, zs_page_putback will free the zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) if (likely(!isolated))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) free_zspage(pool, class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) unpin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) cache_free_handle(pool, handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) EXPORT_SYMBOL_GPL(zs_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) static void zs_object_copy(struct size_class *class, unsigned long dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) unsigned long src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) struct page *s_page, *d_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) unsigned int s_objidx, d_objidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) unsigned long s_off, d_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) void *s_addr, *d_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) int s_size, d_size, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) int written = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) s_size = d_size = class->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) obj_to_location(src, &s_page, &s_objidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) obj_to_location(dst, &d_page, &d_objidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) s_off = (class->size * s_objidx) & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) d_off = (class->size * d_objidx) & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) if (s_off + class->size > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) s_size = PAGE_SIZE - s_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) if (d_off + class->size > PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) d_size = PAGE_SIZE - d_off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) s_addr = kmap_atomic(s_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) d_addr = kmap_atomic(d_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) size = min(s_size, d_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) memcpy(d_addr + d_off, s_addr + s_off, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) written += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) if (written == class->size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) s_off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) s_size -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) d_off += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) d_size -= size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) if (s_off >= PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) kunmap_atomic(d_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) kunmap_atomic(s_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) s_page = get_next_page(s_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) s_addr = kmap_atomic(s_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) d_addr = kmap_atomic(d_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) s_size = class->size - written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) s_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) if (d_off >= PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) kunmap_atomic(d_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) d_page = get_next_page(d_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) d_addr = kmap_atomic(d_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) d_size = class->size - written;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) d_off = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) kunmap_atomic(d_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) kunmap_atomic(s_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) * Find alloced object in zspage from index object and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) * return handle.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) static unsigned long find_alloced_obj(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) struct page *page, int *obj_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) unsigned long head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) int offset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) int index = *obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) unsigned long handle = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) void *addr = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) offset = get_first_obj_offset(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) offset += class->size * index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) while (offset < PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) head = obj_to_head(page, addr + offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) if (head & OBJ_ALLOCATED_TAG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) handle = head & ~OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (trypin_tag(handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) handle = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) offset += class->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) index++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) kunmap_atomic(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) *obj_idx = index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) return handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) struct zs_compact_control {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) /* Source spage for migration which could be a subpage of zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) struct page *s_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) /* Destination page for migration which should be a first page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) * of zspage. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) struct page *d_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) /* Starting object index within @s_page which used for live object
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) * in the subpage. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) int obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) static int migrate_zspage(struct zs_pool *pool, struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) struct zs_compact_control *cc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) unsigned long used_obj, free_obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) unsigned long handle;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) struct page *s_page = cc->s_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) struct page *d_page = cc->d_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) int obj_idx = cc->obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) handle = find_alloced_obj(class, s_page, &obj_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) if (!handle) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) s_page = get_next_page(s_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) if (!s_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) obj_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) /* Stop if there is no more space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) if (zspage_full(class, get_zspage(d_page))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) unpin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) used_obj = handle_to_obj(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) free_obj = obj_malloc(class, get_zspage(d_page), handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) zs_object_copy(class, free_obj, used_obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) obj_idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) * record_obj updates handle's value to free_obj and it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) * invalidate lock bit(ie, HANDLE_PIN_BIT) of handle, which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) * breaks synchronization using pin_tag(e,g, zs_free) so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) * let's keep the lock bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) free_obj |= BIT(HANDLE_PIN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) record_obj(handle, free_obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) unpin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) obj_free(class, used_obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) /* Remember last position in this iteration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) cc->s_page = s_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) cc->obj_idx = obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) static struct zspage *isolate_zspage(struct size_class *class, bool source)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) enum fullness_group fg[2] = {ZS_ALMOST_EMPTY, ZS_ALMOST_FULL};
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) if (!source) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) fg[0] = ZS_ALMOST_FULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) fg[1] = ZS_ALMOST_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) for (i = 0; i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) zspage = list_first_entry_or_null(&class->fullness_list[fg[i]],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) struct zspage, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) if (zspage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) VM_BUG_ON(is_zspage_isolated(zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) remove_zspage(class, zspage, fg[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716) return zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) return zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) * putback_zspage - add @zspage into right class's fullness list
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) * @class: destination class
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * @zspage: target page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) * Return @zspage's fullness_group
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) static enum fullness_group putback_zspage(struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) enum fullness_group fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) VM_BUG_ON(is_zspage_isolated(zspage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) fullness = get_fullness_group(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) insert_zspage(class, zspage, fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) set_zspage_mapping(zspage, class->index, fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) return fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) * To prevent zspage destroy during migration, zspage freeing should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) * hold locks of all pages in the zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) static void lock_zspage(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) struct page *page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) lock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) } while ((page = get_next_page(page)) != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) static int zs_init_fs_context(struct fs_context *fc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) return init_pseudo(fc, ZSMALLOC_MAGIC) ? 0 : -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) static struct file_system_type zsmalloc_fs = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) .name = "zsmalloc",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) .init_fs_context = zs_init_fs_context,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) .kill_sb = kill_anon_super,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) static int zsmalloc_mount(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) zsmalloc_mnt = kern_mount(&zsmalloc_fs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) if (IS_ERR(zsmalloc_mnt))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) ret = PTR_ERR(zsmalloc_mnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) static void zsmalloc_unmount(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) kern_unmount(zsmalloc_mnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) static void migrate_lock_init(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) rwlock_init(&zspage->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) static void migrate_read_lock(struct zspage *zspage) __acquires(&zspage->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) read_lock(&zspage->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) static void migrate_read_unlock(struct zspage *zspage) __releases(&zspage->lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) read_unlock(&zspage->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) static void migrate_write_lock(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) write_lock(&zspage->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805) static void migrate_write_unlock(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) write_unlock(&zspage->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) /* Number of isolated subpage for *page migration* in this zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) static void inc_zspage_isolation(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) zspage->isolated++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) static void dec_zspage_isolation(struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) zspage->isolated--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) static void putback_zspage_deferred(struct zs_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) struct size_class *class,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) fg = putback_zspage(class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) if (fg == ZS_EMPTY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) schedule_work(&pool->free_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) static inline void zs_pool_dec_isolated(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) VM_BUG_ON(atomic_long_read(&pool->isolated_pages) <= 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) atomic_long_dec(&pool->isolated_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) * Checking pool->destroying must happen after atomic_long_dec()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) * for pool->isolated_pages above. Paired with the smp_mb() in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) * zs_unregister_migration().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) smp_mb__after_atomic();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) if (atomic_long_read(&pool->isolated_pages) == 0 && pool->destroying)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) wake_up_all(&pool->migration_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) static void replace_sub_page(struct size_class *class, struct zspage *zspage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) struct page *newpage, struct page *oldpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) struct page *pages[ZS_MAX_PAGES_PER_ZSPAGE] = {NULL, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) int idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) if (page == oldpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) pages[idx] = newpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) pages[idx] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) idx++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) } while ((page = get_next_page(page)) != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) create_page_chain(class, zspage, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) set_first_obj_offset(newpage, get_first_obj_offset(oldpage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) if (unlikely(PageHugeObject(oldpage)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) newpage->index = oldpage->index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) __SetPageMovable(newpage, page_mapping(oldpage));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) static bool zs_page_isolate(struct page *page, isolate_mode_t mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) struct zs_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) enum fullness_group fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) struct address_space *mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) * Page is locked so zspage couldn't be destroyed. For detail, look at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) * lock_zspage in free_zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) VM_BUG_ON_PAGE(!PageMovable(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) VM_BUG_ON_PAGE(PageIsolated(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) zspage = get_zspage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889) * Without class lock, fullness could be stale while class_idx is okay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) * because class_idx is constant unless page is freed so we should get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) * fullness again under class lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) get_zspage_mapping(zspage, &class_idx, &fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) pool = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) if (get_zspage_inuse(zspage) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) /* zspage is isolated for object migration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) if (list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) * If this is first time isolation for the zspage, isolate zspage from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) * size_class to prevent further object allocation from the zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) if (!list_empty(&zspage->list) && !is_zspage_isolated(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) get_zspage_mapping(zspage, &class_idx, &fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) atomic_long_inc(&pool->isolated_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) remove_zspage(class, zspage, fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) inc_zspage_isolation(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) static int zs_page_migrate(struct address_space *mapping, struct page *newpage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) struct page *page, enum migrate_mode mode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) struct zs_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) enum fullness_group fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) struct page *dummy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) void *s_addr, *d_addr, *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) int offset, pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) unsigned long handle, head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) unsigned long old_obj, new_obj;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) unsigned int obj_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) int ret = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) * We cannot support the _NO_COPY case here, because copy needs to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) * happen under the zs lock, which does not work with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) * MIGRATE_SYNC_NO_COPY workflow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) if (mode == MIGRATE_SYNC_NO_COPY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) VM_BUG_ON_PAGE(!PageMovable(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) VM_BUG_ON_PAGE(!PageIsolated(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) zspage = get_zspage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) /* Concurrent compactor cannot migrate any subpage in zspage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) migrate_write_lock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) get_zspage_mapping(zspage, &class_idx, &fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) pool = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) offset = get_first_obj_offset(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) if (!get_zspage_inuse(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) * Set "offset" to end of the page so that every loops
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) * skips unnecessary object scanning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) offset = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) pos = offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) s_addr = kmap_atomic(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973) while (pos < PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) head = obj_to_head(page, s_addr + pos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) if (head & OBJ_ALLOCATED_TAG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) handle = head & ~OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) if (!trypin_tag(handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) goto unpin_objects;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) pos += class->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) * Here, any user cannot access all objects in the zspage so let's move.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) d_addr = kmap_atomic(newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) memcpy(d_addr, s_addr, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) kunmap_atomic(d_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) for (addr = s_addr + offset; addr < s_addr + pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) addr += class->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) head = obj_to_head(page, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) if (head & OBJ_ALLOCATED_TAG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994) handle = head & ~OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) if (!testpin_tag(handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) old_obj = handle_to_obj(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) obj_to_location(old_obj, &dummy, &obj_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) new_obj = (unsigned long)location_to_obj(newpage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) obj_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) new_obj |= BIT(HANDLE_PIN_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) record_obj(handle, new_obj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) replace_sub_page(class, zspage, newpage, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) get_page(newpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010) dec_zspage_isolation(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013) * Page migration is done so let's putback isolated zspage to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) * the list if @page is final isolated subpage in the zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) if (!is_zspage_isolated(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) * We cannot race with zs_destroy_pool() here because we wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) * for isolation to hit zero before we start destroying.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) * Also, we ensure that everyone can see pool->destroying before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) * we start waiting.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) putback_zspage_deferred(pool, class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) zs_pool_dec_isolated(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) if (page_zone(newpage) != page_zone(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) dec_zone_page_state(page, NR_ZSPAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) inc_zone_page_state(newpage, NR_ZSPAGES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) reset_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034) page = newpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) ret = MIGRATEPAGE_SUCCESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) unpin_objects:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) for (addr = s_addr + offset; addr < s_addr + pos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) addr += class->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) head = obj_to_head(page, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) if (head & OBJ_ALLOCATED_TAG) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) handle = head & ~OBJ_ALLOCATED_TAG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) if (!testpin_tag(handle))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) unpin_tag(handle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) kunmap_atomic(s_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) migrate_write_unlock(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) static void zs_page_putback(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) struct zs_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) enum fullness_group fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) struct address_space *mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) struct zspage *zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) VM_BUG_ON_PAGE(!PageMovable(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) VM_BUG_ON_PAGE(!PageIsolated(page), page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) zspage = get_zspage(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) get_zspage_mapping(zspage, &class_idx, &fg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) mapping = page_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) pool = mapping->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) dec_zspage_isolation(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) if (!is_zspage_isolated(zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) * Due to page_lock, we cannot free zspage immediately
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) * so let's defer.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) putback_zspage_deferred(pool, class, zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) zs_pool_dec_isolated(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) static const struct address_space_operations zsmalloc_aops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) .isolate_page = zs_page_isolate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) .migratepage = zs_page_migrate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) .putback_page = zs_page_putback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) static int zs_register_migration(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) pool->inode = alloc_anon_inode(zsmalloc_mnt->mnt_sb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) if (IS_ERR(pool->inode)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) pool->inode = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) pool->inode->i_mapping->private_data = pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) pool->inode->i_mapping->a_ops = &zsmalloc_aops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) static bool pool_isolated_are_drained(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) return atomic_long_read(&pool->isolated_pages) == 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) /* Function for resolving migration */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) static void wait_for_isolated_drain(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115) * We're in the process of destroying the pool, so there are no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) * active allocations. zs_page_isolate() fails for completely free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) * zspages, so we need only wait for the zs_pool's isolated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) * count to hit zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) wait_event(pool->migration_wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) pool_isolated_are_drained(pool));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124) static void zs_unregister_migration(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) pool->destroying = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) * We need a memory barrier here to ensure global visibility of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) * pool->destroying. Thus pool->isolated pages will either be 0 in which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) * case we don't care, or it will be > 0 and pool->destroying will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) * ensure that we wake up once isolation hits 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) smp_mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) wait_for_isolated_drain(pool); /* This can block */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) flush_work(&pool->free_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) iput(pool->inode);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) * Caller should hold page_lock of all pages in the zspage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) * In here, we cannot use zspage meta data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) static void async_free_zspage(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) unsigned int class_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) enum fullness_group fullness;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) struct zspage *zspage, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) LIST_HEAD(free_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) struct zs_pool *pool = container_of(work, struct zs_pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) free_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) for (i = 0; i < ZS_SIZE_CLASSES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) class = pool->size_class[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) if (class->index != i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) list_splice_init(&class->fullness_list[ZS_EMPTY], &free_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) list_for_each_entry_safe(zspage, tmp, &free_pages, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) list_del(&zspage->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) lock_zspage(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) get_zspage_mapping(zspage, &class_idx, &fullness);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) VM_BUG_ON(fullness != ZS_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) class = pool->size_class[class_idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) __free_zspage(pool, pool->size_class[class_idx], zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) static void kick_deferred_free(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) schedule_work(&pool->free_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) static void init_deferred_free(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) INIT_WORK(&pool->free_work, async_free_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) static void SetZsPageMovable(struct zs_pool *pool, struct zspage *zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) struct page *page = get_first_page(zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) WARN_ON(!trylock_page(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) __SetPageMovable(page, pool->inode->i_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) unlock_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) } while ((page = get_next_page(page)) != NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) * Based on the number of unused allocated objects calculate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) * and return the number of pages that we can free.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) static unsigned long zs_can_compact(struct size_class *class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) unsigned long obj_wasted;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) unsigned long obj_allocated = zs_stat_get(class, OBJ_ALLOCATED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) unsigned long obj_used = zs_stat_get(class, OBJ_USED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) if (obj_allocated <= obj_used)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) obj_wasted = obj_allocated - obj_used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) obj_wasted /= class->objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) return obj_wasted * class->pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) static unsigned long __zs_compact(struct zs_pool *pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) struct size_class *class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) struct zs_compact_control cc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) struct zspage *src_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225) struct zspage *dst_zspage = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) unsigned long pages_freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) while ((src_zspage = isolate_zspage(class, true))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) if (!zs_can_compact(class))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) cc.obj_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) cc.s_page = get_first_page(src_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) while ((dst_zspage = isolate_zspage(class, false))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) cc.d_page = get_first_page(dst_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) * If there is no more space in dst_page, resched
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) * and see if anyone had allocated another zspage.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) if (!migrate_zspage(pool, class, &cc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) putback_zspage(class, dst_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) /* Stop if we couldn't find slot */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (dst_zspage == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) putback_zspage(class, dst_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) if (putback_zspage(class, src_zspage) == ZS_EMPTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255) free_zspage(pool, class, src_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) pages_freed += class->pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) spin_lock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) if (src_zspage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) putback_zspage(class, src_zspage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) spin_unlock(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) return pages_freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) unsigned long zs_compact(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) unsigned long pages_freed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) class = pool->size_class[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) if (!class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) if (class->index != i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) pages_freed += __zs_compact(pool, class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) atomic_long_add(pages_freed, &pool->stats.pages_compacted);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) return pages_freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) EXPORT_SYMBOL_GPL(zs_compact);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) void zs_pool_stats(struct zs_pool *pool, struct zs_pool_stats *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) memcpy(stats, &pool->stats, sizeof(struct zs_pool_stats));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) EXPORT_SYMBOL_GPL(zs_pool_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) static unsigned long zs_shrinker_scan(struct shrinker *shrinker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) unsigned long pages_freed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) struct zs_pool *pool = container_of(shrinker, struct zs_pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302) shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) * Compact classes and calculate compaction delta.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) * Can run concurrently with a manually triggered
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) * (by user) compaction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309) pages_freed = zs_compact(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) return pages_freed ? pages_freed : SHRINK_STOP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) static unsigned long zs_shrinker_count(struct shrinker *shrinker,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) struct shrink_control *sc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) unsigned long pages_to_free = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) struct zs_pool *pool = container_of(shrinker, struct zs_pool,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) class = pool->size_class[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) if (!class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) if (class->index != i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) pages_to_free += zs_can_compact(class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) return pages_to_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) static void zs_unregister_shrinker(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) unregister_shrinker(&pool->shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) static int zs_register_shrinker(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) pool->shrinker.scan_objects = zs_shrinker_scan;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) pool->shrinker.count_objects = zs_shrinker_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) pool->shrinker.batch = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) pool->shrinker.seeks = DEFAULT_SEEKS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) return register_shrinker(&pool->shrinker);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) * zs_create_pool - Creates an allocation pool to work from.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) * @name: pool name to be created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) * This function must be called before anything when using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) * the zsmalloc allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) * On success, a pointer to the newly created pool is returned,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) * otherwise NULL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) struct zs_pool *zs_create_pool(const char *name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) struct zs_pool *pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) struct size_class *prev_class = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) pool = kzalloc(sizeof(*pool), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (!pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) init_deferred_free(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) pool->name = kstrdup(name, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) if (!pool->name)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) #ifdef CONFIG_COMPACTION
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) init_waitqueue_head(&pool->migration_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) if (create_cache(pool))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) * Iterate reversely, because, size of size_class that we want to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386) * for merging should be larger or equal to current size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) for (i = ZS_SIZE_CLASSES - 1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) int size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) int pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) int objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) struct size_class *class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) int fullness = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) size = ZS_MIN_ALLOC_SIZE + i * ZS_SIZE_CLASS_DELTA;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) if (size > ZS_MAX_ALLOC_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397) size = ZS_MAX_ALLOC_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) pages_per_zspage = get_pages_per_zspage(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) objs_per_zspage = pages_per_zspage * PAGE_SIZE / size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) * We iterate from biggest down to smallest classes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) * so huge_class_size holds the size of the first huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) * class. Any object bigger than or equal to that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) * endup in the huge class.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) if (pages_per_zspage != 1 && objs_per_zspage != 1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) !huge_class_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) huge_class_size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) * The object uses ZS_HANDLE_SIZE bytes to store the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) * handle. We need to subtract it, because zs_malloc()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) * unconditionally adds handle size before it performs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) * size class search - so object may be smaller than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) * huge class size, yet it still can end up in the huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) * class because it grows by ZS_HANDLE_SIZE extra bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) * right before class lookup.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) huge_class_size -= (ZS_HANDLE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423) * size_class is used for normal zsmalloc operation such
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) * as alloc/free for that size. Although it is natural that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) * have one size_class for each size, there is a chance that we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) * can get more memory utilization if we use one size_class for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) * many different sizes whose size_class have same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) * characteristics. So, we makes size_class point to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) * previous size_class if possible.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) if (prev_class) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) if (can_merge(prev_class, pages_per_zspage, objs_per_zspage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) pool->size_class[i] = prev_class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438) class = kzalloc(sizeof(struct size_class), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) if (!class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) class->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443) class->index = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) class->pages_per_zspage = pages_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) class->objs_per_zspage = objs_per_zspage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) spin_lock_init(&class->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) pool->size_class[i] = class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) for (fullness = ZS_EMPTY; fullness < NR_ZS_FULLNESS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449) fullness++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) INIT_LIST_HEAD(&class->fullness_list[fullness]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) prev_class = class;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) /* debug only, don't abort if it fails */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) zs_pool_stat_create(pool, name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) if (zs_register_migration(pool))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) * Not critical since shrinker is only used to trigger internal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) * defragmentation of the pool which is pretty optional thing. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) * registration fails we still can use the pool normally and user can
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) * trigger compaction manually. Thus, ignore return code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467) zs_register_shrinker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) return pool;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) zs_destroy_pool(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) EXPORT_SYMBOL_GPL(zs_create_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) void zs_destroy_pool(struct zs_pool *pool)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) zs_unregister_shrinker(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) zs_unregister_migration(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) zs_pool_stat_destroy(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) for (i = 0; i < ZS_SIZE_CLASSES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) int fg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) struct size_class *class = pool->size_class[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) if (!class)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) if (class->index != i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) for (fg = ZS_EMPTY; fg < NR_ZS_FULLNESS; fg++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) if (!list_empty(&class->fullness_list[fg])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) pr_info("Freeing non-empty class with size %db, fullness group %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498) class->size, fg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) kfree(class);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) destroy_cache(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) kfree(pool->name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506) kfree(pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) EXPORT_SYMBOL_GPL(zs_destroy_pool);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) static int __init zs_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) ret = zsmalloc_mount();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) ret = cpuhp_setup_state(CPUHP_MM_ZS_PREPARE, "mm/zsmalloc:prepare",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519) zs_cpu_prepare, zs_cpu_dead);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) goto hp_setup_fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) #ifdef CONFIG_ZPOOL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) zpool_register_driver(&zs_zpool_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) zs_stat_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) hp_setup_fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) zsmalloc_unmount();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) static void __exit zs_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) #ifdef CONFIG_ZPOOL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) zpool_unregister_driver(&zs_zpool_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542) zsmalloc_unmount();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) cpuhp_remove_state(CPUHP_MM_ZS_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) zs_stat_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) module_init(zs_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) module_exit(zs_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) MODULE_LICENSE("Dual BSD/GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) MODULE_AUTHOR("Nitin Gupta <ngupta@vflare.org>");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags