^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Test module to generate lockups
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/sched/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static unsigned int time_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) module_param(time_secs, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) MODULE_PARM_DESC(time_secs, "lockup time in seconds, default 0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static unsigned int time_nsecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) module_param(time_nsecs, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) MODULE_PARM_DESC(time_nsecs, "nanoseconds part of lockup time, default 0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) static unsigned int cooldown_secs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) module_param(cooldown_secs, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) MODULE_PARM_DESC(cooldown_secs, "cooldown time between iterations in seconds, default 0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) static unsigned int cooldown_nsecs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) module_param(cooldown_nsecs, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) MODULE_PARM_DESC(cooldown_nsecs, "nanoseconds part of cooldown, default 0");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static unsigned int iterations = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) module_param(iterations, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) MODULE_PARM_DESC(iterations, "lockup iterations, default 1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static bool all_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) module_param(all_cpus, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) MODULE_PARM_DESC(all_cpus, "trigger lockup at all cpus at once");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static int wait_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static char *state = "R";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) module_param(state, charp, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) MODULE_PARM_DESC(state, "wait in 'R' running (default), 'D' uninterruptible, 'K' killable, 'S' interruptible state");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static bool use_hrtimer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) module_param(use_hrtimer, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) MODULE_PARM_DESC(use_hrtimer, "use high-resolution timer for sleeping");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) static bool iowait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) module_param(iowait, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) MODULE_PARM_DESC(iowait, "account sleep time as iowait");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) static bool lock_read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) module_param(lock_read, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) MODULE_PARM_DESC(lock_read, "lock read-write locks for read");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static bool lock_single;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) module_param(lock_single, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) MODULE_PARM_DESC(lock_single, "acquire locks only at one cpu");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static bool reacquire_locks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) module_param(reacquire_locks, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) MODULE_PARM_DESC(reacquire_locks, "release and reacquire locks/irq/preempt between iterations");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static bool touch_softlockup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) module_param(touch_softlockup, bool, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) MODULE_PARM_DESC(touch_softlockup, "touch soft-lockup watchdog between iterations");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) static bool touch_hardlockup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) module_param(touch_hardlockup, bool, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) MODULE_PARM_DESC(touch_hardlockup, "touch hard-lockup watchdog between iterations");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static bool call_cond_resched;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) module_param(call_cond_resched, bool, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) MODULE_PARM_DESC(call_cond_resched, "call cond_resched() between iterations");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static bool measure_lock_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) module_param(measure_lock_wait, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) MODULE_PARM_DESC(measure_lock_wait, "measure lock wait time");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static unsigned long lock_wait_threshold = ULONG_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) module_param(lock_wait_threshold, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) MODULE_PARM_DESC(lock_wait_threshold, "print lock wait time longer than this in nanoseconds, default off");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static bool test_disable_irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) module_param_named(disable_irq, test_disable_irq, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) MODULE_PARM_DESC(disable_irq, "disable interrupts: generate hard-lockups");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static bool disable_softirq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) module_param(disable_softirq, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) MODULE_PARM_DESC(disable_softirq, "disable bottom-half irq handlers");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static bool disable_preempt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) module_param(disable_preempt, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) MODULE_PARM_DESC(disable_preempt, "disable preemption: generate soft-lockups");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static bool lock_rcu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) module_param(lock_rcu, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) MODULE_PARM_DESC(lock_rcu, "grab rcu_read_lock: generate rcu stalls");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) static bool lock_mmap_sem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) module_param(lock_mmap_sem, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) MODULE_PARM_DESC(lock_mmap_sem, "lock mm->mmap_lock: block procfs interfaces");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) static unsigned long lock_rwsem_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) module_param_unsafe(lock_rwsem_ptr, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) MODULE_PARM_DESC(lock_rwsem_ptr, "lock rw_semaphore at address");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static unsigned long lock_mutex_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) module_param_unsafe(lock_mutex_ptr, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) MODULE_PARM_DESC(lock_mutex_ptr, "lock mutex at address");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static unsigned long lock_spinlock_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) module_param_unsafe(lock_spinlock_ptr, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) MODULE_PARM_DESC(lock_spinlock_ptr, "lock spinlock at address");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static unsigned long lock_rwlock_ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) module_param_unsafe(lock_rwlock_ptr, ulong, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) MODULE_PARM_DESC(lock_rwlock_ptr, "lock rwlock at address");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) static unsigned int alloc_pages_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) module_param_unsafe(alloc_pages_nr, uint, 0600);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) MODULE_PARM_DESC(alloc_pages_nr, "allocate and free pages under locks");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) static unsigned int alloc_pages_order;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) module_param(alloc_pages_order, uint, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) MODULE_PARM_DESC(alloc_pages_order, "page order to allocate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) static gfp_t alloc_pages_gfp = GFP_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) module_param_unsafe(alloc_pages_gfp, uint, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) MODULE_PARM_DESC(alloc_pages_gfp, "allocate pages with this gfp_mask, default GFP_KERNEL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) static bool alloc_pages_atomic;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) module_param(alloc_pages_atomic, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) MODULE_PARM_DESC(alloc_pages_atomic, "allocate pages with GFP_ATOMIC");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static bool reallocate_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) module_param(reallocate_pages, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) MODULE_PARM_DESC(reallocate_pages, "free and allocate pages between iterations");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) struct file *test_file;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static struct inode *test_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) static char test_file_path[256];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) module_param_string(file_path, test_file_path, sizeof(test_file_path), 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) MODULE_PARM_DESC(file_path, "file path to test");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static bool test_lock_inode;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) module_param_named(lock_inode, test_lock_inode, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) MODULE_PARM_DESC(lock_inode, "lock file -> inode -> i_rwsem");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static bool test_lock_mapping;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) module_param_named(lock_mapping, test_lock_mapping, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) MODULE_PARM_DESC(lock_mapping, "lock file -> mapping -> i_mmap_rwsem");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static bool test_lock_sb_umount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) module_param_named(lock_sb_umount, test_lock_sb_umount, bool, 0400);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) MODULE_PARM_DESC(lock_sb_umount, "lock file -> sb -> s_umount");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) static atomic_t alloc_pages_failed = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static atomic64_t max_lock_wait = ATOMIC64_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static struct task_struct *main_task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static int master_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) static void test_lock(bool master, bool verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) u64 wait_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) if (measure_lock_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) wait_start = local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (lock_mutex_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) pr_notice("lock mutex %ps\n", (void *)lock_mutex_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) mutex_lock((struct mutex *)lock_mutex_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (lock_rwsem_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) pr_notice("lock rw_semaphore %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) (void *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) down_read((struct rw_semaphore *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) down_write((struct rw_semaphore *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (lock_mmap_sem && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) pr_notice("lock mmap_lock pid=%d\n", main_task->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) mmap_read_lock(main_task->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) mmap_write_lock(main_task->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (test_disable_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (disable_softirq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) local_bh_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (disable_preempt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (lock_rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) if (lock_spinlock_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) pr_notice("lock spinlock %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) (void *)lock_spinlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) spin_lock((spinlock_t *)lock_spinlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (lock_rwlock_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) pr_notice("lock rwlock %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) (void *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) read_lock((rwlock_t *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) write_lock((rwlock_t *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (measure_lock_wait) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) s64 cur_wait = local_clock() - wait_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) s64 max_wait = atomic64_read(&max_lock_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) if (cur_wait < max_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) max_wait = atomic64_cmpxchg(&max_lock_wait,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) max_wait, cur_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) } while (max_wait != cur_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (cur_wait > lock_wait_threshold)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) pr_notice_ratelimited("lock wait %lld ns\n", cur_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) static void test_unlock(bool master, bool verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (lock_rwlock_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) read_unlock((rwlock_t *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) write_unlock((rwlock_t *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) pr_notice("unlock rwlock %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) (void *)lock_rwlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (lock_spinlock_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) spin_unlock((spinlock_t *)lock_spinlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) pr_notice("unlock spinlock %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) (void *)lock_spinlock_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (lock_rcu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) if (disable_preempt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (disable_softirq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) local_bh_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) if (test_disable_irq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) if (lock_mmap_sem && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) mmap_read_unlock(main_task->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) mmap_write_unlock(main_task->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) pr_notice("unlock mmap_lock pid=%d\n", main_task->pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) if (lock_rwsem_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) if (lock_read)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) up_read((struct rw_semaphore *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) up_write((struct rw_semaphore *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pr_notice("unlock rw_semaphore %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) (void *)lock_rwsem_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (lock_mutex_ptr && master) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) mutex_unlock((struct mutex *)lock_mutex_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (verbose)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) pr_notice("unlock mutex %ps\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) (void *)lock_mutex_ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void test_alloc_pages(struct list_head *pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) for (i = 0; i < alloc_pages_nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) page = alloc_pages(alloc_pages_gfp, alloc_pages_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) atomic_inc(&alloc_pages_failed);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) list_add(&page->lru, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void test_free_pages(struct list_head *pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) struct page *page, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) list_for_each_entry_safe(page, next, pages, lru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) __free_pages(page, alloc_pages_order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) INIT_LIST_HEAD(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) static void test_wait(unsigned int secs, unsigned int nsecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (wait_state == TASK_RUNNING) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (secs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) mdelay(secs * MSEC_PER_SEC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (nsecs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) ndelay(nsecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) __set_current_state(wait_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (use_hrtimer) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) ktime_t time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) time = ns_to_ktime((u64)secs * NSEC_PER_SEC + nsecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) schedule_hrtimeout(&time, HRTIMER_MODE_REL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) schedule_timeout(secs * HZ + nsecs_to_jiffies(nsecs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) static void test_lockup(bool master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) u64 lockup_start = local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned int iter = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) LIST_HEAD(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) pr_notice("Start on CPU%d\n", raw_smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) test_lock(master, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) test_alloc_pages(&pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) while (iter++ < iterations && !signal_pending(main_task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) if (iowait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) current->in_iowait = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) test_wait(time_secs, time_nsecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) if (iowait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) current->in_iowait = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (reallocate_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) test_free_pages(&pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (reacquire_locks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) test_unlock(master, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (touch_softlockup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) touch_softlockup_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) if (touch_hardlockup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) touch_nmi_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (call_cond_resched)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) test_wait(cooldown_secs, cooldown_nsecs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (reacquire_locks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) test_lock(master, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (reallocate_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) test_alloc_pages(&pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) pr_notice("Finish on CPU%d in %lld ns\n", raw_smp_processor_id(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) local_clock() - lockup_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) test_free_pages(&pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) test_unlock(master, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) static DEFINE_PER_CPU(struct work_struct, test_works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) static void test_work_fn(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) test_lockup(!lock_single ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) work == per_cpu_ptr(&test_works, master_cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static bool test_kernel_ptr(unsigned long addr, int size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) void *ptr = (void *)addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) char buf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) /* should be at least readable kernel address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (!IS_ENABLED(CONFIG_ALTERNATE_USER_ADDRESS_SPACE) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) (access_ok((void __user *)ptr, 1) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) access_ok((void __user *)ptr + size - 1, 1))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) pr_err("user space ptr invalid in kernel: %#lx\n", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) if (get_kernel_nofault(buf, ptr) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) get_kernel_nofault(buf, ptr + size - 1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) pr_err("invalid kernel ptr: %#lx\n", addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static bool __maybe_unused test_magic(unsigned long addr, int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) unsigned int expected)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) void *ptr = (void *)addr + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) unsigned int magic = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (!addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (get_kernel_nofault(magic, ptr) || magic != expected) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) pr_err("invalid magic at %#lx + %#x = %#x, expected %#x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) addr, offset, magic, expected);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) static int __init test_lockup_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) u64 test_start = local_clock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) main_task = current;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) switch (state[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) case 'S':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) wait_state = TASK_INTERRUPTIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) case 'D':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) wait_state = TASK_UNINTERRUPTIBLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) case 'K':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) wait_state = TASK_KILLABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) case 'R':
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) wait_state = TASK_RUNNING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) pr_err("unknown state=%s\n", state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) if (alloc_pages_atomic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) alloc_pages_gfp = GFP_ATOMIC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) if (test_kernel_ptr(lock_spinlock_ptr, sizeof(spinlock_t)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) test_kernel_ptr(lock_rwlock_ptr, sizeof(rwlock_t)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) test_kernel_ptr(lock_mutex_ptr, sizeof(struct mutex)) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) test_kernel_ptr(lock_rwsem_ptr, sizeof(struct rw_semaphore)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) #ifdef CONFIG_DEBUG_SPINLOCK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (test_magic(lock_spinlock_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) offsetof(spinlock_t, rlock.magic),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) SPINLOCK_MAGIC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) test_magic(lock_rwlock_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) offsetof(rwlock_t, magic),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) RWLOCK_MAGIC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) test_magic(lock_mutex_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) offsetof(struct mutex, wait_lock.rlock.magic),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) SPINLOCK_MAGIC) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) test_magic(lock_rwsem_ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) offsetof(struct rw_semaphore, wait_lock.magic),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) SPINLOCK_MAGIC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) if ((wait_state != TASK_RUNNING ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) (call_cond_resched && !reacquire_locks) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) (alloc_pages_nr && gfpflags_allow_blocking(alloc_pages_gfp))) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) (test_disable_irq || disable_softirq || disable_preempt ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) lock_rcu || lock_spinlock_ptr || lock_rwlock_ptr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) pr_err("refuse to sleep in atomic context\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (lock_mmap_sem && !main_task->mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) pr_err("no mm to lock mmap_lock\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (test_file_path[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) test_file = filp_open(test_file_path, O_RDONLY, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (IS_ERR(test_file)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) pr_err("failed to open %s: %ld\n", test_file_path, PTR_ERR(test_file));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return PTR_ERR(test_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) test_inode = file_inode(test_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) } else if (test_lock_inode ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) test_lock_mapping ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) test_lock_sb_umount) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) pr_err("no file to lock\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) if (test_lock_inode && test_inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) lock_rwsem_ptr = (unsigned long)&test_inode->i_rwsem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (test_lock_mapping && test_file && test_file->f_mapping)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) lock_rwsem_ptr = (unsigned long)&test_file->f_mapping->i_mmap_rwsem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (test_lock_sb_umount && test_inode)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) lock_rwsem_ptr = (unsigned long)&test_inode->i_sb->s_umount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) pr_notice("START pid=%d time=%u +%u ns cooldown=%u +%u ns iterations=%u state=%s %s%s%s%s%s%s%s%s%s%s%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) main_task->pid, time_secs, time_nsecs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) cooldown_secs, cooldown_nsecs, iterations, state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) all_cpus ? "all_cpus " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) iowait ? "iowait " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) test_disable_irq ? "disable_irq " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) disable_softirq ? "disable_softirq " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) disable_preempt ? "disable_preempt " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) lock_rcu ? "lock_rcu " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) lock_read ? "lock_read " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) touch_softlockup ? "touch_softlockup " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) touch_hardlockup ? "touch_hardlockup " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) call_cond_resched ? "call_cond_resched " : "",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) reacquire_locks ? "reacquire_locks " : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (alloc_pages_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) pr_notice("ALLOCATE PAGES nr=%u order=%u gfp=%pGg %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) alloc_pages_nr, alloc_pages_order, &alloc_pages_gfp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) reallocate_pages ? "reallocate_pages " : "");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) if (all_cpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) master_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) INIT_WORK(per_cpu_ptr(&test_works, cpu), test_work_fn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) queue_work_on(cpu, system_highpri_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) per_cpu_ptr(&test_works, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) flush_work(per_cpu_ptr(&test_works, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) test_lockup(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) if (measure_lock_wait)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) pr_notice("Maximum lock wait: %lld ns\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) atomic64_read(&max_lock_wait));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) if (alloc_pages_nr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) pr_notice("Page allocation failed %u times\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) atomic_read(&alloc_pages_failed));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) pr_notice("FINISH in %llu ns\n", local_clock() - test_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) if (test_file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) fput(test_file);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (signal_pending(main_task))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) module_init(test_lockup_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) MODULE_LICENSE("GPL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) MODULE_IMPORT_NS(VFS_internal_I_am_really_a_filesystem_and_am_NOT_a_driver);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) MODULE_AUTHOR("Konstantin Khlebnikov <khlebnikov@yandex-team.ru>");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) MODULE_DESCRIPTION("Test module to generate lockups");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags