^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Detect hard and soft lockups on a system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * started by Don Zickus, Copyright (C) 2010 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Note: Most of this code is borrowed heavily from the original softlockup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * detector, so thanks to Ingo for the initial implementation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Some chunks also taken from the old x86-specific nmi watchdog code, thanks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * to those contributors as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #define pr_fmt(fmt) "watchdog: " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/tick.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/sched/clock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/sched/debug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/sched/isolation.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <asm/irq_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/kvm_para.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <trace/hooks/softlockup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) static DEFINE_MUTEX(watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #if defined(CONFIG_HARDLOCKUP_DETECTOR) || defined(CONFIG_HAVE_NMI_WATCHDOG)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) # define WATCHDOG_DEFAULT (SOFT_WATCHDOG_ENABLED | NMI_WATCHDOG_ENABLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) # define NMI_WATCHDOG_DEFAULT 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) # define WATCHDOG_DEFAULT (SOFT_WATCHDOG_ENABLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) # define NMI_WATCHDOG_DEFAULT 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) unsigned long __read_mostly watchdog_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) int __read_mostly watchdog_user_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) int __read_mostly nmi_watchdog_user_enabled = NMI_WATCHDOG_DEFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) int __read_mostly soft_watchdog_user_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) int __read_mostly watchdog_thresh = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) static int __read_mostly nmi_watchdog_available;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) struct cpumask watchdog_cpumask __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) unsigned long *watchdog_cpumask_bits = cpumask_bits(&watchdog_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #ifdef CONFIG_HARDLOCKUP_DETECTOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) # ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) int __read_mostly sysctl_hardlockup_all_cpu_backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) # endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) ATOMIC_NOTIFIER_HEAD(hardlock_notifier_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Should we panic when a soft-lockup or hard-lockup occurs:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned int __read_mostly hardlockup_panic =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) CONFIG_BOOTPARAM_HARDLOCKUP_PANIC_VALUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * We may not want to enable hard lockup detection by default in all cases,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * for example when running the kernel as a guest on a hypervisor. In these
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * cases this function can be called to disable hard lockup detection. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * function should only be executed once by the boot processor before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * kernel command line parameters are parsed, because otherwise it is not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) * possible to override this in hardlockup_panic_setup().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) void __init hardlockup_detector_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) nmi_watchdog_user_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static int __init hardlockup_panic_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) if (!strncmp(str, "panic", 5))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) hardlockup_panic = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) else if (!strncmp(str, "nopanic", 7))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) hardlockup_panic = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) else if (!strncmp(str, "0", 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) nmi_watchdog_user_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) else if (!strncmp(str, "1", 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) nmi_watchdog_user_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) __setup("nmi_watchdog=", hardlockup_panic_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #endif /* CONFIG_HARDLOCKUP_DETECTOR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) #ifdef CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static cpumask_t __read_mostly watchdog_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) static DEFINE_PER_CPU(bool, hard_watchdog_warn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static DEFINE_PER_CPU(bool, watchdog_nmi_touch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) static unsigned int watchdog_next_cpu(unsigned int cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) int watchdog_nmi_enable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * The new cpu will be marked online before the first hrtimer interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * runs on it. If another cpu tests for a hardlockup on the new cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * before it has run its first hrtimer, it will get a false positive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Touch the watchdog on the new cpu to delay the first check for at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * least 3 sampling periods to guarantee one hrtimer has run on the new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) per_cpu(watchdog_nmi_touch, cpu) = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) cpumask_set_cpu(cpu, &watchdog_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) void watchdog_nmi_disable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) unsigned int next_cpu = watchdog_next_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Offlining this cpu will cause the cpu before this one to start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * checking the one after this one. If this cpu just finished checking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * the next cpu and updating hrtimer_interrupts_saved, and then the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * previous cpu checks it within one sample period, it will trigger a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * false positive. Touch the watchdog on the next cpu to prevent it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) if (next_cpu < nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) per_cpu(watchdog_nmi_touch, next_cpu) = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) cpumask_clear_cpu(cpu, &watchdog_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) * These functions can be overridden if an architecture implements its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) * own hardlockup detector.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * watchdog_nmi_enable/disable can be implemented to start and stop when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * softlockup watchdog threads start and stop. The arch must select the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * SOFTLOCKUP_DETECTOR Kconfig.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) int __weak watchdog_nmi_enable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) hardlockup_detector_perf_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) void __weak watchdog_nmi_disable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) hardlockup_detector_perf_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #endif /* CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) /* Return 0, if a NMI watchdog is available. Error code otherwise */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) int __weak __init watchdog_nmi_probe(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return hardlockup_detector_perf_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) * watchdog_nmi_stop - Stop the watchdog for reconfiguration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * The reconfiguration steps are:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * watchdog_nmi_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * update_variables();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * watchdog_nmi_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) void __weak watchdog_nmi_stop(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * watchdog_nmi_start - Start the watchdog after reconfiguration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * Counterpart to watchdog_nmi_stop().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) * The following variables have been updated in update_variables() and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) * contain the currently valid configuration:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) * - watchdog_enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * - watchdog_thresh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * - watchdog_cpumask
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) void __weak watchdog_nmi_start(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) * lockup_detector_update_enable - Update the sysctl enable bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * Caller needs to make sure that the NMI/perf watchdogs are off, so this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * can't race with watchdog_nmi_disable().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static void lockup_detector_update_enable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) watchdog_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (!watchdog_user_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (nmi_watchdog_available && nmi_watchdog_user_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) watchdog_enabled |= NMI_WATCHDOG_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (soft_watchdog_user_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) watchdog_enabled |= SOFT_WATCHDOG_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #ifdef CONFIG_SOFTLOCKUP_DETECTOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) #define SOFTLOCKUP_RESET ULONG_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) int __read_mostly sysctl_softlockup_all_cpu_backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static struct cpumask watchdog_allowed_mask __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /* Global variables, exported for sysctl */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) unsigned int __read_mostly softlockup_panic =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) CONFIG_BOOTPARAM_SOFTLOCKUP_PANIC_VALUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) static bool softlockup_initialized __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) static u64 __read_mostly sample_period;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) static DEFINE_PER_CPU(unsigned long, watchdog_touch_ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) static DEFINE_PER_CPU(struct hrtimer, watchdog_hrtimer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static DEFINE_PER_CPU(bool, softlockup_touch_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static DEFINE_PER_CPU(unsigned long, hrtimer_interrupts_saved);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) static unsigned long soft_lockup_nmi_warn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) static int __init nowatchdog_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) watchdog_user_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) __setup("nowatchdog", nowatchdog_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static int __init nosoftlockup_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) soft_watchdog_user_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) __setup("nosoftlockup", nosoftlockup_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) static int __init watchdog_thresh_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) get_option(&str, &watchdog_thresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) __setup("watchdog_thresh=", watchdog_thresh_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) static void __lockup_detector_cleanup(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * Hard-lockup warnings should be triggered after just a few seconds. Soft-
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * lockups can have false positives under extreme conditions. So we generally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * want a higher threshold for soft lockups than for hard lockups. So we couple
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * the thresholds with a factor: we make the soft threshold twice the amount of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) * time the hard threshold is.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) static int get_softlockup_thresh(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) return watchdog_thresh * 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * Returns seconds, approximately. We don't need nanosecond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) * resolution, and we don't need to waste time with a big divide when
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * 2^30ns == 1.074s.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static unsigned long get_timestamp(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) return running_clock() >> 30LL; /* 2^30 ~= 10^9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) static void set_sample_period(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * convert watchdog_thresh from seconds to ns
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * the divide by 5 is to give hrtimer several chances (two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * or three with the current relation between the soft
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) * and hard thresholds) to increment before the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) * hardlockup detector generates a warning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) sample_period = get_softlockup_thresh() * ((u64)NSEC_PER_SEC / 5);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) watchdog_update_hrtimer_threshold(sample_period);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /* Commands for resetting the watchdog */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static void update_touch_ts(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) __this_cpu_write(watchdog_touch_ts, get_timestamp());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * touch_softlockup_watchdog_sched - touch watchdog on scheduler stalls
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) * Call when the scheduler may have stalled for legitimate reasons
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) * preventing the watchdog task from executing - e.g. the scheduler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * entering idle state. This should only be used for scheduler events.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) * Use touch_softlockup_watchdog() for everything else.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) notrace void touch_softlockup_watchdog_sched(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * Preemption can be enabled. It doesn't matter which CPU's timestamp
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * gets zeroed here, so use the raw_ operation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) raw_cpu_write(watchdog_touch_ts, SOFTLOCKUP_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) notrace void touch_softlockup_watchdog(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) touch_softlockup_watchdog_sched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) wq_watchdog_touch(raw_smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) EXPORT_SYMBOL(touch_softlockup_watchdog);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) void touch_all_softlockup_watchdogs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * watchdog_mutex cannpt be taken here, as this might be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * from (soft)interrupt context, so the access to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) * watchdog_allowed_cpumask might race with a concurrent update.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) * The watchdog time stamp can race against a concurrent real
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) * update as well, the only side effect might be a cycle delay for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * the softlockup check.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) for_each_cpu(cpu, &watchdog_allowed_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) per_cpu(watchdog_touch_ts, cpu) = SOFTLOCKUP_RESET;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) wq_watchdog_touch(-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) void touch_softlockup_watchdog_sync(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) __this_cpu_write(softlockup_touch_sync, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) __this_cpu_write(watchdog_touch_ts, SOFTLOCKUP_RESET);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static int is_softlockup(unsigned long touch_ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) unsigned long now = get_timestamp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if ((watchdog_enabled & SOFT_WATCHDOG_ENABLED) && watchdog_thresh){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* Warn about unreasonable delays. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) if (time_after(now, touch_ts + get_softlockup_thresh()))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) return now - touch_ts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) /* watchdog detector functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) bool is_hardlockup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) unsigned long hrint = __this_cpu_read(hrtimer_interrupts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (__this_cpu_read(hrtimer_interrupts_saved) == hrint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) __this_cpu_write(hrtimer_interrupts_saved, hrint);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) #ifdef CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) static unsigned int watchdog_next_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) cpumask_t cpus = watchdog_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) unsigned int next_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) next_cpu = cpumask_next(cpu, &cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) if (next_cpu >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) next_cpu = cpumask_first(&cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) if (next_cpu == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) return nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return next_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static int is_hardlockup_other_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) unsigned long hrint = per_cpu(hrtimer_interrupts, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) if (per_cpu(hrtimer_interrupts_saved, cpu) == hrint)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) per_cpu(hrtimer_interrupts_saved, cpu) = hrint;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) static void watchdog_check_hardlockup_other_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) unsigned int next_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * Test for hardlockups every 3 samples. The sample period is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * watchdog_thresh * 2 / 5, so 3 samples gets us back to slightly over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) * watchdog_thresh (over by 20%).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (__this_cpu_read(hrtimer_interrupts) % 3 != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) /* check for a hardlockup on the next cpu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) next_cpu = watchdog_next_cpu(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) if (next_cpu >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) if (per_cpu(watchdog_nmi_touch, next_cpu) == true) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) per_cpu(watchdog_nmi_touch, next_cpu) = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (is_hardlockup_other_cpu(next_cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) /* only warn once */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (per_cpu(hard_watchdog_warn, next_cpu) == true)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (hardlockup_panic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) panic("Watchdog detected hard LOCKUP on cpu %u", next_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) WARN(1, "Watchdog detected hard LOCKUP on cpu %u", next_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) atomic_notifier_call_chain(&hardlock_notifier_list, 0, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) per_cpu(hard_watchdog_warn, next_cpu) = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) per_cpu(hard_watchdog_warn, next_cpu) = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) static inline void watchdog_check_hardlockup_other_cpu(void) { return; }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static void watchdog_interrupt_count(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __this_cpu_inc(hrtimer_interrupts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) static DEFINE_PER_CPU(struct completion, softlockup_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) static DEFINE_PER_CPU(struct cpu_stop_work, softlockup_stop_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * The watchdog thread function - touches the timestamp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * It only runs once every sample_period seconds (4 seconds by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) * default) to reset the softlockup timestamp. If this gets delayed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * for more than 2*watchdog_thresh seconds then the debug-printout
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * triggers in watchdog_timer_fn().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static int softlockup_fn(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) update_touch_ts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) complete(this_cpu_ptr(&softlockup_completion));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /* watchdog kicker functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) static enum hrtimer_restart watchdog_timer_fn(struct hrtimer *hrtimer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) unsigned long touch_ts = __this_cpu_read(watchdog_touch_ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) struct pt_regs *regs = get_irq_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) int duration;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) int softlockup_all_cpu_backtrace = sysctl_softlockup_all_cpu_backtrace;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) if (!watchdog_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) return HRTIMER_NORESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) /* test for hardlockups on the next cpu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) if (IS_ENABLED(CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) watchdog_check_hardlockup_other_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) /* kick the hardlockup detector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) watchdog_interrupt_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) /* kick the softlockup detector */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) if (completion_done(this_cpu_ptr(&softlockup_completion))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) reinit_completion(this_cpu_ptr(&softlockup_completion));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) stop_one_cpu_nowait(smp_processor_id(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) softlockup_fn, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) this_cpu_ptr(&softlockup_stop_work));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) /* .. and repeat */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) hrtimer_forward_now(hrtimer, ns_to_ktime(sample_period));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) if (touch_ts == SOFTLOCKUP_RESET) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) if (unlikely(__this_cpu_read(softlockup_touch_sync))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * If the time stamp was touched atomically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * make sure the scheduler tick is up to date.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) __this_cpu_write(softlockup_touch_sync, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) sched_clock_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) /* Clear the guest paused flag on watchdog reset */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) kvm_check_and_clear_guest_paused();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) update_touch_ts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) /* check for a softlockup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * This is done by making sure a high priority task is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * being scheduled. The task touches the watchdog to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * indicate it is getting cpu time. If it hasn't then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * this is a good indication some task is hogging the cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) duration = is_softlockup(touch_ts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (unlikely(duration)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * If a virtual machine is stopped by the host it can look to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * the watchdog like a soft lockup, check to see if the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) * stopped the vm before we issue the warning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (kvm_check_and_clear_guest_paused())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * Prevent multiple soft-lockup reports if one cpu is already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) * engaged in dumping all cpu back traces.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (softlockup_all_cpu_backtrace) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (test_and_set_bit_lock(0, &soft_lockup_nmi_warn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) /* Start period for the next softlockup warning. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) update_touch_ts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) pr_emerg("BUG: soft lockup - CPU#%d stuck for %us! [%s:%d]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) smp_processor_id(), duration,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) current->comm, task_pid_nr(current));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) print_modules();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) print_irqtrace_events(current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) show_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) if (softlockup_all_cpu_backtrace) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) trigger_allbutself_cpu_backtrace();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) clear_bit_unlock(0, &soft_lockup_nmi_warn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) trace_android_vh_watchdog_timer_softlockup(duration, regs, !!softlockup_panic);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) add_taint(TAINT_SOFTLOCKUP, LOCKDEP_STILL_OK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (softlockup_panic)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) panic("softlockup: hung tasks");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return HRTIMER_RESTART;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) static void watchdog_enable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct completion *done = this_cpu_ptr(&softlockup_completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) WARN_ON_ONCE(cpu != smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) init_completion(done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) complete(done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) * Start the timer first to prevent the NMI watchdog triggering
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) * before the timer has a chance to fire.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) hrtimer_init(hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) hrtimer->function = watchdog_timer_fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) hrtimer_start(hrtimer, ns_to_ktime(sample_period),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) HRTIMER_MODE_REL_PINNED_HARD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) /* Initialize timestamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) update_touch_ts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) /* Enable the perf event */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if (watchdog_enabled & NMI_WATCHDOG_ENABLED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) watchdog_nmi_enable(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) else if (IS_ENABLED(CONFIG_HARDLOCKUP_DETECTOR_OTHER_CPU))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) watchdog_nmi_enable(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) static void watchdog_disable(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct hrtimer *hrtimer = this_cpu_ptr(&watchdog_hrtimer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) WARN_ON_ONCE(cpu != smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) * Disable the perf event first. That prevents that a large delay
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * between disabling the timer and disabling the perf event causes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) * the perf NMI to detect a false positive.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) watchdog_nmi_disable(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) hrtimer_cancel(hrtimer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) wait_for_completion(this_cpu_ptr(&softlockup_completion));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) static int softlockup_stop_fn(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) watchdog_disable(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) static void softlockup_stop_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (!softlockup_initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) for_each_cpu(cpu, &watchdog_allowed_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) smp_call_on_cpu(cpu, softlockup_stop_fn, NULL, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) cpumask_clear(&watchdog_allowed_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) static int softlockup_start_fn(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) watchdog_enable(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) static void softlockup_start_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) cpumask_copy(&watchdog_allowed_mask, &watchdog_cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) for_each_cpu(cpu, &watchdog_allowed_mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) smp_call_on_cpu(cpu, softlockup_start_fn, NULL, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) int lockup_detector_online_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) watchdog_enable(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) int lockup_detector_offline_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) if (cpumask_test_cpu(cpu, &watchdog_allowed_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) watchdog_disable(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) static void lockup_detector_reconfigure(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) watchdog_nmi_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) softlockup_stop_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) set_sample_period();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) lockup_detector_update_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (watchdog_enabled && watchdog_thresh)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) softlockup_start_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) watchdog_nmi_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * Must be called outside the cpus locked section to prevent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * recursive locking in the perf code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) __lockup_detector_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * Create the watchdog thread infrastructure and configure the detector(s).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * The threads are not unparked as watchdog_allowed_mask is empty. When
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * the threads are successfully initialized, take the proper locks and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * unpark the threads in the watchdog_cpumask if the watchdog is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) static __init void lockup_detector_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * If sysctl is off and watchdog got disabled on the command line,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * nothing to do here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) lockup_detector_update_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (!IS_ENABLED(CONFIG_SYSCTL) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) !(watchdog_enabled && watchdog_thresh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) mutex_lock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) lockup_detector_reconfigure();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) softlockup_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) mutex_unlock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) #else /* CONFIG_SOFTLOCKUP_DETECTOR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) static void lockup_detector_reconfigure(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) watchdog_nmi_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) lockup_detector_update_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) watchdog_nmi_start();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) static inline void lockup_detector_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) lockup_detector_reconfigure();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) #endif /* !CONFIG_SOFTLOCKUP_DETECTOR */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) static void __lockup_detector_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) lockdep_assert_held(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) hardlockup_detector_perf_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) * lockup_detector_cleanup - Cleanup after cpu hotplug or sysctl changes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) * Caller must not hold the cpu hotplug rwsem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) void lockup_detector_cleanup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) mutex_lock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) __lockup_detector_cleanup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) mutex_unlock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) * lockup_detector_soft_poweroff - Interface to stop lockup detector(s)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) * Special interface for parisc. It prevents lockup detector warnings from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) * the default pm_poweroff() function which busy loops forever.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) void lockup_detector_soft_poweroff(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) watchdog_enabled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) #ifdef CONFIG_SYSCTL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* Propagate any changes to the watchdog threads */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) static void proc_watchdog_update(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) /* Remove impossible cpus to keep sysctl output clean. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) cpumask_and(&watchdog_cpumask, &watchdog_cpumask, cpu_possible_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) lockup_detector_reconfigure();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * common function for watchdog, nmi_watchdog and soft_watchdog parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) * caller | table->data points to | 'which'
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * -------------------|----------------------------|--------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * proc_watchdog | watchdog_user_enabled | NMI_WATCHDOG_ENABLED |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * | | SOFT_WATCHDOG_ENABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * -------------------|----------------------------|--------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * proc_nmi_watchdog | nmi_watchdog_user_enabled | NMI_WATCHDOG_ENABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * -------------------|----------------------------|--------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * proc_soft_watchdog | soft_watchdog_user_enabled | SOFT_WATCHDOG_ENABLED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) static int proc_watchdog_common(int which, struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) int err, old, *param = table->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) mutex_lock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) if (!write) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) * On read synchronize the userspace interface. This is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) * racy snapshot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *param = (watchdog_enabled & which) != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) old = READ_ONCE(*param);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) if (!err && old != READ_ONCE(*param))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) proc_watchdog_update();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) mutex_unlock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * /proc/sys/kernel/watchdog
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) int proc_watchdog(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return proc_watchdog_common(NMI_WATCHDOG_ENABLED|SOFT_WATCHDOG_ENABLED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * /proc/sys/kernel/nmi_watchdog
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) int proc_nmi_watchdog(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (!nmi_watchdog_available && write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return -ENOTSUPP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) return proc_watchdog_common(NMI_WATCHDOG_ENABLED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) * /proc/sys/kernel/soft_watchdog
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) int proc_soft_watchdog(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) return proc_watchdog_common(SOFT_WATCHDOG_ENABLED,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) * /proc/sys/kernel/watchdog_thresh
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) int proc_watchdog_thresh(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) int err, old;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) mutex_lock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) old = READ_ONCE(watchdog_thresh);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) err = proc_dointvec_minmax(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (!err && write && old != READ_ONCE(watchdog_thresh))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) proc_watchdog_update();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) mutex_unlock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) * The cpumask is the mask of possible cpus that the watchdog can run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * on, not the mask of cpus it is actually running on. This allows the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * user to specify a mask that will include cpus that have not yet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * been brought online, if desired.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) int proc_watchdog_cpumask(struct ctl_table *table, int write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) void *buffer, size_t *lenp, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) mutex_lock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) err = proc_do_large_bitmap(table, write, buffer, lenp, ppos);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (!err && write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) proc_watchdog_update();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) mutex_unlock(&watchdog_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) #endif /* CONFIG_SYSCTL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) void __init lockup_detector_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) if (tick_nohz_full_enabled())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) pr_info("Disabling watchdog on nohz_full cores by default\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) cpumask_copy(&watchdog_cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) housekeeping_cpumask(HK_FLAG_TIMER));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) if (!watchdog_nmi_probe())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) nmi_watchdog_available = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) lockup_detector_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags