^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * linux/kernel/profile.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Simple profiling. Manages a direct-mapped profile hit count buffer,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * with configurable resolution, support for restricting the cpus on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * which profiling is done, and switching between cpu time and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * schedule() calls via kernel command line parameters passed at boot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Scheduler profiling support, Arjan van de Ven and Ingo Molnar,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Red Hat, July 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Consolidation of architecture support code for profiling,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Nadia Yvette Chambers, Oracle, July 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * Amortized hit count accounting via per-cpu open-addressed hashtables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * to resolve timer interrupt livelocks, Nadia Yvette Chambers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Oracle, 2004
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/sched/stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/irq_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct profile_hit {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) u32 pc, hits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #define PROFILE_GRPSHIFT 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define PROFILE_GRPSZ (1 << PROFILE_GRPSHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #define NR_PROFILE_HIT (PAGE_SIZE/sizeof(struct profile_hit))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #define NR_PROFILE_GRP (NR_PROFILE_HIT/PROFILE_GRPSZ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static atomic_t *prof_buffer;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static unsigned long prof_len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static unsigned short int prof_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) int prof_on __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) EXPORT_SYMBOL_GPL(prof_on);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) static cpumask_var_t prof_cpu_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) static DEFINE_PER_CPU(struct profile_hit *[2], cpu_profile_hits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static DEFINE_PER_CPU(int, cpu_profile_flip);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) static DEFINE_MUTEX(profile_flip_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) int profile_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static const char schedstr[] = "schedule";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static const char sleepstr[] = "sleep";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) static const char kvmstr[] = "kvm";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) int par;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) if (!strncmp(str, sleepstr, strlen(sleepstr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #ifdef CONFIG_SCHEDSTATS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) force_schedstat_enabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) prof_on = SLEEP_PROFILING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) if (str[strlen(sleepstr)] == ',')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) str += strlen(sleepstr) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) if (get_option(&str, &par))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) pr_info("kernel sleep profiling enabled (shift: %u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) prof_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) pr_warn("kernel sleep profiling requires CONFIG_SCHEDSTATS\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #endif /* CONFIG_SCHEDSTATS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) } else if (!strncmp(str, schedstr, strlen(schedstr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) prof_on = SCHED_PROFILING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) if (str[strlen(schedstr)] == ',')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) str += strlen(schedstr) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) if (get_option(&str, &par))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) pr_info("kernel schedule profiling enabled (shift: %u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) prof_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) } else if (!strncmp(str, kvmstr, strlen(kvmstr))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) prof_on = KVM_PROFILING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) if (str[strlen(kvmstr)] == ',')
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) str += strlen(kvmstr) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (get_option(&str, &par))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) pr_info("kernel KVM profiling enabled (shift: %u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) prof_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) } else if (get_option(&str, &par)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) prof_shift = clamp(par, 0, BITS_PER_LONG - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) prof_on = CPU_PROFILING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) pr_info("kernel profiling enabled (shift: %u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) prof_shift);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) __setup("profile=", profile_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int __ref profile_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int buffer_bytes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (!prof_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /* only text is profiled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) prof_len = (_etext - _stext) >> prof_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) buffer_bytes = prof_len*sizeof(atomic_t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) if (!alloc_cpumask_var(&prof_cpu_mask, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) cpumask_copy(prof_cpu_mask, cpu_possible_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) prof_buffer = kzalloc(buffer_bytes, GFP_KERNEL|__GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (prof_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) prof_buffer = alloc_pages_exact(buffer_bytes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) GFP_KERNEL|__GFP_ZERO|__GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (prof_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) prof_buffer = vzalloc(buffer_bytes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) if (prof_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) free_cpumask_var(prof_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* Profile event notifications */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) static BLOCKING_NOTIFIER_HEAD(task_exit_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static ATOMIC_NOTIFIER_HEAD(task_free_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static BLOCKING_NOTIFIER_HEAD(munmap_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) void profile_task_exit(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) blocking_notifier_call_chain(&task_exit_notifier, 0, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) int profile_handoff_task(struct task_struct *task)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) ret = atomic_notifier_call_chain(&task_free_notifier, 0, task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return (ret == NOTIFY_OK) ? 1 : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) void profile_munmap(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) blocking_notifier_call_chain(&munmap_notifier, 0, (void *)addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) int task_handoff_register(struct notifier_block *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return atomic_notifier_chain_register(&task_free_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) EXPORT_SYMBOL_GPL(task_handoff_register);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) int task_handoff_unregister(struct notifier_block *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) return atomic_notifier_chain_unregister(&task_free_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) EXPORT_SYMBOL_GPL(task_handoff_unregister);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int profile_event_register(enum profile_type type, struct notifier_block *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) case PROFILE_TASK_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) err = blocking_notifier_chain_register(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) &task_exit_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) case PROFILE_MUNMAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) err = blocking_notifier_chain_register(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) &munmap_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) EXPORT_SYMBOL_GPL(profile_event_register);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) int profile_event_unregister(enum profile_type type, struct notifier_block *n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) int err = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) switch (type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) case PROFILE_TASK_EXIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) err = blocking_notifier_chain_unregister(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) &task_exit_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) case PROFILE_MUNMAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) err = blocking_notifier_chain_unregister(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) &munmap_notifier, n);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) EXPORT_SYMBOL_GPL(profile_event_unregister);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #if defined(CONFIG_SMP) && defined(CONFIG_PROC_FS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * Each cpu has a pair of open-addressed hashtables for pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) * profile hits. read_profile() IPI's all cpus to request them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * to flip buffers and flushes their contents to prof_buffer itself.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * Flip requests are serialized by the profile_flip_mutex. The sole
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * use of having a second hashtable is for avoiding cacheline
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * contention that would otherwise happen during flushes of pending
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * profile hits required for the accuracy of reported profile hits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * and so resurrect the interrupt livelock issue.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) * The open-addressed hashtables are indexed by profile buffer slot
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) * and hold the number of pending hits to that profile buffer slot on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) * a cpu in an entry. When the hashtable overflows, all pending hits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * are accounted to their corresponding profile buffer slots with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * atomic_add() and the hashtable emptied. As numerous pending hits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * may be accounted to a profile buffer slot in a hashtable entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * this amortizes a number of atomic profile buffer increments likely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) * to be far larger than the number of entries in the hashtable,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) * particularly given that the number of distinct profile buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) * positions to which hits are accounted during short intervals (e.g.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * several seconds) is usually very small. Exclusion from buffer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * flipping is provided by interrupt disablement (note that for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * SCHED_PROFILING or SLEEP_PROFILING profile_hit() may be called from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * process context).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * The hash function is meant to be lightweight as opposed to strong,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * and was vaguely inspired by ppc64 firmware-supported inverted
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * pagetable hash functions, but uses a full hashtable full of finite
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * collision chains, not just pairs of them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * -- nyc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static void __profile_flip_buffers(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) per_cpu(cpu_profile_flip, cpu) = !per_cpu(cpu_profile_flip, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static void profile_flip_buffers(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) int i, j, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) mutex_lock(&profile_flip_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) j = per_cpu(cpu_profile_flip, get_cpu());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) on_each_cpu(__profile_flip_buffers, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[j];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) for (i = 0; i < NR_PROFILE_HIT; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (!hits[i].hits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) if (hits[i].pc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) hits[i].pc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) hits[i].hits = hits[i].pc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) mutex_unlock(&profile_flip_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) static void profile_discard_flip_buffers(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) int i, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) mutex_lock(&profile_flip_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) i = per_cpu(cpu_profile_flip, get_cpu());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) on_each_cpu(__profile_flip_buffers, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct profile_hit *hits = per_cpu(cpu_profile_hits, cpu)[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) memset(hits, 0, NR_PROFILE_HIT*sizeof(struct profile_hit));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) mutex_unlock(&profile_flip_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) unsigned long primary, secondary, flags, pc = (unsigned long)__pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) int i, j, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct profile_hit *hits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) pc = min((pc - (unsigned long)_stext) >> prof_shift, prof_len - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) i = primary = (pc & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) secondary = (~(pc << 1) & (NR_PROFILE_GRP - 1)) << PROFILE_GRPSHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) hits = per_cpu(cpu_profile_hits, cpu)[per_cpu(cpu_profile_flip, cpu)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (!hits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * We buffer the global profiler buffer into a per-CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * queue and thus reduce the number of global (and possibly
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * NUMA-alien) accesses. The write-queue is self-coalescing:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) for (j = 0; j < PROFILE_GRPSZ; ++j) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (hits[i + j].pc == pc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) hits[i + j].hits += nr_hits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) } else if (!hits[i + j].hits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) hits[i + j].pc = pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) hits[i + j].hits = nr_hits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) i = (i + secondary) & (NR_PROFILE_HIT - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) } while (i != primary);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) * Add the current hit(s) and flush the write-queue out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * to the global buffer:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) atomic_add(nr_hits, &prof_buffer[pc]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) for (i = 0; i < NR_PROFILE_HIT; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) atomic_add(hits[i].hits, &prof_buffer[hits[i].pc]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) hits[i].pc = hits[i].hits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) static int profile_dead_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (cpumask_available(prof_cpu_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) cpumask_clear_cpu(cpu, prof_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) for (i = 0; i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (per_cpu(cpu_profile_hits, cpu)[i]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) page = virt_to_page(per_cpu(cpu_profile_hits, cpu)[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) per_cpu(cpu_profile_hits, cpu)[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) static int profile_prepare_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) int i, node = cpu_to_mem(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) per_cpu(cpu_profile_flip, cpu) = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) for (i = 0; i < 2; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) if (per_cpu(cpu_profile_hits, cpu)[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) page = __alloc_pages_node(node, GFP_KERNEL | __GFP_ZERO, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) profile_dead_cpu(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) per_cpu(cpu_profile_hits, cpu)[i] = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) static int profile_online_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) if (cpumask_available(prof_cpu_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) cpumask_set_cpu(cpu, prof_cpu_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #else /* !CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #define profile_flip_buffers() do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) #define profile_discard_flip_buffers() do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) static void do_profile_hits(int type, void *__pc, unsigned int nr_hits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) unsigned long pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) pc = ((unsigned long)__pc - (unsigned long)_stext) >> prof_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) atomic_add(nr_hits, &prof_buffer[min(pc, prof_len - 1)]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) #endif /* !CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) void profile_hits(int type, void *__pc, unsigned int nr_hits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (prof_on != type || !prof_buffer)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) do_profile_hits(type, __pc, nr_hits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) EXPORT_SYMBOL_GPL(profile_hits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) void profile_tick(int type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) struct pt_regs *regs = get_irq_regs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) if (!user_mode(regs) && cpumask_available(prof_cpu_mask) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) cpumask_test_cpu(smp_processor_id(), prof_cpu_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) profile_hit(type, (void *)profile_pc(regs));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) #ifdef CONFIG_PROC_FS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) static int prof_cpu_mask_proc_show(struct seq_file *m, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) seq_printf(m, "%*pb\n", cpumask_pr_args(prof_cpu_mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) static int prof_cpu_mask_proc_open(struct inode *inode, struct file *file)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return single_open(file, prof_cpu_mask_proc_show, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) static ssize_t prof_cpu_mask_proc_write(struct file *file,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) const char __user *buffer, size_t count, loff_t *pos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) cpumask_var_t new_value;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if (!alloc_cpumask_var(&new_value, GFP_KERNEL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) err = cpumask_parse_user(buffer, count, new_value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (!err) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) cpumask_copy(prof_cpu_mask, new_value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) err = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) free_cpumask_var(new_value);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) static const struct proc_ops prof_cpu_mask_proc_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) .proc_open = prof_cpu_mask_proc_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) .proc_read = seq_read,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) .proc_lseek = seq_lseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) .proc_release = single_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) .proc_write = prof_cpu_mask_proc_write,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) void create_prof_cpu_mask(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) /* create /proc/irq/prof_cpu_mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) proc_create("irq/prof_cpu_mask", 0600, NULL, &prof_cpu_mask_proc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * This function accesses profiling information. The returned data is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * binary: the sampling step and the actual contents of the profile
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * buffer. Use of the program readprofile is recommended in order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * get meaningful info out of these data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static ssize_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) read_profile(struct file *file, char __user *buf, size_t count, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) unsigned long p = *ppos;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) ssize_t read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) char *pnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) unsigned long sample_step = 1UL << prof_shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) profile_flip_buffers();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) if (p >= (prof_len+1)*sizeof(unsigned int))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if (count > (prof_len+1)*sizeof(unsigned int) - p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) count = (prof_len+1)*sizeof(unsigned int) - p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) read = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) while (p < sizeof(unsigned int) && count > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) if (put_user(*((char *)(&sample_step)+p), buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) buf++; p++; count--; read++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) pnt = (char *)prof_buffer + p - sizeof(atomic_t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) if (copy_to_user(buf, (void *)pnt, count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) read += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) *ppos += read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) return read;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) * Writing to /proc/profile resets the counters
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) * Writing a 'profiling multiplier' value into it also re-sets the profiling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) * interrupt frequency, on architectures that support this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) static ssize_t write_profile(struct file *file, const char __user *buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) size_t count, loff_t *ppos)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) extern int setup_profiling_timer(unsigned int multiplier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) if (count == sizeof(int)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) unsigned int multiplier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) if (copy_from_user(&multiplier, buf, sizeof(int)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) if (setup_profiling_timer(multiplier))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) profile_discard_flip_buffers();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) memset(prof_buffer, 0, prof_len * sizeof(atomic_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) return count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) static const struct proc_ops profile_proc_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) .proc_read = read_profile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) .proc_write = write_profile,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) .proc_lseek = default_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) int __ref create_proc_profile(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) struct proc_dir_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) enum cpuhp_state online_state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) int err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) if (!prof_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) err = cpuhp_setup_state(CPUHP_PROFILE_PREPARE, "PROFILE_PREPARE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) profile_prepare_cpu, profile_dead_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) err = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "AP_PROFILE_ONLINE",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) profile_online_cpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) if (err < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) goto err_state_prep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) online_state = err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) entry = proc_create("profile", S_IWUSR | S_IRUGO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) NULL, &profile_proc_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) if (!entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) goto err_state_onl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) proc_set_size(entry, (1 + prof_len) * sizeof(atomic_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) err_state_onl:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) cpuhp_remove_state(online_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) err_state_prep:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) cpuhp_remove_state(CPUHP_PROFILE_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) subsys_initcall(create_proc_profile);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) #endif /* CONFIG_PROC_FS */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags