^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Derived from arch/i386/kernel/irq.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1992 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Adapted from arch/i386 by Gary Thomas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Updated and modified by Cort Dougan <cort@fsmlabs.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 1996-2001 Cort Dougan
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Adapted for Power Macintosh by Paul Mackerras
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Copyright (C) 1996 Paul Mackerras (paulus@cs.anu.edu.au)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * This file contains the code used by various IRQ handling routines:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * asking for different IRQ's should be done through these routines
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * instead of just grabbing them. Thus setups with different IRQ numbers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * shouldn't result in any weird surprises, and installing new handlers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * should be easier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * The MPC8xx has an interrupt mask in the SIU. If a bit is set, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * interrupt is _enabled_. As expected, IRQ0 is bit 0 in the 32-bit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * mask register (of which only 16 are defined), hence the weird shifting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * and complement of the cached_irq_mask. I want to be able to stuff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * this right into the SIU SMASK register.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * Many of the prep/chrp functions are conditional compiled on CONFIG_PPC_8xx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * to reduce code space and undefined function references.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #undef DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/threads.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/ioport.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/timex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/radix-tree.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/pci.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/of.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/of_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <asm/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <asm/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <asm/prom.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #include <asm/machdep.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <asm/udbg.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <asm/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <asm/livepatch.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <asm/asm-prototypes.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <asm/hw_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <asm/paca.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #include <asm/firmware.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #include <asm/lv1call.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #include <asm/dbell.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #define CREATE_TRACE_POINTS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #include <asm/trace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #include <asm/cpu_has_feature.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t, irq_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) EXPORT_PER_CPU_SYMBOL(irq_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) #ifdef CONFIG_PPC32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) atomic_t ppc_n_lost_interrupts;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #ifdef CONFIG_TAU_INT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) extern int tau_initialized;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) u32 tau_interrupts(unsigned long cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) #endif /* CONFIG_PPC32 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) int distribute_irqs = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) static inline notrace unsigned long get_irq_happened(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) unsigned long happened;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) __asm__ __volatile__("lbz %0,%1(13)"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) : "=r" (happened) : "i" (offsetof(struct paca_struct, irq_happened)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) return happened;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #ifdef CONFIG_PPC_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* This is called whenever we are re-enabling interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * and returns either 0 (nothing to do) or 500/900/280 if
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * there's an EE, DEC or DBELL to generate.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * This is called in two contexts: From arch_local_irq_restore()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * before soft-enabling interrupts, and from the exception exit
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * path when returning from an interrupt from a soft-disabled to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * a soft enabled context. In both case we have interrupts hard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) * We take care of only clearing the bits we handled in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * PACA irq_happened field since we can only re-emit one at a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * time and we don't want to "lose" one.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) notrace unsigned int __check_irq_replay(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * We use local_paca rather than get_paca() to avoid all
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * the debug_smp_processor_id() business in this low level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) * function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) unsigned char happened = local_paca->irq_happened;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * We are responding to the next interrupt, so interrupt-off
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * latencies should be reset here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) trace_hardirqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) trace_hardirqs_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) if (happened & PACA_IRQ_DEC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) local_paca->irq_happened &= ~PACA_IRQ_DEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) return 0x900;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) if (happened & PACA_IRQ_EE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) local_paca->irq_happened &= ~PACA_IRQ_EE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) return 0x500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) if (happened & PACA_IRQ_DBELL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) local_paca->irq_happened &= ~PACA_IRQ_DBELL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) return 0x280;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) if (happened & PACA_IRQ_HARD_DIS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) /* There should be nothing left ! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) BUG_ON(local_paca->irq_happened != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * This is specifically called by assembly code to re-enable interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * if they are currently disabled. This is typically called before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * schedule() or do_signal() when returning to userspace. We do it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) * in C to avoid the burden of dealing with lockdep etc...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) * NOTE: This is called with interrupts hard disabled but not marked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) * as such in paca->irq_happened, so we need to resync this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void notrace restore_interrupts(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) if (irqs_disabled()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) __hard_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) #endif /* CONFIG_PPC_BOOK3E */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) void replay_soft_interrupts(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) struct pt_regs regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) * Be careful here, calling these interrupt handlers can cause
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) * softirqs to be raised, which they may run when calling irq_exit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * which will cause local_irq_enable() to be run, which can then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * recurse into this function. Don't keep any state across
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) * interrupt handler calls which may change underneath us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) * We use local_paca rather than get_paca() to avoid all the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) * debug_smp_processor_id() business in this low level function.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) ppc_save_regs(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) regs.softe = IRQS_ENABLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) WARN_ON_ONCE(mfmsr() & MSR_EE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * Force the delivery of pending soft-disabled interrupts on PS3.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * Any HV call will have this side effect.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) if (firmware_has_feature(FW_FEATURE_PS3_LV1)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) u64 tmp, tmp2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) lv1_get_version_info(&tmp, &tmp2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Check if an hypervisor Maintenance interrupt happened.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) * This is a higher priority interrupt than the others, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * replay it first.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_HMI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) local_paca->irq_happened &= ~PACA_IRQ_HMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) regs.trap = 0xe60;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) handle_hmi_exception(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (local_paca->irq_happened & PACA_IRQ_DEC) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) local_paca->irq_happened &= ~PACA_IRQ_DEC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) regs.trap = 0x900;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) timer_interrupt(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) if (local_paca->irq_happened & PACA_IRQ_EE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) local_paca->irq_happened &= ~PACA_IRQ_EE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) regs.trap = 0x500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) do_IRQ(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (IS_ENABLED(CONFIG_PPC_DOORBELL) && (local_paca->irq_happened & PACA_IRQ_DBELL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) local_paca->irq_happened &= ~PACA_IRQ_DBELL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (IS_ENABLED(CONFIG_PPC_BOOK3E))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) regs.trap = 0x280;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) regs.trap = 0xa00;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) doorbell_exception(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /* Book3E does not support soft-masking PMI interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) if (IS_ENABLED(CONFIG_PPC_BOOK3S) && (local_paca->irq_happened & PACA_IRQ_PMI)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) local_paca->irq_happened &= ~PACA_IRQ_PMI;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) regs.trap = 0xf00;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) performance_monitor_exception(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) if (!(local_paca->irq_happened & PACA_IRQ_HARD_DIS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) if (local_paca->irq_happened & ~PACA_IRQ_HARD_DIS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * We are responding to the next interrupt, so interrupt-off
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * latencies should be reset here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) trace_hardirqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) trace_hardirqs_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #if defined(CONFIG_PPC_BOOK3S_64) && defined(CONFIG_PPC_KUAP)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) static inline void replay_soft_interrupts_irqrestore(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) unsigned long kuap_state = get_kuap();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) * Check if anything calls local_irq_enable/restore() when KUAP is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) * disabled (user access enabled). We handle that case here by saving
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) * and re-locking AMR but we shouldn't get here in the first place,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * hence the warning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) kuap_check_amr();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) if (kuap_state != AMR_KUAP_BLOCKED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) set_kuap(AMR_KUAP_BLOCKED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) replay_soft_interrupts();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (kuap_state != AMR_KUAP_BLOCKED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) set_kuap(kuap_state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #define replay_soft_interrupts_irqrestore() replay_soft_interrupts()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) notrace void arch_local_irq_restore(unsigned long mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) unsigned char irq_happened;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) /* Write the new soft-enabled value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) irq_soft_mask_set(mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) if (mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * From this point onward, we can take interrupts, preempt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * etc... unless we got hard-disabled. We check if an event
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * happened. If none happened, we know we can just return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * We may have preempted before the check below, in which case
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) * we are checking the "new" CPU instead of the old one. This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * is only a problem if an event happened on the "old" CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * External interrupt events will have caused interrupts to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) * be hard-disabled, so there is no problem, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) * cannot have preempted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) irq_happened = get_irq_happened();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) if (!irq_happened) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) WARN_ON_ONCE(!(mfmsr() & MSR_EE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) /* We need to hard disable to replay. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) if (!(irq_happened & PACA_IRQ_HARD_DIS)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) WARN_ON_ONCE(!(mfmsr() & MSR_EE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) __hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * We should already be hard disabled here. We had bugs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * where that wasn't the case so let's dbl check it and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * warn if we are wrong. Only do that when IRQ tracing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) * is enabled as mfmsr() can be costly.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) if (IS_ENABLED(CONFIG_PPC_IRQ_SOFT_MASK_DEBUG)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) if (WARN_ON_ONCE(mfmsr() & MSR_EE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) __hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) if (irq_happened == PACA_IRQ_HARD_DIS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) local_paca->irq_happened = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) __hard_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * Disable preempt here, so that the below preempt_enable will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) * perform resched if required (a replayed interrupt may set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * need_resched).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) irq_soft_mask_set(IRQS_ALL_DISABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) trace_hardirqs_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) replay_soft_interrupts_irqrestore();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) local_paca->irq_happened = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) trace_hardirqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) irq_soft_mask_set(IRQS_ENABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) __hard_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) EXPORT_SYMBOL(arch_local_irq_restore);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * This is a helper to use when about to go into idle low-power
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * when the latter has the side effect of re-enabling interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * (such as calling H_CEDE under pHyp).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * You call this function with interrupts soft-disabled (this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * already the case when ppc_md.power_save is called). The function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * will return whether to enter power save or just return.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * In the former case, it will have notified lockdep of interrupts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) * being re-enabled and generally sanitized the lazy irq state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) * and in the latter case it will leave with interrupts hard
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) * disabled and marked as such, so the local_irq_enable() call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) * in arch_cpu_idle() will properly re-enable everything.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) bool prep_irq_for_idle(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) * First we need to hard disable to ensure no interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * occurs before we effectively enter the low power state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) __hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) * If anything happened while we were soft-disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * we return now and do not enter the low power state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) if (lazy_irq_pending())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) /* Tell lockdep we are about to re-enable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) trace_hardirqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) * Mark interrupts as soft-enabled and clear the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) * PACA_IRQ_HARD_DIS from the pending mask since we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) * are about to hard enable as well as a side effect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) * of entering the low power state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) irq_soft_mask_set(IRQS_ENABLED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) /* Tell the caller to enter the low power state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) #ifdef CONFIG_PPC_BOOK3S
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * This is for idle sequences that return with IRQs off, but the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * idle state itself wakes on interrupt. Tell the irq tracer that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * IRQs are enabled for the duration of idle so it does not get long
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * off times. Must be paired with fini_irq_for_idle_irqsoff.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) bool prep_irq_for_idle_irqsoff(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) WARN_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * First we need to hard disable to ensure no interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * occurs before we effectively enter the low power state
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) __hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) * If anything happened while we were soft-disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) * we return now and do not enter the low power state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (lazy_irq_pending())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) /* Tell lockdep we are about to re-enable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) trace_hardirqs_on();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * Take the SRR1 wakeup reason, index into this table to find the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) * appropriate irq_happened bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * Sytem reset exceptions taken in idle state also come through here,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * but they are NMI interrupts so do not need to wait for IRQs to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) * restored, and should be taken as early as practical. These are marked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * with 0xff in the table. The Power ISA specifies 0100b as the system
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * reset interrupt reason.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) #define IRQ_SYSTEM_RESET 0xff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) static const u8 srr1_to_lazyirq[0x10] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) 0, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) PACA_IRQ_DBELL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) IRQ_SYSTEM_RESET,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) PACA_IRQ_DBELL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) PACA_IRQ_DEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) PACA_IRQ_EE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) PACA_IRQ_EE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) PACA_IRQ_HMI,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) 0, 0, 0, 0, 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) void replay_system_reset(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) struct pt_regs regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) ppc_save_regs(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) regs.trap = 0x100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) get_paca()->in_nmi = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) system_reset_exception(®s);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) get_paca()->in_nmi = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) EXPORT_SYMBOL_GPL(replay_system_reset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) void irq_set_pending_from_srr1(unsigned long srr1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) unsigned int idx = (srr1 & SRR1_WAKEMASK_P8) >> 18;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) u8 reason = srr1_to_lazyirq[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * Take the system reset now, which is immediately after registers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * are restored from idle. It's an NMI, so interrupts need not be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * re-enabled before it is taken.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (unlikely(reason == IRQ_SYSTEM_RESET)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) replay_system_reset();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) if (reason == PACA_IRQ_DBELL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) * When doorbell triggers a system reset wakeup, the message
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) * is not cleared, so if the doorbell interrupt is replayed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) * and the IPI handled, the doorbell interrupt would still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) * fire when EE is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * To avoid taking the superfluous doorbell interrupt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * execute a msgclr here before the interrupt is replayed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) ppc_msgclr(PPC_DBELL_MSGTYPE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * The 0 index (SRR1[42:45]=b0000) must always evaluate to 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * so this can be called unconditionally with the SRR1 wake
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) * reason as returned by the idle code, which uses 0 to mean no
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) * interrupt.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * If a future CPU was to designate this as an interrupt reason,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * then a new index for no interrupt must be assigned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) local_paca->irq_happened |= reason;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) #endif /* CONFIG_PPC_BOOK3S */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) * Force a replay of the external interrupt handler on this CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) void force_external_irq_replay(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * This must only be called with interrupts soft-disabled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * the replay will happen when re-enabling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) WARN_ON(!arch_irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * Interrupts must always be hard disabled before irq_happened is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * modified (to prevent lost update in case of interrupt between
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) * load and store).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) __hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) local_paca->irq_happened |= PACA_IRQ_HARD_DIS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) /* Indicate in the PACA that we have an interrupt to replay */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) local_paca->irq_happened |= PACA_IRQ_EE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) #endif /* CONFIG_PPC64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) int arch_show_interrupts(struct seq_file *p, int prec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) #if defined(CONFIG_PPC32) && defined(CONFIG_TAU_INT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (tau_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) seq_printf(p, "%*s: ", prec, "TAU");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) seq_printf(p, "%10u ", tau_interrupts(j));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) seq_puts(p, " PowerPC Thermal Assist (cpu temp)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) #endif /* CONFIG_PPC32 && CONFIG_TAU_INT */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) seq_printf(p, "%*s: ", prec, "LOC");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) seq_printf(p, " Local timer interrupts for timer event device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) seq_printf(p, "%*s: ", prec, "BCT");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) seq_printf(p, "%10u ", per_cpu(irq_stat, j).broadcast_irqs_event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) seq_printf(p, " Broadcast timer interrupts for timer event device\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) seq_printf(p, "%*s: ", prec, "LOC");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) seq_printf(p, "%10u ", per_cpu(irq_stat, j).timer_irqs_others);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) seq_printf(p, " Local timer interrupts for others\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) seq_printf(p, "%*s: ", prec, "SPU");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) seq_printf(p, "%10u ", per_cpu(irq_stat, j).spurious_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) seq_printf(p, " Spurious interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) seq_printf(p, "%*s: ", prec, "PMI");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) seq_printf(p, "%10u ", per_cpu(irq_stat, j).pmu_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) seq_printf(p, " Performance monitoring interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) seq_printf(p, "%*s: ", prec, "MCE");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) seq_printf(p, "%10u ", per_cpu(irq_stat, j).mce_exceptions);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) seq_printf(p, " Machine check exceptions\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (cpu_has_feature(CPU_FTR_HVMODE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) seq_printf(p, "%*s: ", prec, "HMI");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) seq_printf(p, "%10u ", paca_ptrs[j]->hmi_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) seq_printf(p, " Hypervisor Maintenance Interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) seq_printf(p, "%*s: ", prec, "NMI");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) seq_printf(p, "%10u ", per_cpu(irq_stat, j).sreset_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) seq_printf(p, " System Reset interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) #ifdef CONFIG_PPC_WATCHDOG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) seq_printf(p, "%*s: ", prec, "WDG");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) seq_printf(p, "%10u ", per_cpu(irq_stat, j).soft_nmi_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) seq_printf(p, " Watchdog soft-NMI interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) #ifdef CONFIG_PPC_DOORBELL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) if (cpu_has_feature(CPU_FTR_DBELL)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) seq_printf(p, "%*s: ", prec, "DBL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) for_each_online_cpu(j)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) seq_printf(p, "%10u ", per_cpu(irq_stat, j).doorbell_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) seq_printf(p, " Doorbell interrupts\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) * /proc/stat helpers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) u64 arch_irq_stat_cpu(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) u64 sum = per_cpu(irq_stat, cpu).timer_irqs_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) sum += per_cpu(irq_stat, cpu).broadcast_irqs_event;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) sum += per_cpu(irq_stat, cpu).pmu_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) sum += per_cpu(irq_stat, cpu).mce_exceptions;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) sum += per_cpu(irq_stat, cpu).spurious_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) sum += per_cpu(irq_stat, cpu).timer_irqs_others;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) sum += paca_ptrs[cpu]->hmi_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) sum += per_cpu(irq_stat, cpu).sreset_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) #ifdef CONFIG_PPC_WATCHDOG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) sum += per_cpu(irq_stat, cpu).soft_nmi_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) #ifdef CONFIG_PPC_DOORBELL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) sum += per_cpu(irq_stat, cpu).doorbell_irqs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) return sum;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) static inline void check_stack_overflow(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) long sp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) if (!IS_ENABLED(CONFIG_DEBUG_STACKOVERFLOW))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) sp = current_stack_pointer & (THREAD_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) /* check for stack overflow: is there less than 2KB free? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) if (unlikely(sp < 2048)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) pr_err("do_IRQ: stack overflow: %ld\n", sp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) void __do_irq(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) unsigned int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) irq_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) trace_irq_entry(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) * Query the platform PIC for the interrupt & ack it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * This will typically lower the interrupt line to the CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) irq = ppc_md.get_irq();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* We can hard enable interrupts now to allow perf interrupts */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) may_hard_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) /* And finally process it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (unlikely(!irq))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) __this_cpu_inc(irq_stat.spurious_irqs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) generic_handle_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) trace_irq_exit(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) irq_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) void do_IRQ(struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) struct pt_regs *old_regs = set_irq_regs(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) void *cursp, *irqsp, *sirqsp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) /* Switch to the irq stack to handle this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) cursp = (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) irqsp = hardirq_ctx[raw_smp_processor_id()];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) sirqsp = softirq_ctx[raw_smp_processor_id()];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) check_stack_overflow();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) /* Already there ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (unlikely(cursp == irqsp || cursp == sirqsp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) __do_irq(regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) set_irq_regs(old_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) /* Switch stack and call */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) call_do_irq(regs, irqsp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) set_irq_regs(old_regs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) static void *__init alloc_vm_stack(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) return __vmalloc_node(THREAD_SIZE, THREAD_ALIGN, THREADINFO_GFP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) NUMA_NO_NODE, (void *)_RET_IP_);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static void __init vmap_irqstack_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) for_each_possible_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) softirq_ctx[i] = alloc_vm_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) hardirq_ctx[i] = alloc_vm_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) void __init init_IRQ(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) if (IS_ENABLED(CONFIG_VMAP_STACK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) vmap_irqstack_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (ppc_md.init_IRQ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) ppc_md.init_IRQ();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) #if defined(CONFIG_BOOKE) || defined(CONFIG_40x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) void *critirq_ctx[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) void *dbgirq_ctx[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) void *mcheckirq_ctx[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) void *softirq_ctx[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) void *hardirq_ctx[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) void do_softirq_own_stack(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) call_do_softirq(softirq_ctx[smp_processor_id()]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) irq_hw_number_t virq_to_hw(unsigned int virq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) struct irq_data *irq_data = irq_get_irq_data(virq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return WARN_ON(!irq_data) ? 0 : irq_data->hwirq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) EXPORT_SYMBOL_GPL(virq_to_hw);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) int irq_choose_cpu(const struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) int cpuid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) if (cpumask_equal(mask, cpu_online_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) static int irq_rover;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) static DEFINE_RAW_SPINLOCK(irq_rover_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /* Round-robin distribution... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) do_round_robin:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) raw_spin_lock_irqsave(&irq_rover_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) irq_rover = cpumask_next(irq_rover, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (irq_rover >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) irq_rover = cpumask_first(cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) cpuid = irq_rover;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) raw_spin_unlock_irqrestore(&irq_rover_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) cpuid = cpumask_first_and(mask, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) if (cpuid >= nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) goto do_round_robin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return get_hard_smp_processor_id(cpuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) int irq_choose_cpu(const struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) return hard_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) static int __init setup_noirqdistrib(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) distribute_irqs = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) __setup("noirqdistrib", setup_noirqdistrib);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) #endif /* CONFIG_PPC64 */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags