Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Interrupt descriptor table related code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <asm/cpu_entry_area.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/set_memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <asm/proto.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/desc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/hw_irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #define DPL0		0x0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #define DPL3		0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define DEFAULT_STACK	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define G(_vector, _addr, _ist, _type, _dpl, _segment)	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	{						\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 		.vector		= _vector,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 		.bits.ist	= _ist,			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 		.bits.type	= _type,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 		.bits.dpl	= _dpl,			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 		.bits.p		= 1,			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 		.addr		= _addr,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 		.segment	= _segment,		\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) /* Interrupt gate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #define INTG(_vector, _addr)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 	G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL0, __KERNEL_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) /* System interrupt gate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) #define SYSG(_vector, _addr)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	G(_vector, _addr, DEFAULT_STACK, GATE_INTERRUPT, DPL3, __KERNEL_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * Interrupt gate with interrupt stack. The _ist index is the index in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * the tss.ist[] array, but for the descriptor it needs to start at 1.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #define ISTG(_vector, _addr, _ist)			\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 	G(_vector, _addr, _ist + 1, GATE_INTERRUPT, DPL0, __KERNEL_CS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) /* Task gate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #define TSKG(_vector, _gdt)				\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	G(_vector, NULL, DEFAULT_STACK, GATE_TASK, DPL0, _gdt << 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) #define IDT_TABLE_SIZE		(IDT_ENTRIES * sizeof(gate_desc))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) static bool idt_setup_done __initdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  * Early traps running on the DEFAULT_STACK because the other interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  * stacks work only after cpu_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) static const __initconst struct idt_data early_idts[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	INTG(X86_TRAP_DB,		asm_exc_debug),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	SYSG(X86_TRAP_BP,		asm_exc_int3),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	 * Not possible on 64-bit. See idt_setup_early_pf() for details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	INTG(X86_TRAP_PF,		asm_exc_page_fault),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  * The default IDT entries which are set up in trap_init() before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  * cpu_init() is invoked. Interrupt stacks cannot be used at that point and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72)  * the traps which use them are reinitialized with IST after cpu_init() has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73)  * set up TSS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) static const __initconst struct idt_data def_idts[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	INTG(X86_TRAP_DE,		asm_exc_divide_error),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	INTG(X86_TRAP_NMI,		asm_exc_nmi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	INTG(X86_TRAP_BR,		asm_exc_bounds),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	INTG(X86_TRAP_UD,		asm_exc_invalid_op),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	INTG(X86_TRAP_NM,		asm_exc_device_not_available),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	INTG(X86_TRAP_OLD_MF,		asm_exc_coproc_segment_overrun),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	INTG(X86_TRAP_TS,		asm_exc_invalid_tss),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	INTG(X86_TRAP_NP,		asm_exc_segment_not_present),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	INTG(X86_TRAP_SS,		asm_exc_stack_segment),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	INTG(X86_TRAP_GP,		asm_exc_general_protection),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	INTG(X86_TRAP_SPURIOUS,		asm_exc_spurious_interrupt_bug),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	INTG(X86_TRAP_MF,		asm_exc_coprocessor_error),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	INTG(X86_TRAP_AC,		asm_exc_alignment_check),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	INTG(X86_TRAP_XF,		asm_exc_simd_coprocessor_error),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	TSKG(X86_TRAP_DF,		GDT_ENTRY_DOUBLEFAULT_TSS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	INTG(X86_TRAP_DF,		asm_exc_double_fault),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	INTG(X86_TRAP_DB,		asm_exc_debug),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) #ifdef CONFIG_X86_MCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	INTG(X86_TRAP_MC,		asm_exc_machine_check),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	SYSG(X86_TRAP_OF,		asm_exc_overflow),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) #if defined(CONFIG_IA32_EMULATION)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 	SYSG(IA32_SYSCALL_VECTOR,	entry_INT80_compat),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #elif defined(CONFIG_X86_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	SYSG(IA32_SYSCALL_VECTOR,	entry_INT80_32),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * The APIC and SMP idt entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) static const __initconst struct idt_data apic_idts[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	INTG(RESCHEDULE_VECTOR,			asm_sysvec_reschedule_ipi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	INTG(CALL_FUNCTION_VECTOR,		asm_sysvec_call_function),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	INTG(CALL_FUNCTION_SINGLE_VECTOR,	asm_sysvec_call_function_single),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	INTG(IRQ_MOVE_CLEANUP_VECTOR,		asm_sysvec_irq_move_cleanup),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	INTG(REBOOT_VECTOR,			asm_sysvec_reboot),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) #ifdef CONFIG_X86_THERMAL_VECTOR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	INTG(THERMAL_APIC_VECTOR,		asm_sysvec_thermal),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #ifdef CONFIG_X86_MCE_THRESHOLD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	INTG(THRESHOLD_APIC_VECTOR,		asm_sysvec_threshold),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) #ifdef CONFIG_X86_MCE_AMD
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	INTG(DEFERRED_ERROR_VECTOR,		asm_sysvec_deferred_error),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #ifdef CONFIG_X86_LOCAL_APIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	INTG(LOCAL_TIMER_VECTOR,		asm_sysvec_apic_timer_interrupt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	INTG(X86_PLATFORM_IPI_VECTOR,		asm_sysvec_x86_platform_ipi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) # ifdef CONFIG_HAVE_KVM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	INTG(POSTED_INTR_VECTOR,		asm_sysvec_kvm_posted_intr_ipi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	INTG(POSTED_INTR_WAKEUP_VECTOR,		asm_sysvec_kvm_posted_intr_wakeup_ipi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	INTG(POSTED_INTR_NESTED_VECTOR,		asm_sysvec_kvm_posted_intr_nested_ipi),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) # ifdef CONFIG_IRQ_WORK
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	INTG(IRQ_WORK_VECTOR,			asm_sysvec_irq_work),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	INTG(SPURIOUS_APIC_VECTOR,		asm_sysvec_spurious_apic_interrupt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	INTG(ERROR_APIC_VECTOR,			asm_sysvec_error_interrupt),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* Must be page-aligned because the real IDT is used in the cpu entry area */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static gate_desc idt_table[IDT_ENTRIES] __page_aligned_bss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static struct desc_ptr idt_descr __ro_after_init = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	.size		= IDT_TABLE_SIZE - 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	.address	= (unsigned long) idt_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) void load_current_idt(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	lockdep_assert_irqs_disabled();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	load_idt(&idt_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #ifdef CONFIG_X86_F00F_BUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) bool idt_is_f00f_address(unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	return ((address - idt_descr.address) >> 3) == 6;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static __init void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) idt_setup_from_table(gate_desc *idt, const struct idt_data *t, int size, bool sys)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	gate_desc desc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	for (; size > 0; t++, size--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 		idt_init_desc(&desc, t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		write_idt_entry(idt, t->vector, &desc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		if (sys)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 			set_bit(t->vector, system_vectors);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) static __init void set_intr_gate(unsigned int n, const void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	struct idt_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	init_idt_data(&data, n, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	idt_setup_from_table(idt_table, &data, 1, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)  * idt_setup_early_traps - Initialize the idt table with early traps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)  * On X8664 these traps do not use interrupt stacks as they can't work
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197)  * before cpu_init() is invoked and sets up TSS. The IST variants are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)  * installed after that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void __init idt_setup_early_traps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	idt_setup_from_table(idt_table, early_idts, ARRAY_SIZE(early_idts),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 			     true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	load_idt(&idt_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)  * idt_setup_traps - Initialize the idt table with default traps
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) void __init idt_setup_traps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 	idt_setup_from_table(idt_table, def_idts, ARRAY_SIZE(def_idts), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) #ifdef CONFIG_X86_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  * Early traps running on the DEFAULT_STACK because the other interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  * stacks work only after cpu_init().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) static const __initconst struct idt_data early_pf_idts[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	INTG(X86_TRAP_PF,		asm_exc_page_fault),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)  * The exceptions which use Interrupt stacks. They are setup after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)  * cpu_init() when the TSS has been initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) static const __initconst struct idt_data ist_idts[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	ISTG(X86_TRAP_DB,	asm_exc_debug,			IST_INDEX_DB),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	ISTG(X86_TRAP_NMI,	asm_exc_nmi,			IST_INDEX_NMI),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	ISTG(X86_TRAP_DF,	asm_exc_double_fault,		IST_INDEX_DF),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #ifdef CONFIG_X86_MCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	ISTG(X86_TRAP_MC,	asm_exc_machine_check,		IST_INDEX_MCE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) #ifdef CONFIG_AMD_MEM_ENCRYPT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	ISTG(X86_TRAP_VC,	asm_exc_vmm_communication,	IST_INDEX_VC),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)  * idt_setup_early_pf - Initialize the idt table with early pagefault handler
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)  * On X8664 this does not use interrupt stacks as they can't work before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)  * cpu_init() is invoked and sets up TSS. The IST variant is installed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)  * after that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)  * Note, that X86_64 cannot install the real #PF handler in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)  * idt_setup_early_traps() because the memory intialization needs the #PF
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)  * handler from the early_idt_handler_array to initialize the early page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)  * tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) void __init idt_setup_early_pf(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	idt_setup_from_table(idt_table, early_pf_idts,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 			     ARRAY_SIZE(early_pf_idts), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)  * idt_setup_ist_traps - Initialize the idt table with traps using IST
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) void __init idt_setup_ist_traps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	idt_setup_from_table(idt_table, ist_idts, ARRAY_SIZE(ist_idts), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) static void __init idt_map_in_cea(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	 * Set the IDT descriptor to a fixed read-only location in the cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	 * entry area, so that the "sidt" instruction will not leak the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	 * location of the kernel, and to defend the IDT against arbitrary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	 * memory write vulnerabilities.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	cea_set_pte(CPU_ENTRY_AREA_RO_IDT_VADDR, __pa_symbol(idt_table),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 		    PAGE_KERNEL_RO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	idt_descr.address = CPU_ENTRY_AREA_RO_IDT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)  * idt_setup_apic_and_irq_gates - Setup APIC/SMP and normal interrupt gates
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) void __init idt_setup_apic_and_irq_gates(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 	int i = FIRST_EXTERNAL_VECTOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	idt_setup_from_table(idt_table, apic_idts, ARRAY_SIZE(apic_idts), true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	for_each_clear_bit_from(i, system_vectors, FIRST_SYSTEM_VECTOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 		entry = irq_entries_start + 8 * (i - FIRST_EXTERNAL_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		set_intr_gate(i, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) #ifdef CONFIG_X86_LOCAL_APIC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	for_each_clear_bit_from(i, system_vectors, NR_VECTORS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 		 * Don't set the non assigned system vectors in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		 * system_vectors bitmap. Otherwise they show up in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		 * /proc/interrupts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		entry = spurious_entries_start + 8 * (i - FIRST_SYSTEM_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 		set_intr_gate(i, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	/* Map IDT into CPU entry area and reload it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	idt_map_in_cea();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	load_idt(&idt_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	/* Make the IDT table read only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	set_memory_ro((unsigned long)&idt_table, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	idt_setup_done = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)  * idt_setup_early_handler - Initializes the idt table with early handlers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) void __init idt_setup_early_handler(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	for (i = 0; i < NUM_EXCEPTION_VECTORS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		set_intr_gate(i, early_idt_handler_array[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) #ifdef CONFIG_X86_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	for ( ; i < NR_VECTORS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 		set_intr_gate(i, early_ignore_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 	load_idt(&idt_descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)  * idt_invalidate - Invalidate interrupt descriptor table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)  * @addr:	The virtual address of the 'invalid' IDT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) void idt_invalidate(void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 	struct desc_ptr idt = { .address = (unsigned long) addr, .size = 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	load_idt(&idt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) void __init alloc_intr_gate(unsigned int n, const void *addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	if (WARN_ON(n < FIRST_SYSTEM_VECTOR))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 	if (WARN_ON(idt_setup_done))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 	if (!WARN_ON(test_and_set_bit(n, system_vectors)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 		set_intr_gate(n, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) }