Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * SMP Support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * Copyright (C) 1999, 2001, 2003 David Mosberger-Tang <davidm@hpl.hp.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * Lots of stuff stolen from arch/alpha/kernel/smp.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * 01/05/16 Rohit Seth <rohit.seth@intel.com>  IA64-SMP functions. Reorganized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * the existing code (on the lines of x86 port).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * 00/09/11 David Mosberger <davidm@hpl.hp.com> Do loops_per_jiffy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * calibration on each CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * 00/08/23 Asit Mallick <asit.k.mallick@intel.com> fixed logical processor id
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * 00/03/31 Rohit Seth <rohit.seth@intel.com>	Fixes for Bootstrap Processor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * & cpu_online_map now gets done here (instead of setup.c)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * 99/10/05 davidm	Update to bring it in sync with new command-line processing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  *  scheme.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * 10/13/00 Goutham Rao <goutham.rao@intel.com> Updated smp_call_function and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  *		smp_call_function_single to resend IPI on timeouts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <linux/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #include <linux/efi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) #include <linux/bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #include <linux/kexec.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #include <asm/current.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #include <asm/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #include <asm/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #include <asm/sal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #include <asm/unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #include <asm/mca.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #include <asm/xtp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  * Note: alignment of 4 entries/cacheline was empirically determined
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52)  * to be a good tradeoff between hot cachelines & spreading the array
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53)  * across too many cacheline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) static struct local_tlb_flush_counts {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	unsigned int count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) } __attribute__((__aligned__(32))) local_tlb_flush_counts[NR_CPUS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned short [NR_CPUS],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 				     shadow_flush_counts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) #define IPI_CALL_FUNC		0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) #define IPI_CPU_STOP		1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) #define IPI_CALL_FUNC_SINGLE	2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) #define IPI_KDUMP_CPU_STOP	3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) /* This needs to be cacheline aligned because it is written to by *other* CPUs.  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) static DEFINE_PER_CPU_SHARED_ALIGNED(unsigned long, ipi_operation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) extern void cpu_halt (void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) stop_this_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	 * Remove this CPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	set_cpu_online(smp_processor_id(), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	max_xtp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	cpu_halt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) cpu_die(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	max_xtp();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	cpu_halt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	/* Should never be here */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	for (;;);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) irqreturn_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) handle_IPI (int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	int this_cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	unsigned long *pending_ipis = &__ia64_per_cpu_var(ipi_operation);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	unsigned long ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	mb();	/* Order interrupt and bit testing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	while ((ops = xchg(pending_ipis, 0)) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		mb();	/* Order bit clearing and data access. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 			unsigned long which;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 			which = ffz(~ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 			ops &= ~(1 << which);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 			switch (which) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 			case IPI_CPU_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 				stop_this_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 			case IPI_CALL_FUNC:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 				generic_smp_call_function_interrupt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 			case IPI_CALL_FUNC_SINGLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 				generic_smp_call_function_single_interrupt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #ifdef CONFIG_KEXEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 			case IPI_KDUMP_CPU_STOP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 				unw_init_running(kdump_cpu_freeze, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 			default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 				printk(KERN_CRIT "Unknown IPI on CPU %d: %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 						this_cpu, which);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		} while (ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 		mb();	/* Order data access and bit testing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) send_IPI_single (int dest_cpu, int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	set_bit(op, &per_cpu(ipi_operation, dest_cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	ia64_send_ipi(dest_cpu, IA64_IPI_VECTOR, IA64_IPI_DM_INT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) send_IPI_allbutself (int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 	for_each_online_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		if (i != smp_processor_id())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 			send_IPI_single(i, op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) send_IPI_mask(const struct cpumask *mask, int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	for_each_cpu(cpu, mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 			send_IPI_single(cpu, op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) send_IPI_all (int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	for_each_online_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		send_IPI_single(i, op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) send_IPI_self (int op)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	send_IPI_single(smp_processor_id(), op);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #ifdef CONFIG_KEXEC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) kdump_smp_send_stop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)  	send_IPI_allbutself(IPI_KDUMP_CPU_STOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) kdump_smp_send_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	unsigned int cpu, self_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	self_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		if (cpu != self_cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 			if(kdump_status[cpu] == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 				ia64_send_ipi(cpu, 0, IA64_IPI_DM_INIT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) smp_send_reschedule (int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	ia64_send_ipi(cpu, IA64_IPI_RESCHEDULE, IA64_IPI_DM_INT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) EXPORT_SYMBOL_GPL(smp_send_reschedule);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230)  * Called with preemption disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) static void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) smp_send_local_flush_tlb (int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	ia64_send_ipi(cpu, IA64_IPI_LOCAL_TLB_FLUSH, IA64_IPI_DM_INT, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) smp_local_flush_tlb(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	 * Use atomic ops. Otherwise, the load/increment/store sequence from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	 * a "++" operation can have the line stolen between the load & store.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 	 * The overhead of the atomic op in negligible in this case & offers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	 * significant benefit for the brief periods where lots of cpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	 * are simultaneously flushing TLBs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	ia64_fetchadd(1, &local_tlb_flush_counts[smp_processor_id()].count, acq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	local_flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) #define FLUSH_DELAY	5 /* Usec backoff to eliminate excessive cacheline bouncing */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) smp_flush_tlb_cpumask(cpumask_t xcpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	unsigned short *counts = __ia64_per_cpu_var(shadow_flush_counts);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	cpumask_t cpumask = xcpumask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 	int mycpu, cpu, flush_mycpu = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	mycpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	for_each_cpu(cpu, &cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 		counts[cpu] = local_tlb_flush_counts[cpu].count & 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	for_each_cpu(cpu, &cpumask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 		if (cpu == mycpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 			flush_mycpu = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 			smp_send_local_flush_tlb(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	if (flush_mycpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 		smp_local_flush_tlb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	for_each_cpu(cpu, &cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		while(counts[cpu] == (local_tlb_flush_counts[cpu].count & 0xffff))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 			udelay(FLUSH_DELAY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) smp_flush_tlb_all (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	on_each_cpu((void (*)(void *))local_flush_tlb_all, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) smp_flush_tlb_mm (struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 	cpumask_var_t cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	/* this happens for the common case of a single-threaded fork():  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	if (likely(mm == current->active_mm && atomic_read(&mm->mm_users) == 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	{
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		local_finish_flush_tlb_mm(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	if (!alloc_cpumask_var(&cpus, GFP_ATOMIC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 		smp_call_function((void (*)(void *))local_finish_flush_tlb_mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 			mm, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 		cpumask_copy(cpus, mm_cpumask(mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 		smp_call_function_many(cpus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 			(void (*)(void *))local_finish_flush_tlb_mm, mm, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 		free_cpumask_var(cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	local_finish_flush_tlb_mm(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 	preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) void arch_send_call_function_single_ipi(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 	send_IPI_single(cpu, IPI_CALL_FUNC_SINGLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) void arch_send_call_function_ipi_mask(const struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	send_IPI_mask(mask, IPI_CALL_FUNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)  * this function calls the 'stop' function on all other CPUs in the system.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) smp_send_stop (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	send_IPI_allbutself(IPI_CPU_STOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) int
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) setup_profiling_timer (unsigned int multiplier)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }