^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /* smp.c: Sparc64 SMP support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (C) 1997, 2007, 2008 David S. Miller (davem@davemloft.net)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/sched/hotplug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/threads.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/kernel_stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/cache.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/jiffies.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/profile.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/ftrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/kgdb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/head.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/cpudata.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/hvtramp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/timer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/setup.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/irq_regs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <asm/oplib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <asm/starfire.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <asm/prom.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <asm/mdesc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <asm/ldc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <asm/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <asm/pcr.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include "cpumap.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include "kernel.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) DEFINE_PER_CPU(cpumask_t, cpu_sibling_map) = CPU_MASK_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) cpumask_t cpu_core_map[NR_CPUS] __read_mostly =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) { [0 ... NR_CPUS-1] = CPU_MASK_NONE };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) cpumask_t cpu_core_sib_map[NR_CPUS] __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) [0 ... NR_CPUS-1] = CPU_MASK_NONE };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) cpumask_t cpu_core_sib_cache_map[NR_CPUS] __read_mostly = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) [0 ... NR_CPUS - 1] = CPU_MASK_NONE };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) EXPORT_PER_CPU_SYMBOL(cpu_sibling_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) EXPORT_SYMBOL(cpu_core_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) EXPORT_SYMBOL(cpu_core_sib_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) EXPORT_SYMBOL(cpu_core_sib_cache_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static cpumask_t smp_commenced_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) static DEFINE_PER_CPU(bool, poke);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static bool cpu_poke;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) void smp_info(struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) seq_printf(m, "State:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) for_each_online_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) seq_printf(m, "CPU%d:\t\tonline\n", i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) void smp_bogo(struct seq_file *m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) for_each_online_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) seq_printf(m,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) "Cpu%dClkTck\t: %016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) i, cpu_data(i).clock_tick);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) extern void setup_sparc64_timer(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static volatile unsigned long callin_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) void smp_callin(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) int cpuid = hard_smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) __local_per_cpu_offset = __per_cpu_offset(cpuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) sun4v_ktsb_register();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) __flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) setup_sparc64_timer();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) if (cheetah_pcache_forced_on)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) cheetah_enable_pcache();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) callin_flag = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) __asm__ __volatile__("membar #Sync\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) "flush %%g6" : : : "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) /* Clear this or we will die instantly when we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * schedule back to this idler...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) current_thread_info()->new_child = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) /* Attach to the address space of init_task. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) mmgrab(&init_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) current->active_mm = &init_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* inform the notifiers about the new cpu */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) notify_cpu_starting(cpuid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) while (!cpumask_test_cpu(cpuid, &smp_commenced_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) set_cpu_online(cpuid, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) void cpu_panic(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) printk("CPU[%d]: Returns from cpu_idle!\n", smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) panic("SMP bolixed\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* This tick register synchronization scheme is taken entirely from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * the ia64 port, see arch/ia64/kernel/smpboot.c for details and credit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) * The only change I've made is to rework it so that the master
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) * initiates the synchonization instead of the slave. -DaveM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #define MASTER 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define SLAVE (SMP_CACHE_BYTES/sizeof(unsigned long))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #define NUM_ROUNDS 64 /* magic value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #define NUM_ITERS 5 /* likewise */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) static DEFINE_RAW_SPINLOCK(itc_sync_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static unsigned long go[SLAVE + 1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #define DEBUG_TICK_SYNC 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) static inline long get_delta (long *rt, long *master)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) unsigned long best_t0 = 0, best_t1 = ~0UL, best_tm = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) unsigned long tcenter, t0, t1, tm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) for (i = 0; i < NUM_ITERS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) t0 = tick_ops->get_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) go[MASTER] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) membar_safe("#StoreLoad");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) while (!(tm = go[SLAVE]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) go[SLAVE] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) t1 = tick_ops->get_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if (t1 - t0 < best_t1 - best_t0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) best_t0 = t0, best_t1 = t1, best_tm = tm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) *rt = best_t1 - best_t0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) *master = best_tm - best_t0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* average best_t0 and best_t1 without overflow: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) tcenter = (best_t0/2 + best_t1/2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (best_t0 % 2 + best_t1 % 2 == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) tcenter++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) return tcenter - best_tm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void smp_synchronize_tick_client(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) long i, delta, adj, adjust_latency = 0, done = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) unsigned long flags, rt, master_time_stamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) #if DEBUG_TICK_SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) long rt; /* roundtrip time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) long master; /* master's timestamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) long diff; /* difference between midpoint and master's timestamp */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) long lat; /* estimate of itc adjustment latency */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) } t[NUM_ROUNDS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) go[MASTER] = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) while (go[MASTER])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) for (i = 0; i < NUM_ROUNDS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) delta = get_delta(&rt, &master_time_stamp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (delta == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) done = 1; /* let's lock on to this... */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) if (!done) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) if (i > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) adjust_latency += -delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) adj = -delta + adjust_latency/4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) adj = -delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) tick_ops->add_tick(adj);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) #if DEBUG_TICK_SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) t[i].rt = rt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) t[i].master = master_time_stamp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) t[i].diff = delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) t[i].lat = adjust_latency/4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) #if DEBUG_TICK_SYNC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) for (i = 0; i < NUM_ROUNDS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) printk("rt=%5ld master=%5ld diff=%5ld adjlat=%5ld\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) t[i].rt, t[i].master, t[i].diff, t[i].lat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) printk(KERN_INFO "CPU %d: synchronized TICK with master CPU "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) "(last diff %ld cycles, maxerr %lu cycles)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) smp_processor_id(), delta, rt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) static void smp_start_sync_tick_client(int cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) static void smp_synchronize_one_tick(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) unsigned long flags, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) go[MASTER] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) smp_start_sync_tick_client(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) /* wait for client to be ready */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) while (!go[MASTER])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /* now let the client proceed into his loop */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) go[MASTER] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) membar_safe("#StoreLoad");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) raw_spin_lock_irqsave(&itc_sync_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) for (i = 0; i < NUM_ROUNDS*NUM_ITERS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) while (!go[MASTER])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) go[MASTER] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) go[SLAVE] = tick_ops->get_tick();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) membar_safe("#StoreLoad");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) raw_spin_unlock_irqrestore(&itc_sync_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) static void ldom_startcpu_cpuid(unsigned int cpu, unsigned long thread_reg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) void **descrp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) extern unsigned long sparc64_ttable_tl0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) extern unsigned long kern_locked_tte_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) struct hvtramp_descr *hdesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) unsigned long trampoline_ra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) struct trap_per_cpu *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) u64 tte_vaddr, tte_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) hdesc = kzalloc(sizeof(*hdesc) +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) (sizeof(struct hvtramp_mapping) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) num_kernel_image_mappings - 1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (!hdesc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) printk(KERN_ERR "ldom_startcpu_cpuid: Cannot allocate "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) "hvtramp_descr.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) *descrp = hdesc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) hdesc->cpu = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) hdesc->num_mappings = num_kernel_image_mappings;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) tb = &trap_block[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) hdesc->fault_info_va = (unsigned long) &tb->fault_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) hdesc->fault_info_pa = kimage_addr_to_ra(&tb->fault_info);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) hdesc->thread_reg = thread_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) tte_vaddr = (unsigned long) KERNBASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) tte_data = kern_locked_tte_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) for (i = 0; i < hdesc->num_mappings; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) hdesc->maps[i].vaddr = tte_vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) hdesc->maps[i].tte = tte_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) tte_vaddr += 0x400000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) tte_data += 0x400000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) trampoline_ra = kimage_addr_to_ra(hv_cpu_startup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) hv_err = sun4v_cpu_start(cpu, trampoline_ra,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) kimage_addr_to_ra(&sparc64_ttable_tl0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) __pa(hdesc));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) printk(KERN_ERR "ldom_startcpu_cpuid: sun4v_cpu_start() "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) "gives error %lu\n", hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) extern unsigned long sparc64_cpu_startup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /* The OBP cpu startup callback truncates the 3rd arg cookie to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * 32-bits (I think) so to be safe we have it read the pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) * contained here so we work on >4GB machines. -DaveM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static struct thread_info *cpu_new_thread = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) static int smp_boot_one_cpu(unsigned int cpu, struct task_struct *idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) unsigned long entry =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) (unsigned long)(&sparc64_cpu_startup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) unsigned long cookie =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) (unsigned long)(&cpu_new_thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) void *descr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) int timeout, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) callin_flag = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) cpu_new_thread = task_thread_info(idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (tlb_type == hypervisor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) #if defined(CONFIG_SUN_LDOMS) && defined(CONFIG_HOTPLUG_CPU)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (ldom_domaining_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) ldom_startcpu_cpuid(cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) (unsigned long) cpu_new_thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) &descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) prom_startcpu_cpuid(cpu, entry, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) struct device_node *dp = of_find_node_by_cpuid(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) prom_startcpu(dp->phandle, entry, cookie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) for (timeout = 0; timeout < 50000; timeout++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) if (callin_flag)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) udelay(100);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (callin_flag) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) printk("Processor %d is stuck.\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) cpu_new_thread = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) kfree(descr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) static void spitfire_xcall_helper(u64 data0, u64 data1, u64 data2, u64 pstate, unsigned long cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) u64 result, target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) int stuck, tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) if (this_is_starfire) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /* map to real upaid */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) cpu = (((cpu & 0x3c) << 1) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) ((cpu & 0x40) >> 4) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) (cpu & 0x3));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) target = (cpu << 14) | 0x70;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) again:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) /* Ok, this is the real Spitfire Errata #54.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) * One must read back from a UDB internal register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) * after writes to the UDB interrupt dispatch, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) * before the membar Sync for that write.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) * So we use the high UDB control register (ASI 0x7f,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) * ADDR 0x20) for the dummy read. -DaveM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) tmp = 0x40;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) "wrpr %1, %2, %%pstate\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) "stxa %4, [%0] %3\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) "stxa %5, [%0+%8] %3\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) "add %0, %8, %0\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) "stxa %6, [%0+%8] %3\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) "membar #Sync\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) "stxa %%g0, [%7] %3\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) "membar #Sync\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) "mov 0x20, %%g1\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) "ldxa [%%g1] 0x7f, %%g0\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) "membar #Sync"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) : "=r" (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) : "r" (pstate), "i" (PSTATE_IE), "i" (ASI_INTR_W),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) "r" (data0), "r" (data1), "r" (data2), "r" (target),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) "r" (0x10), "0" (tmp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) : "g1");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) /* NOTE: PSTATE_IE is still clear. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) stuck = 100000;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) __asm__ __volatile__("ldxa [%%g0] %1, %0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) : "=r" (result)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) : "i" (ASI_INTR_DISPATCH_STAT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) if (result == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) : : "r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) stuck -= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) if (stuck == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) } while (result & 0x1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) : : "r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) if (stuck == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) printk("CPU[%d]: mondo stuckage result[%016llx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) smp_processor_id(), result);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) udelay(2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) goto again;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) static void spitfire_xcall_deliver(struct trap_per_cpu *tb, int cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) u64 *mondo, data0, data1, data2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) u16 *cpu_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) u64 pstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) cpu_list = __va(tb->cpu_list_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) mondo = __va(tb->cpu_mondo_block_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) data0 = mondo[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) data1 = mondo[1];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) data2 = mondo[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) for (i = 0; i < cnt; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) spitfire_xcall_helper(data0, data1, data2, pstate, cpu_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) /* Cheetah now allows to send the whole 64-bytes of data in the interrupt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * packet, but we have no use for that. However we do take advantage of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * the new pipelining feature (ie. dispatch to multiple cpus simultaneously).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) static void cheetah_xcall_deliver(struct trap_per_cpu *tb, int cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) int nack_busy_id, is_jbus, need_more;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) u64 *mondo, pstate, ver, busy_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) u16 *cpu_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) cpu_list = __va(tb->cpu_list_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) mondo = __va(tb->cpu_mondo_block_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) /* Unfortunately, someone at Sun had the brilliant idea to make the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) * busy/nack fields hard-coded by ITID number for this Ultra-III
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) * derivative processor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) __asm__ ("rdpr %%ver, %0" : "=r" (ver));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) is_jbus = ((ver >> 32) == __JALAPENO_ID ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) (ver >> 32) == __SERRANO_ID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) __asm__ __volatile__("rdpr %%pstate, %0" : "=r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) need_more = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) __asm__ __volatile__("wrpr %0, %1, %%pstate\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) : : "r" (pstate), "i" (PSTATE_IE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) /* Setup the dispatch data registers. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) __asm__ __volatile__("stxa %0, [%3] %6\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) "stxa %1, [%4] %6\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) "stxa %2, [%5] %6\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) "membar #Sync\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) : /* no outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) : "r" (mondo[0]), "r" (mondo[1]), "r" (mondo[2]),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) "r" (0x40), "r" (0x50), "r" (0x60),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) "i" (ASI_INTR_W));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) nack_busy_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) busy_mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) for (i = 0; i < cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) u64 target, nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) nr = cpu_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) if (nr == 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) target = (nr << 14) | 0x70;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (is_jbus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) busy_mask |= (0x1UL << (nr * 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) target |= (nack_busy_id << 24);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) busy_mask |= (0x1UL <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) (nack_busy_id * 2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) "stxa %%g0, [%0] %1\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) "membar #Sync\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) : /* no outputs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) : "r" (target), "i" (ASI_INTR_W));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) nack_busy_id++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (nack_busy_id == 32) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) need_more = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) /* Now, poll for completion. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) u64 dispatch_stat, nack_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) long stuck;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) stuck = 100000 * nack_busy_id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) nack_mask = busy_mask << 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) __asm__ __volatile__("ldxa [%%g0] %1, %0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) : "=r" (dispatch_stat)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) : "i" (ASI_INTR_DISPATCH_STAT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (!(dispatch_stat & (busy_mask | nack_mask))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) : : "r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) if (unlikely(need_more)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) int i, this_cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) for (i = 0; i < cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) if (cpu_list[i] == 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) cpu_list[i] = 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) this_cnt++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if (this_cnt == 32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) if (!--stuck)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) } while (dispatch_stat & busy_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) __asm__ __volatile__("wrpr %0, 0x0, %%pstate"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) : : "r" (pstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (dispatch_stat & busy_mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) /* Busy bits will not clear, continue instead
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) * of freezing up on this cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) printk("CPU[%d]: mondo stuckage result[%016llx]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) smp_processor_id(), dispatch_stat);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) int i, this_busy_nack = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) /* Delay some random time with interrupts enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) * to prevent deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) udelay(2 * nack_busy_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) /* Clear out the mask bits for cpus which did not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) * NACK us.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) for (i = 0; i < cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) u64 check_mask, nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) nr = cpu_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (nr == 0xffff)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) if (is_jbus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) check_mask = (0x2UL << (2*nr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) check_mask = (0x2UL <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) this_busy_nack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) if ((dispatch_stat & check_mask) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) cpu_list[i] = 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) this_busy_nack += 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) if (this_busy_nack == 64)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) #define CPU_MONDO_COUNTER(cpuid) (cpu_mondo_counter[cpuid])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) #define MONDO_USEC_WAIT_MIN 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) #define MONDO_USEC_WAIT_MAX 100
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) #define MONDO_RETRY_LIMIT 500000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /* Multi-cpu list version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Deliver xcalls to 'cnt' number of cpus in 'cpu_list'.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * Sometimes not all cpus receive the mondo, requiring us to re-send
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) * the mondo until all cpus have received, or cpus are truly stuck
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * unable to receive mondo, and we timeout.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * Occasionally a target cpu strand is borrowed briefly by hypervisor to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) * perform guest service, such as PCIe error handling. Consider the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) * service time, 1 second overall wait is reasonable for 1 cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) * Here two in-between mondo check wait time are defined: 2 usec for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) * single cpu quick turn around and up to 100usec for large cpu count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) * Deliver mondo to large number of cpus could take longer, we adjusts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) * the retry count as long as target cpus are making forward progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) static void hypervisor_xcall_deliver(struct trap_per_cpu *tb, int cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) int this_cpu, tot_cpus, prev_sent, i, rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) int usec_wait, retries, tot_retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) u16 first_cpu = 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) unsigned long xc_rcvd = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) unsigned long status;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) int ecpuerror_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) int enocpu_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) u16 *cpu_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) u16 cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) this_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) cpu_list = __va(tb->cpu_list_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) usec_wait = cnt * MONDO_USEC_WAIT_MIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) if (usec_wait > MONDO_USEC_WAIT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) usec_wait = MONDO_USEC_WAIT_MAX;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) retries = tot_retries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) tot_cpus = cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) prev_sent = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) int n_sent, mondo_delivered, target_cpu_busy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) status = sun4v_cpu_mondo_send(cnt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) tb->cpu_list_pa,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) tb->cpu_mondo_block_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) /* HV_EOK means all cpus received the xcall, we're done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) if (likely(status == HV_EOK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) goto xcall_done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* If not these non-fatal errors, panic */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) if (unlikely((status != HV_EWOULDBLOCK) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) (status != HV_ECPUERROR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) (status != HV_ENOCPU)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) goto fatal_errors;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* First, see if we made any forward progress.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) * Go through the cpu_list, count the target cpus that have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) * received our mondo (n_sent), and those that did not (rem).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) * Re-pack cpu_list with the cpus remain to be retried in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) * front - this simplifies tracking the truly stalled cpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) * The hypervisor indicates successful sends by setting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) * cpu list entries to the value 0xffff.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) * EWOULDBLOCK means some target cpus did not receive the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) * mondo and retry usually helps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) * ECPUERROR means at least one target cpu is in error state,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * it's usually safe to skip the faulty cpu and retry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * ENOCPU means one of the target cpu doesn't belong to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * domain, perhaps offlined which is unexpected, but not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) * fatal and it's okay to skip the offlined cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) rem = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) n_sent = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) for (i = 0; i < cnt; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) cpu = cpu_list[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (likely(cpu == 0xffff)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) n_sent++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) } else if ((status == HV_ECPUERROR) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) (sun4v_cpu_state(cpu) == HV_CPU_STATE_ERROR)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) ecpuerror_id = cpu + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) } else if (status == HV_ENOCPU && !cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) enocpu_id = cpu + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) cpu_list[rem++] = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) /* No cpu remained, we're done. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (rem == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) /* Otherwise, update the cpu count for retry. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) cnt = rem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /* Record the overall number of mondos received by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) * first of the remaining cpus.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) if (first_cpu != cpu_list[0]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) first_cpu = cpu_list[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) xc_rcvd = CPU_MONDO_COUNTER(first_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) /* Was any mondo delivered successfully? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) mondo_delivered = (n_sent > prev_sent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) prev_sent = n_sent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) /* or, was any target cpu busy processing other mondos? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) target_cpu_busy = (xc_rcvd < CPU_MONDO_COUNTER(first_cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) xc_rcvd = CPU_MONDO_COUNTER(first_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /* Retry count is for no progress. If we're making progress,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * reset the retry count.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (likely(mondo_delivered || target_cpu_busy)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) tot_retries += retries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) retries = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) } else if (unlikely(retries > MONDO_RETRY_LIMIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) goto fatal_mondo_timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) /* Delay a little bit to let other cpus catch up on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * their cpu mondo queue work.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (!mondo_delivered)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) udelay(usec_wait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) retries++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) } while (1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) xcall_done:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (unlikely(ecpuerror_id > 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) was in error state\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) this_cpu, ecpuerror_id - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) } else if (unlikely(enocpu_id > 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) pr_crit("CPU[%d]: SUN4V mondo cpu error, target cpu(%d) does not belong to the domain\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) this_cpu, enocpu_id - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) fatal_errors:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) /* fatal errors include bad alignment, etc */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) pr_crit("CPU[%d]: Args were cnt(%d) cpulist_pa(%lx) mondo_block_pa(%lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) this_cpu, tot_cpus, tb->cpu_list_pa, tb->cpu_mondo_block_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) panic("Unexpected SUN4V mondo error %lu\n", status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) fatal_mondo_timeout:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) /* some cpus being non-responsive to the cpu mondo */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) pr_crit("CPU[%d]: SUN4V mondo timeout, cpu(%d) made no forward progress after %d retries. Total target cpus(%d).\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) this_cpu, first_cpu, (tot_retries + retries), tot_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) panic("SUN4V mondo timeout panic\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) static void (*xcall_deliver_impl)(struct trap_per_cpu *, int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) static void xcall_deliver(u64 data0, u64 data1, u64 data2, const cpumask_t *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) struct trap_per_cpu *tb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) int this_cpu, i, cnt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) u16 *cpu_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) u64 *mondo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) /* We have to do this whole thing with interrupts fully disabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) * Otherwise if we send an xcall from interrupt context it will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) * corrupt both our mondo block and cpu list state.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) * One consequence of this is that we cannot use timeout mechanisms
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) * that depend upon interrupts being delivered locally. So, for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) * example, we cannot sample jiffies and expect it to advance.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) * Fortunately, udelay() uses %stick/%tick so we can use that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) this_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) tb = &trap_block[this_cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) mondo = __va(tb->cpu_mondo_block_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) mondo[0] = data0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) mondo[1] = data1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) mondo[2] = data2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) cpu_list = __va(tb->cpu_list_pa);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) /* Setup the initial cpu list. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) cnt = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) for_each_cpu(i, mask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (i == this_cpu || !cpu_online(i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) cpu_list[cnt++] = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (cnt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) xcall_deliver_impl(tb, cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /* Send cross call to all processors mentioned in MASK_P
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * except self. Really, there are only two cases currently,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * "cpu_online_mask" and "mm_cpumask(mm)".
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) static void smp_cross_call_masked(unsigned long *func, u32 ctx, u64 data1, u64 data2, const cpumask_t *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) u64 data0 = (((u64)ctx)<<32 | (((u64)func) & 0xffffffff));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) xcall_deliver(data0, data1, data2, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) /* Send cross call to all processors except self. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) static void smp_cross_call(unsigned long *func, u32 ctx, u64 data1, u64 data2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) smp_cross_call_masked(func, ctx, data1, data2, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) extern unsigned long xcall_sync_tick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) static void smp_start_sync_tick_client(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) xcall_deliver((u64) &xcall_sync_tick, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) extern unsigned long xcall_call_function;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) void arch_send_call_function_ipi_mask(const struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) xcall_deliver((u64) &xcall_call_function, 0, 0, mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) extern unsigned long xcall_call_function_single;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) void arch_send_call_function_single_ipi(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) xcall_deliver((u64) &xcall_call_function_single, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) void __irq_entry smp_call_function_client(int irq, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) clear_softint(1 << irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) irq_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) generic_smp_call_function_interrupt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) irq_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) void __irq_entry smp_call_function_single_client(int irq, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) clear_softint(1 << irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) irq_enter();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) generic_smp_call_function_single_interrupt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) irq_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) static void tsb_sync(void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) struct trap_per_cpu *tp = &trap_block[raw_smp_processor_id()];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) struct mm_struct *mm = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /* It is not valid to test "current->active_mm == mm" here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * The value of "current" is not changed atomically with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * switch_mm(). But that's OK, we just need to check the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * current cpu's trap block PGD physical address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (tp->pgd_paddr == __pa(mm->pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) tsb_context_switch(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) void smp_tsb_sync(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) smp_call_function_many(mm_cpumask(mm), tsb_sync, mm, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) extern unsigned long xcall_flush_tlb_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) extern unsigned long xcall_flush_tlb_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) extern unsigned long xcall_flush_tlb_kernel_range;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) extern unsigned long xcall_fetch_glob_regs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) extern unsigned long xcall_fetch_glob_pmu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) extern unsigned long xcall_fetch_glob_pmu_n4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) extern unsigned long xcall_receive_signal;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) extern unsigned long xcall_new_mmu_context_version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) #ifdef CONFIG_KGDB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) extern unsigned long xcall_kgdb_capture;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) #ifdef DCACHE_ALIASING_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) extern unsigned long xcall_flush_dcache_page_cheetah;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) extern unsigned long xcall_flush_dcache_page_spitfire;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) static inline void __local_flush_dcache_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) #ifdef DCACHE_ALIASING_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) __flush_dcache_page(page_address(page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) ((tlb_type == spitfire) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) page_mapping_file(page) != NULL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (page_mapping_file(page) != NULL &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) tlb_type == spitfire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) __flush_icache_page(__pa(page_address(page)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) void smp_flush_dcache_page_impl(struct page *page, int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) int this_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) #ifdef CONFIG_DEBUG_DCFLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) atomic_inc(&dcpage_flushes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) this_cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) if (cpu == this_cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) __local_flush_dcache_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) } else if (cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) void *pg_addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) u64 data0 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) if (tlb_type == spitfire) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) data0 = ((u64)&xcall_flush_dcache_page_spitfire);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) if (page_mapping_file(page) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) data0 |= ((u64)1 << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) #ifdef DCACHE_ALIASING_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) data0 = ((u64)&xcall_flush_dcache_page_cheetah);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) if (data0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) xcall_deliver(data0, __pa(pg_addr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) (u64) pg_addr, cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) #ifdef CONFIG_DEBUG_DCFLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) atomic_inc(&dcpage_flushes_xcall);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) void flush_dcache_page_all(struct mm_struct *mm, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) void *pg_addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) u64 data0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) #ifdef CONFIG_DEBUG_DCFLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) atomic_inc(&dcpage_flushes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) data0 = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) pg_addr = page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) if (tlb_type == spitfire) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) data0 = ((u64)&xcall_flush_dcache_page_spitfire);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) if (page_mapping_file(page) != NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) data0 |= ((u64)1 << 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) } else if (tlb_type == cheetah || tlb_type == cheetah_plus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) #ifdef DCACHE_ALIASING_POSSIBLE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) data0 = ((u64)&xcall_flush_dcache_page_cheetah);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) if (data0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) xcall_deliver(data0, __pa(pg_addr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) (u64) pg_addr, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) #ifdef CONFIG_DEBUG_DCFLUSH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) atomic_inc(&dcpage_flushes_xcall);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) __local_flush_dcache_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) #ifdef CONFIG_KGDB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) void kgdb_roundup_cpus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) smp_cross_call(&xcall_kgdb_capture, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) void smp_fetch_global_regs(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) smp_cross_call(&xcall_fetch_glob_regs, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) void smp_fetch_global_pmu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) if (tlb_type == hypervisor &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) sun4v_chip_type >= SUN4V_CHIP_NIAGARA4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) smp_cross_call(&xcall_fetch_glob_pmu_n4, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) smp_cross_call(&xcall_fetch_glob_pmu, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) /* We know that the window frames of the user have been flushed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) * to the stack before we get here because all callers of us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * are flush_tlb_*() routines, and these run after flush_cache_*()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * which performs the flushw.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * mm->cpu_vm_mask is a bit mask of which cpus an address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) * space has (potentially) executed on, this is the heuristic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) * we use to limit cross calls.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) /* This currently is only used by the hugetlb arch pre-fault
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) * hook on UltraSPARC-III+ and later when changing the pagesize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) * bits of the context register for an address space.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) void smp_flush_tlb_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) u32 ctx = CTX_HWBITS(mm->context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) smp_cross_call_masked(&xcall_flush_tlb_mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) ctx, 0, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) mm_cpumask(mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) __flush_tlb_mm(ctx, SECONDARY_CONTEXT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct tlb_pending_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) unsigned long ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) unsigned long nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) unsigned long *vaddrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) static void tlb_pending_func(void *info)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) struct tlb_pending_info *t = info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) __flush_tlb_pending(t->ctx, t->nr, t->vaddrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) void smp_flush_tlb_pending(struct mm_struct *mm, unsigned long nr, unsigned long *vaddrs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) u32 ctx = CTX_HWBITS(mm->context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) struct tlb_pending_info info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) info.ctx = ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) info.nr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) info.vaddrs = vaddrs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) smp_call_function_many(mm_cpumask(mm), tlb_pending_func,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) &info, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) __flush_tlb_pending(ctx, nr, vaddrs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) void smp_flush_tlb_page(struct mm_struct *mm, unsigned long vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) unsigned long context = CTX_HWBITS(mm->context);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) smp_cross_call_masked(&xcall_flush_tlb_page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) context, vaddr, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) mm_cpumask(mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) __flush_tlb_page(context, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) void smp_flush_tlb_kernel_range(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) start &= PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) end = PAGE_ALIGN(end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) if (start != end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) smp_cross_call(&xcall_flush_tlb_kernel_range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) 0, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) __flush_tlb_kernel_range(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) /* CPU capture. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) /* #define CAPTURE_DEBUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) extern unsigned long xcall_capture;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) static atomic_t smp_capture_depth = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) static atomic_t smp_capture_registry = ATOMIC_INIT(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) static unsigned long penguins_are_doing_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) void smp_capture(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) int result = atomic_add_return(1, &smp_capture_depth);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) if (result == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) int ncpus = num_online_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) #ifdef CAPTURE_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) printk("CPU[%d]: Sending penguins to jail...",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) penguins_are_doing_time = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) atomic_inc(&smp_capture_registry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) smp_cross_call(&xcall_capture, 0, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) while (atomic_read(&smp_capture_registry) != ncpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) #ifdef CAPTURE_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) printk("done\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) void smp_release(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) if (atomic_dec_and_test(&smp_capture_depth)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) #ifdef CAPTURE_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) printk("CPU[%d]: Giving pardon to "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) "imprisoned penguins\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) penguins_are_doing_time = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) membar_safe("#StoreLoad");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) atomic_dec(&smp_capture_registry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) /* Imprisoned penguins run with %pil == PIL_NORMAL_MAX, but PSTATE_IE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) * set, so they can service tlb flush xcalls...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) extern void prom_world(int);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) void __irq_entry smp_penguin_jailcell(int irq, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) clear_softint(1 << irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) __asm__ __volatile__("flushw");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) prom_world(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) atomic_inc(&smp_capture_registry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) membar_safe("#StoreLoad");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) while (penguins_are_doing_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) atomic_dec(&smp_capture_registry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) prom_world(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) /* /proc/profile writes can call this, don't __init it please. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) int setup_profiling_timer(unsigned int multiplier)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) void __init smp_prepare_cpus(unsigned int max_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) void smp_prepare_boot_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) void __init smp_setup_processor_id(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) if (tlb_type == spitfire)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) xcall_deliver_impl = spitfire_xcall_deliver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) else if (tlb_type == cheetah || tlb_type == cheetah_plus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) xcall_deliver_impl = cheetah_xcall_deliver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) xcall_deliver_impl = hypervisor_xcall_deliver;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) void __init smp_fill_in_cpu_possible_map(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) int possible_cpus = num_possible_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (possible_cpus > nr_cpu_ids)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) possible_cpus = nr_cpu_ids;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) for (i = 0; i < possible_cpus; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) set_cpu_possible(i, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) for (; i < NR_CPUS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) set_cpu_possible(i, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) void smp_fill_in_sib_core_maps(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) for_each_present_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) unsigned int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) cpumask_clear(&cpu_core_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) if (cpu_data(i).core_id == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) cpumask_set_cpu(i, &cpu_core_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) for_each_present_cpu(j) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) if (cpu_data(i).core_id ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) cpu_data(j).core_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) cpumask_set_cpu(j, &cpu_core_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) for_each_present_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) unsigned int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) for_each_present_cpu(j) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) if (cpu_data(i).max_cache_id ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) cpu_data(j).max_cache_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) cpumask_set_cpu(j, &cpu_core_sib_cache_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) if (cpu_data(i).sock_id == cpu_data(j).sock_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) cpumask_set_cpu(j, &cpu_core_sib_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) for_each_present_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) unsigned int j;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) cpumask_clear(&per_cpu(cpu_sibling_map, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) if (cpu_data(i).proc_id == -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) cpumask_set_cpu(i, &per_cpu(cpu_sibling_map, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) for_each_present_cpu(j) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (cpu_data(i).proc_id ==
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) cpu_data(j).proc_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) cpumask_set_cpu(j, &per_cpu(cpu_sibling_map, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) int __cpu_up(unsigned int cpu, struct task_struct *tidle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) int ret = smp_boot_one_cpu(cpu, tidle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (!ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) cpumask_set_cpu(cpu, &smp_commenced_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) while (!cpu_online(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) if (!cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) /* On SUN4V, writes to %tick and %stick are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) * not allowed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) if (tlb_type != hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) smp_synchronize_one_tick(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) void cpu_play_dead(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) unsigned long pstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) idle_task_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) if (tlb_type == hypervisor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) struct trap_per_cpu *tb = &trap_block[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) sun4v_cpu_qconf(HV_CPU_QUEUE_CPU_MONDO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) tb->cpu_mondo_pa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) sun4v_cpu_qconf(HV_CPU_QUEUE_DEVICE_MONDO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) tb->dev_mondo_pa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) sun4v_cpu_qconf(HV_CPU_QUEUE_RES_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) tb->resum_mondo_pa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) sun4v_cpu_qconf(HV_CPU_QUEUE_NONRES_ERROR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) tb->nonresum_mondo_pa, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) cpumask_clear_cpu(cpu, &smp_commenced_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) membar_safe("#Sync");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) "rdpr %%pstate, %0\n\t"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) "wrpr %0, %1, %%pstate"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) : "=r" (pstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) : "i" (PSTATE_IE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) while (1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) int __cpu_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) cpuinfo_sparc *c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) for_each_cpu(i, &cpu_core_map[cpu])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) cpumask_clear_cpu(cpu, &cpu_core_map[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) cpumask_clear(&cpu_core_map[cpu]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) for_each_cpu(i, &per_cpu(cpu_sibling_map, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) cpumask_clear_cpu(cpu, &per_cpu(cpu_sibling_map, i));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) cpumask_clear(&per_cpu(cpu_sibling_map, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) c = &cpu_data(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) c->core_id = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) c->proc_id = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) /* Make sure no interrupts point to this cpu. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) fixup_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) mdelay(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) set_cpu_online(cpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) cpu_map_rebuild();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) void __cpu_die(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) for (i = 0; i < 100; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) smp_rmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) if (!cpumask_test_cpu(cpu, &smp_commenced_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) msleep(100);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) if (cpumask_test_cpu(cpu, &smp_commenced_mask)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) printk(KERN_ERR "CPU %u didn't die...\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) #if defined(CONFIG_SUN_LDOMS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) int limit = 100;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) hv_err = sun4v_cpu_stop(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) if (hv_err == HV_EOK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) set_cpu_present(cpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) } while (--limit > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) if (limit <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) printk(KERN_ERR "sun4v_cpu_stop() fails err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) void __init smp_cpus_done(unsigned int max_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) static void send_cpu_ipi(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) xcall_deliver((u64) &xcall_receive_signal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) 0, 0, cpumask_of(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) void scheduler_poke(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) if (!cpu_poke)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) if (!__this_cpu_read(poke))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) __this_cpu_write(poke, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) set_softint(1 << PIL_SMP_RECEIVE_SIGNAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) static unsigned long send_cpu_poke(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) per_cpu(poke, cpu) = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) hv_err = sun4v_cpu_poke(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) if (hv_err != HV_EOK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) per_cpu(poke, cpu) = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) pr_err_ratelimited("%s: sun4v_cpu_poke() fails err=%lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) __func__, hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) return hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) void smp_send_reschedule(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) if (cpu == smp_processor_id()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) WARN_ON_ONCE(preemptible());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) set_softint(1 << PIL_SMP_RECEIVE_SIGNAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) /* Use cpu poke to resume idle cpu if supported. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (cpu_poke && idle_cpu(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) unsigned long ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) ret = send_cpu_poke(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) if (ret == HV_EOK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) /* Use IPI in following cases:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) * - cpu poke not supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) * - cpu not idle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) * - send_cpu_poke() returns with error
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) send_cpu_ipi(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) void smp_init_cpu_poke(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) unsigned long major;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) unsigned long minor;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (tlb_type != hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) ret = sun4v_hvapi_get(HV_GRP_CORE, &major, &minor);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) pr_debug("HV_GRP_CORE is not registered\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) if (major == 1 && minor >= 6) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) /* CPU POKE is registered. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) cpu_poke = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) pr_debug("CPU_POKE not supported\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) void __irq_entry smp_receive_signal_client(int irq, struct pt_regs *regs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) clear_softint(1 << irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) scheduler_ipi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) static void stop_this_cpu(void *dummy)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) set_cpu_online(smp_processor_id(), false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) prom_stopself();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) void smp_send_stop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) if (tlb_type == hypervisor) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) int this_cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) #ifdef CONFIG_SERIAL_SUNHV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) sunhv_migrate_hvcons_irq(this_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) for_each_online_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) if (cpu == this_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) set_cpu_online(cpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) #ifdef CONFIG_SUN_LDOMS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) if (ldom_domaining_enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) unsigned long hv_err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) hv_err = sun4v_cpu_stop(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (hv_err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) printk(KERN_ERR "sun4v_cpu_stop() "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) "failed err=%lu\n", hv_err);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) prom_stopcpu_cpuid(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) smp_call_function(stop_this_cpu, NULL, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) * pcpu_alloc_bootmem - NUMA friendly alloc_bootmem wrapper for percpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) * @cpu: cpu to allocate for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) * @size: size allocation in bytes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) * @align: alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) * Allocate @size bytes aligned at @align for cpu @cpu. This wrapper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) * does the right thing for NUMA regardless of the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) * configuration.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) * Pointer to the allocated area on success, NULL on failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) static void * __init pcpu_alloc_bootmem(unsigned int cpu, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) size_t align)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) const unsigned long goal = __pa(MAX_DMA_ADDRESS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) #ifdef CONFIG_NEED_MULTIPLE_NODES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) int node = cpu_to_node(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) if (!node_online(node) || !NODE_DATA(node)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) ptr = memblock_alloc_from(size, align, goal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) pr_info("cpu %d has no node %d or node-local memory\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) cpu, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) pr_debug("per cpu data for cpu%d %lu bytes at %016lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) cpu, size, __pa(ptr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) ptr = memblock_alloc_try_nid(size, align, goal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) MEMBLOCK_ALLOC_ACCESSIBLE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) pr_debug("per cpu data for cpu%d %lu bytes on node%d at "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) "%016lx\n", cpu, size, node, __pa(ptr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) return memblock_alloc_from(size, align, goal);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) static void __init pcpu_free_bootmem(void *ptr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) memblock_free(__pa(ptr), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) if (cpu_to_node(from) == cpu_to_node(to))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) return LOCAL_DISTANCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) return REMOTE_DISTANCE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) static void __init pcpu_populate_pte(unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) pgd_t *pgd = pgd_offset_k(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) if (pgd_none(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) pud_t *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) goto err_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) pgd_populate(&init_mm, pgd, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) if (p4d_none(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) pud_t *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) goto err_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) p4d_populate(&init_mm, p4d, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) if (pud_none(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) pmd_t *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) goto err_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) pud_populate(&init_mm, pud, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) if (!pmd_present(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) pte_t *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) new = memblock_alloc_from(PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) goto err_alloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) pmd_populate_kernel(&init_mm, pmd, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) err_alloc:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) __func__, PAGE_SIZE, PAGE_SIZE, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) void __init setup_per_cpu_areas(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) unsigned long delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) int rc = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) if (pcpu_chosen_fc != PCPU_FC_PAGE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) PERCPU_DYNAMIC_RESERVE, 4 << 20,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) pcpu_cpu_distance,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) pcpu_alloc_bootmem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) pcpu_free_bootmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) pr_warn("PERCPU: %s allocator failed (%d), "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) "falling back to page size\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649) pcpu_fc_names[pcpu_chosen_fc], rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) rc = pcpu_page_first_chunk(PERCPU_MODULE_RESERVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) pcpu_alloc_bootmem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) pcpu_free_bootmem,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) pcpu_populate_pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) panic("cannot initialize percpu area (err=%d)", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) __per_cpu_offset(cpu) = delta + pcpu_unit_offsets[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) /* Setup %g5 for the boot cpu. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) __local_per_cpu_offset = __per_cpu_offset(smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) of_fill_in_cpu_data();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) if (tlb_type == hypervisor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) mdesc_fill_in_cpu_data(cpu_all_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags