^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Xtensa SMP support functions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * License. See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2008 - 2013 Tensilica Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Chris Zankel <chris@zankel.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Joe Taylor <joe@tensilica.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Pete Delaney <piet@tensilica.com
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/cpumask.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/irqdomain.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/sched/hotplug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sched/task_stack.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/reboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/seq_file.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/thread_info.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <asm/kdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/mxregs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/platform.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/traps.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) # if XCHAL_HAVE_S32C1I == 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) # error "The S32C1I option is required for SMP."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) # endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) static void system_invalidate_dcache_range(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned long size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) static void system_flush_invalidate_dcache_range(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) unsigned long size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /* IPI (Inter Process Interrupt) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #define IPI_IRQ 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) static irqreturn_t ipi_interrupt(int irq, void *dev_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) void ipi_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) unsigned irq = irq_create_mapping(NULL, IPI_IRQ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (request_irq(irq, ipi_interrupt, IRQF_PERCPU, "ipi", NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) pr_err("Failed to request irq %u (ipi)\n", irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static inline unsigned int get_core_count(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) /* Bits 18..21 of SYSCFGID contain the core count minus 1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) unsigned int syscfgid = get_er(SYSCFGID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) return ((syscfgid >> 18) & 0xf) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) static inline int get_core_id(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) /* Bits 0...18 of SYSCFGID contain the core id */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) unsigned int core_id = get_er(SYSCFGID);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) return core_id & 0x3fff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) void __init smp_prepare_cpus(unsigned int max_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) for_each_possible_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) set_cpu_present(i, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) void __init smp_init_cpus(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) unsigned int ncpus = get_core_count();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) unsigned int core_id = get_core_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) pr_info("%s: Core Count = %d\n", __func__, ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) pr_info("%s: Core Id = %d\n", __func__, core_id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (ncpus > NR_CPUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ncpus = NR_CPUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) pr_info("%s: limiting core count by %d\n", __func__, ncpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) for (i = 0; i < ncpus; ++i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) set_cpu_possible(i, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) void __init smp_prepare_boot_cpu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) BUG_ON(cpu != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) cpu_asid_cache(cpu) = ASID_USER_FIRST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) void __init smp_cpus_done(unsigned int max_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) static int boot_secondary_processors = 1; /* Set with xt-gdb via .xt-gdb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) static DECLARE_COMPLETION(cpu_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) void secondary_start_kernel(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct mm_struct *mm = &init_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) init_mmu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) #ifdef CONFIG_DEBUG_MISC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) if (boot_secondary_processors == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) pr_debug("%s: boot_secondary_processors:%d; Hanging cpu:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) __func__, boot_secondary_processors, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) for (;;)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) __asm__ __volatile__ ("waiti " __stringify(LOCKLEVEL));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) pr_debug("%s: boot_secondary_processors:%d; Booting cpu:%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) __func__, boot_secondary_processors, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /* Init EXCSAVE1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) secondary_trap_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* All kernel threads share the same mm context. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) mmget(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) mmgrab(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) current->active_mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) cpumask_set_cpu(cpu, mm_cpumask(mm));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) enter_lazy_tlb(mm, current);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) trace_hardirqs_off();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) calibrate_delay();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) notify_cpu_starting(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) secondary_init_irq();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) local_timer_setup(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) set_cpu_online(cpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) local_irq_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) complete(&cpu_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) cpu_startup_entry(CPUHP_AP_ONLINE_IDLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void mx_cpu_start(void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned cpu = (unsigned)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) unsigned long run_stall_mask = get_er(MPSCORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) set_er(run_stall_mask & ~(1u << cpu), MPSCORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) __func__, cpu, run_stall_mask, get_er(MPSCORE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) static void mx_cpu_stop(void *p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) unsigned cpu = (unsigned)p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) unsigned long run_stall_mask = get_er(MPSCORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) set_er(run_stall_mask | (1u << cpu), MPSCORE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) pr_debug("%s: cpu: %d, run_stall_mask: %lx ---> %lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) __func__, cpu, run_stall_mask, get_er(MPSCORE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) unsigned long cpu_start_id __cacheline_aligned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) unsigned long cpu_start_ccount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) static int boot_secondary(unsigned int cpu, struct task_struct *ts)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned long timeout = jiffies + msecs_to_jiffies(1000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) unsigned long ccount;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) WRITE_ONCE(cpu_start_id, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* Pairs with the third memw in the cpu_restart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) system_flush_invalidate_dcache_range((unsigned long)&cpu_start_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) sizeof(cpu_start_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) smp_call_function_single(0, mx_cpu_start, (void *)cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) for (i = 0; i < 2; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) ccount = get_ccount();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) while (!ccount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) WRITE_ONCE(cpu_start_ccount, ccount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) * Pairs with the first two memws in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) * .Lboot_secondary.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) ccount = READ_ONCE(cpu_start_ccount);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) } while (ccount && time_before(jiffies, timeout));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (ccount) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) smp_call_function_single(0, mx_cpu_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) (void *)cpu, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) WRITE_ONCE(cpu_start_ccount, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) return -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) int __cpu_up(unsigned int cpu, struct task_struct *idle)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) if (cpu_asid_cache(cpu) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) cpu_asid_cache(cpu) = ASID_USER_FIRST;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) start_info.stack = (unsigned long)task_pt_regs(idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) pr_debug("%s: Calling wakeup_secondary(cpu:%d, idle:%p, sp: %08lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) __func__, cpu, idle, start_info.stack);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) init_completion(&cpu_running);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) ret = boot_secondary(cpu, idle);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) wait_for_completion_timeout(&cpu_running,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) msecs_to_jiffies(1000));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) if (!cpu_online(cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) ret = -EIO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) pr_err("CPU %u failed to boot\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) #ifdef CONFIG_HOTPLUG_CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * __cpu_disable runs on the processor to be shutdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) int __cpu_disable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * Take this CPU offline. Once we clear this, we can't return,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * and we must not schedule until we're ready to give up the cpu.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) set_cpu_online(cpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) * OK - migrate IRQs away from this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) migrate_irqs();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) * Flush user cache and TLB mappings, and then remove this CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * from the vm mask set of all processes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) local_flush_cache_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) local_flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) invalidate_page_directory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) clear_tasks_mm_cpumask(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) static void platform_cpu_kill(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) smp_call_function_single(0, mx_cpu_stop, (void *)cpu, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * called on the thread which is asking for a CPU to be shutdown -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * waits until shutdown has completed, or it is timed out.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) void __cpu_die(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) unsigned long timeout = jiffies + msecs_to_jiffies(1000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) while (time_before(jiffies, timeout)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) system_invalidate_dcache_range((unsigned long)&cpu_start_id,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) sizeof(cpu_start_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /* Pairs with the second memw in the cpu_restart */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) if (READ_ONCE(cpu_start_id) == -cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) platform_cpu_kill(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) pr_err("CPU%u: unable to kill\n", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) void arch_cpu_idle_dead(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) cpu_die();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * Called from the idle thread for the CPU which has been shutdown.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) * Note that we disable IRQs here, but do not re-enable them
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) * before returning to the caller. This is also the behaviour
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) * of the other hotplug-cpu capable cores, so presumably coming
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) * out of idle fixes this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) void __ref cpu_die(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) idle_task_exit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) __asm__ __volatile__(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) " movi a2, cpu_restart\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) " jx a2\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) #endif /* CONFIG_HOTPLUG_CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) enum ipi_msg_type {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) IPI_RESCHEDULE = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) IPI_CALL_FUNC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) IPI_CPU_STOP,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) IPI_MAX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static const struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) const char *short_text;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) const char *long_text;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) } ipi_text[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) { .short_text = "RES", .long_text = "Rescheduling interrupts" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) { .short_text = "CAL", .long_text = "Function call interrupts" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) { .short_text = "DIE", .long_text = "CPU shutdown interrupts" },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) struct ipi_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) unsigned long ipi_count[IPI_MAX];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) static DEFINE_PER_CPU(struct ipi_data, ipi_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) static void send_ipi_message(const struct cpumask *callmask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) enum ipi_msg_type msg_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) int index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) unsigned long mask = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) for_each_cpu(index, callmask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) mask |= 1 << index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) set_er(mask, MIPISET(msg_id));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) void arch_send_call_function_ipi_mask(const struct cpumask *mask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) send_ipi_message(mask, IPI_CALL_FUNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) void arch_send_call_function_single_ipi(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) send_ipi_message(cpumask_of(cpu), IPI_CALL_FUNC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) void smp_send_reschedule(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) send_ipi_message(cpumask_of(cpu), IPI_RESCHEDULE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) void smp_send_stop(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct cpumask targets;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) cpumask_copy(&targets, cpu_online_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) cpumask_clear_cpu(smp_processor_id(), &targets);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) send_ipi_message(&targets, IPI_CPU_STOP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static void ipi_cpu_stop(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) set_cpu_online(cpu, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) machine_halt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) irqreturn_t ipi_interrupt(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) unsigned int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct ipi_data *ipi = &per_cpu(ipi_data, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) unsigned int msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) msg = get_er(MIPICAUSE(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) set_er(msg, MIPICAUSE(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) if (!msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) if (msg & (1 << IPI_CALL_FUNC)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) ++ipi->ipi_count[IPI_CALL_FUNC];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) generic_smp_call_function_interrupt();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (msg & (1 << IPI_RESCHEDULE)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) ++ipi->ipi_count[IPI_RESCHEDULE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) scheduler_ipi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) if (msg & (1 << IPI_CPU_STOP)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) ++ipi->ipi_count[IPI_CPU_STOP];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) ipi_cpu_stop(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) void show_ipi_list(struct seq_file *p, int prec)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) for (i = 0; i < IPI_MAX; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) seq_printf(p, "%*s:", prec, ipi_text[i].short_text);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) for_each_online_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) seq_printf(p, " %10lu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) per_cpu(ipi_data, cpu).ipi_count[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) seq_printf(p, " %s\n", ipi_text[i].long_text);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) int setup_profiling_timer(unsigned int multiplier)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) pr_debug("setup_profiling_timer %d\n", multiplier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) /* TLB flush functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) struct flush_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) unsigned long addr1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) unsigned long addr2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) static void ipi_flush_tlb_all(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) local_flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) void flush_tlb_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) on_each_cpu(ipi_flush_tlb_all, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) static void ipi_flush_tlb_mm(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) local_flush_tlb_mm(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) void flush_tlb_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) on_each_cpu(ipi_flush_tlb_mm, mm, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) static void ipi_flush_tlb_page(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) local_flush_tlb_page(fd->vma, fd->addr1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) void flush_tlb_page(struct vm_area_struct *vma, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) .vma = vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) .addr1 = addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) on_each_cpu(ipi_flush_tlb_page, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) static void ipi_flush_tlb_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) local_flush_tlb_range(fd->vma, fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) void flush_tlb_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) .vma = vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) .addr2 = end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) on_each_cpu(ipi_flush_tlb_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) static void ipi_flush_tlb_kernel_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) local_flush_tlb_kernel_range(fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) void flush_tlb_kernel_range(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) .addr2 = end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) on_each_cpu(ipi_flush_tlb_kernel_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) /* Cache flush functions */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) static void ipi_flush_cache_all(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) local_flush_cache_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) void flush_cache_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) on_each_cpu(ipi_flush_cache_all, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) static void ipi_flush_cache_page(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) local_flush_cache_page(fd->vma, fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) void flush_cache_page(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) unsigned long address, unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) .vma = vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) .addr1 = address,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) .addr2 = pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) on_each_cpu(ipi_flush_cache_page, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) static void ipi_flush_cache_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) local_flush_cache_range(fd->vma, fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) void flush_cache_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) .vma = vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) .addr2 = end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) on_each_cpu(ipi_flush_cache_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) static void ipi_flush_icache_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) local_flush_icache_range(fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) void flush_icache_range(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) .addr2 = end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) on_each_cpu(ipi_flush_icache_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) EXPORT_SYMBOL(flush_icache_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) /* ------------------------------------------------------------------------- */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) static void ipi_invalidate_dcache_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) __invalidate_dcache_range(fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) static void system_invalidate_dcache_range(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) .addr2 = size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) on_each_cpu(ipi_invalidate_dcache_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) static void ipi_flush_invalidate_dcache_range(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) struct flush_data *fd = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) __flush_invalidate_dcache_range(fd->addr1, fd->addr2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) static void system_flush_invalidate_dcache_range(unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct flush_data fd = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) .addr1 = start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) .addr2 = size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) on_each_cpu(ipi_flush_invalidate_dcache_range, &fd, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags