^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * kernel/stop_machine.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2008, 2005 IBM Corporation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 2008, 2005 Rusty Russell rusty@rustcorp.com.au
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 2010 SUSE Linux Products GmbH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Copyright (C) 2010 Tejun Heo <tj@kernel.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/compiler.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/completion.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/stop_machine.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/kallsyms.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/smpboot.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/sched/wake_q.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* the actual stopper, one per every possible cpu, enabled on online cpus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) struct cpu_stopper {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) struct task_struct *thread;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) raw_spinlock_t lock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) bool enabled; /* is this stopper enabled? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) struct list_head works; /* list of pending works */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) struct cpu_stop_work stop_work; /* for stop_cpus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static DEFINE_PER_CPU(struct cpu_stopper, cpu_stopper);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) static bool stop_machine_initialized = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) /* static data for stop_cpus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) static DEFINE_MUTEX(stop_cpus_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static bool stop_cpus_in_progress;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) static void cpu_stop_init_done(struct cpu_stop_done *done, unsigned int nr_todo)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) memset(done, 0, sizeof(*done));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) atomic_set(&done->nr_todo, nr_todo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) init_completion(&done->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) /* signal completion unless @done is NULL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) static void cpu_stop_signal_done(struct cpu_stop_done *done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) if (atomic_dec_and_test(&done->nr_todo))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) complete(&done->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) static void __cpu_stop_queue_work(struct cpu_stopper *stopper,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) struct cpu_stop_work *work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) struct wake_q_head *wakeq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) list_add_tail(&work->list, &stopper->works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) wake_q_add(wakeq, stopper->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) /* queue @work to @stopper. if offline, @work is completed immediately */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) static bool cpu_stop_queue_work(unsigned int cpu, struct cpu_stop_work *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) DEFINE_WAKE_Q(wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) bool enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) raw_spin_lock_irqsave(&stopper->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) enabled = stopper->enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) __cpu_stop_queue_work(stopper, work, &wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) else if (work->done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) cpu_stop_signal_done(work->done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) raw_spin_unlock_irqrestore(&stopper->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) wake_up_q(&wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * stop_one_cpu - stop a cpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * @cpu: cpu to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * @fn: function to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) * @arg: argument to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) * Execute @fn(@arg) on @cpu. @fn is run in a process context with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * the highest priority preempting any task on the cpu and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * monopolizing it. This function returns after the execution is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * This function doesn't guarantee @cpu stays online till @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * completes. If @cpu goes down in the middle, execution may happen
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * partially or fully on different cpus. @fn should either be ready
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) * for that or the caller should ensure that @cpu stays online until
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) * this function completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * -ENOENT if @fn(@arg) was not executed because @cpu was offline;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * otherwise, the return value of @fn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) int stop_one_cpu(unsigned int cpu, cpu_stop_fn_t fn, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) struct cpu_stop_done done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) struct cpu_stop_work work = { .fn = fn, .arg = arg, .done = &done };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) cpu_stop_init_done(&done, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (!cpu_stop_queue_work(cpu, &work))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * In case @cpu == smp_proccessor_id() we can avoid a sleep+wakeup
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * cycle by doing a preemption:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) wait_for_completion(&done.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) return done.ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) /* This controls the threads on each CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) enum multi_stop_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) /* Dummy starting state for thread. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) MULTI_STOP_NONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) /* Awaiting everyone to be scheduled. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) MULTI_STOP_PREPARE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Disable interrupts. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) MULTI_STOP_DISABLE_IRQ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) /* Run the function */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) MULTI_STOP_RUN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /* Exit */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) MULTI_STOP_EXIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct multi_stop_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) cpu_stop_fn_t fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) void *data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) /* Like num_online_cpus(), but hotplug cpu uses us, so we need this. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) unsigned int num_threads;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) const struct cpumask *active_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) enum multi_stop_state state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) atomic_t thread_ack;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static void set_state(struct multi_stop_data *msdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) enum multi_stop_state newstate)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* Reset ack counter. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) atomic_set(&msdata->thread_ack, msdata->num_threads);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) WRITE_ONCE(msdata->state, newstate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) /* Last one to ack a state moves to the next state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void ack_state(struct multi_stop_data *msdata)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) if (atomic_dec_and_test(&msdata->thread_ack))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) set_state(msdata, msdata->state + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) notrace void __weak stop_machine_yield(const struct cpumask *cpumask)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /* This is the cpu_stop function which stops the CPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) static int multi_cpu_stop(void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) struct multi_stop_data *msdata = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) enum multi_stop_state newstate, curstate = MULTI_STOP_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) int cpu = smp_processor_id(), err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) const struct cpumask *cpumask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) bool is_active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) * When called from stop_machine_from_inactive_cpu(), irq might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * already be disabled. Save the state and restore it on exit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) local_save_flags(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) if (!msdata->active_cpus) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) cpumask = cpu_online_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) is_active = cpu == cpumask_first(cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) cpumask = msdata->active_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) is_active = cpumask_test_cpu(cpu, cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /* Simple state machine */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) /* Chill out and ensure we re-read multi_stop_state. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) stop_machine_yield(cpumask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) newstate = READ_ONCE(msdata->state);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (newstate != curstate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) curstate = newstate;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) switch (curstate) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) case MULTI_STOP_DISABLE_IRQ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) local_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) case MULTI_STOP_RUN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (is_active)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) err = msdata->fn(msdata->data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) ack_state(msdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) } else if (curstate > MULTI_STOP_PREPARE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * At this stage all other CPUs we depend on must spin
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * in the same loop. Any reason for hard-lockup should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) * be detected and reported on their side.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) touch_nmi_watchdog();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) rcu_momentary_dyntick_idle();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) } while (curstate != MULTI_STOP_EXIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static int cpu_stop_queue_two_works(int cpu1, struct cpu_stop_work *work1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) int cpu2, struct cpu_stop_work *work2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) struct cpu_stopper *stopper1 = per_cpu_ptr(&cpu_stopper, cpu1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) struct cpu_stopper *stopper2 = per_cpu_ptr(&cpu_stopper, cpu2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) DEFINE_WAKE_Q(wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) int err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * The waking up of stopper threads has to happen in the same
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) * scheduling context as the queueing. Otherwise, there is a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) * possibility of one of the above stoppers being woken up by another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) * CPU, and preempting us. This will cause us to not wake up the other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) * stopper forever.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) raw_spin_lock_irq(&stopper1->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) raw_spin_lock_nested(&stopper2->lock, SINGLE_DEPTH_NESTING);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (!stopper1->enabled || !stopper2->enabled) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) err = -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * Ensure that if we race with __stop_cpus() the stoppers won't get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * queued up in reverse order leading to system deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) * We can't miss stop_cpus_in_progress if queue_stop_cpus_work() has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * queued a work on cpu1 but not on cpu2, we hold both locks.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * It can be falsely true but it is safe to spin until it is cleared,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * queue_stop_cpus_work() does everything under preempt_disable().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) if (unlikely(stop_cpus_in_progress)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) err = -EDEADLK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) err = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) __cpu_stop_queue_work(stopper1, work1, &wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) __cpu_stop_queue_work(stopper2, work2, &wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) raw_spin_unlock(&stopper2->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) raw_spin_unlock_irq(&stopper1->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) if (unlikely(err == -EDEADLK)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) while (stop_cpus_in_progress)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) wake_up_q(&wakeq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) return err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * stop_two_cpus - stops two cpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * @cpu1: the cpu to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) * @cpu2: the other cpu to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) * @fn: function to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * @arg: argument to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) * Stops both the current and specified CPU and runs @fn on one of them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) * returns when both are completed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) int stop_two_cpus(unsigned int cpu1, unsigned int cpu2, cpu_stop_fn_t fn, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) struct cpu_stop_done done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) struct cpu_stop_work work1, work2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) struct multi_stop_data msdata;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) msdata = (struct multi_stop_data){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) .fn = fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) .data = arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) .num_threads = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .active_cpus = cpumask_of(cpu1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) work1 = work2 = (struct cpu_stop_work){
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) .fn = multi_cpu_stop,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) .arg = &msdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) .done = &done
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) cpu_stop_init_done(&done, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) set_state(&msdata, MULTI_STOP_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) if (cpu1 > cpu2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) swap(cpu1, cpu2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) if (cpu_stop_queue_two_works(cpu1, &work1, cpu2, &work2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) wait_for_completion(&done.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) return done.ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) * stop_one_cpu_nowait - stop a cpu but don't wait for completion
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) * @cpu: cpu to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) * @fn: function to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) * @arg: argument to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) * @work_buf: pointer to cpu_stop_work structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) * Similar to stop_one_cpu() but doesn't wait for completion. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) * caller is responsible for ensuring @work_buf is currently unused
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) * and will remain untouched until stopper starts executing @fn.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * Don't care.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) * true if cpu_stop_work was queued successfully and @fn will be called,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) * false otherwise.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) bool stop_one_cpu_nowait(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) struct cpu_stop_work *work_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) *work_buf = (struct cpu_stop_work){ .fn = fn, .arg = arg, };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) return cpu_stop_queue_work(cpu, work_buf);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) EXPORT_SYMBOL_GPL(stop_one_cpu_nowait);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) * stop_one_cpu_async - stop a cpu and wait for completion in a separated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) * function: stop_wait_work()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) * @cpu: cpu to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) * @fn: function to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) * @arg: argument to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) * @work_buf: pointer to cpu_stop_work structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) * 0 if cpu_stop_work was queued successfully and @fn will be called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) * ENOENT if @fn(@arg) was not executed because @cpu was offline.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) int stop_one_cpu_async(unsigned int cpu, cpu_stop_fn_t fn, void *arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) struct cpu_stop_work *work_buf,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) struct cpu_stop_done *done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) cpu_stop_init_done(done, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) work_buf->done = done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) work_buf->fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) work_buf->arg = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) if (cpu_stop_queue_work(cpu, work_buf))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) work_buf->done = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) * cpu_stop_work_wait - wait for a stop initiated by stop_one_cpu_async().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) * @work_buf: pointer to cpu_stop_work structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) void cpu_stop_work_wait(struct cpu_stop_work *work_buf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) struct cpu_stop_done *done = work_buf->done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) wait_for_completion(&done->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) work_buf->done = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) static bool queue_stop_cpus_work(const struct cpumask *cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) cpu_stop_fn_t fn, void *arg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) struct cpu_stop_done *done)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) struct cpu_stop_work *work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) bool queued = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * Disable preemption while queueing to avoid getting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) * preempted by a stopper which might wait for other stoppers
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * to enter @fn which can lead to deadlock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) stop_cpus_in_progress = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) for_each_cpu(cpu, cpumask) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) work = &per_cpu(cpu_stopper.stop_work, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) work->fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) work->arg = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) work->done = done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) if (cpu_stop_queue_work(cpu, work))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) queued = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) stop_cpus_in_progress = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return queued;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) static int __stop_cpus(const struct cpumask *cpumask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) cpu_stop_fn_t fn, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) struct cpu_stop_done done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) cpu_stop_init_done(&done, cpumask_weight(cpumask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) if (!queue_stop_cpus_work(cpumask, fn, arg, &done))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) wait_for_completion(&done.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) return done.ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) * stop_cpus - stop multiple cpus
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) * @cpumask: cpus to stop
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) * @fn: function to execute
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) * @arg: argument to @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * Execute @fn(@arg) on online cpus in @cpumask. On each target cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * @fn is run in a process context with the highest priority
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * preempting any task on the cpu and monopolizing it. This function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * returns after all executions are complete.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * This function doesn't guarantee the cpus in @cpumask stay online
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * till @fn completes. If some cpus go down in the middle, execution
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * on the cpu may happen partially or fully on different cpus. @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) * should either be ready for that or the caller should ensure that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * the cpus stay online until this function completes.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * All stop_cpus() calls are serialized making it safe for @fn to wait
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * for all cpus to start executing it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) * Might sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) * -ENOENT if @fn(@arg) was not executed at all because all cpus in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) * @cpumask were offline; otherwise, 0 if all executions of @fn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) * returned 0, any non zero return value if any returned non zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) static int stop_cpus(const struct cpumask *cpumask, cpu_stop_fn_t fn, void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) /* static works are used, process one request at a time */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) mutex_lock(&stop_cpus_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) ret = __stop_cpus(cpumask, fn, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) mutex_unlock(&stop_cpus_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) static int cpu_stop_should_run(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) int run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) raw_spin_lock_irqsave(&stopper->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) run = !list_empty(&stopper->works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) raw_spin_unlock_irqrestore(&stopper->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) return run;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) static void cpu_stopper_thread(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) struct cpu_stop_work *work;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) repeat:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) work = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) raw_spin_lock_irq(&stopper->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) if (!list_empty(&stopper->works)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) work = list_first_entry(&stopper->works,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) struct cpu_stop_work, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) list_del_init(&work->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) raw_spin_unlock_irq(&stopper->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) if (work) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) cpu_stop_fn_t fn = work->fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) void *arg = work->arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) struct cpu_stop_done *done = work->done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /* cpu stop callbacks must not sleep, make in_atomic() == T */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) preempt_count_inc();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) ret = fn(arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) if (done) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) done->ret = ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) cpu_stop_signal_done(done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) preempt_count_dec();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) WARN_ONCE(preempt_count(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) "cpu_stop: %ps(%p) leaked preempt count\n", fn, arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) goto repeat;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) void stop_machine_park(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) * Lockless. cpu_stopper_thread() will take stopper->lock and flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) * the pending works before it parks, until then it is fine to queue
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) * the new works.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) stopper->enabled = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) kthread_park(stopper->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) extern void sched_set_stop_task(int cpu, struct task_struct *stop);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) static void cpu_stop_create(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) sched_set_stop_task(cpu, per_cpu(cpu_stopper.thread, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) static void cpu_stop_park(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) WARN_ON(!list_empty(&stopper->works));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) void stop_machine_unpark(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) stopper->enabled = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) kthread_unpark(stopper->thread);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) static struct smp_hotplug_thread cpu_stop_threads = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) .store = &cpu_stopper.thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) .thread_should_run = cpu_stop_should_run,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) .thread_fn = cpu_stopper_thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) .thread_comm = "migration/%u",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) .create = cpu_stop_create,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) .park = cpu_stop_park,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) .selfparking = true,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) static int __init cpu_stop_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) struct cpu_stopper *stopper = &per_cpu(cpu_stopper, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) raw_spin_lock_init(&stopper->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) INIT_LIST_HEAD(&stopper->works);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) BUG_ON(smpboot_register_percpu_thread(&cpu_stop_threads));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) stop_machine_unpark(raw_smp_processor_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) stop_machine_initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) early_initcall(cpu_stop_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) int stop_machine_cpuslocked(cpu_stop_fn_t fn, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) const struct cpumask *cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) struct multi_stop_data msdata = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) .fn = fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) .data = data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) .num_threads = num_online_cpus(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) .active_cpus = cpus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) lockdep_assert_cpus_held();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) if (!stop_machine_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) * Handle the case where stop_machine() is called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) * early in boot before stop_machine() has been
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) * initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) WARN_ON_ONCE(msdata.num_threads != 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) hard_irq_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) ret = (*fn)(data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) /* Set the initial state and stop all online cpus. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) set_state(&msdata, MULTI_STOP_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) return stop_cpus(cpu_online_mask, multi_cpu_stop, &msdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) int stop_machine(cpu_stop_fn_t fn, void *data, const struct cpumask *cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /* No CPUs can come up or down during this. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) cpus_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) ret = stop_machine_cpuslocked(fn, data, cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) cpus_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) EXPORT_SYMBOL_GPL(stop_machine);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * stop_machine_from_inactive_cpu - stop_machine() from inactive CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * @fn: the function to run
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * @data: the data ptr for the @fn()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * @cpus: the cpus to run the @fn() on (NULL = any online cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * This is identical to stop_machine() but can be called from a CPU which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * is not active. The local CPU is in the process of hotplug (so no other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * CPU hotplug can start) and not marked active and doesn't have enough
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * context to sleep.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * This function provides stop_machine() functionality for such state by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * using busy-wait for synchronization and executing @fn directly for local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * CPU.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * CONTEXT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * Local CPU is inactive. Temporarily stops all active CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * RETURNS:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) * 0 if all executions of @fn returned 0, any non zero return value if any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) * returned non zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) int stop_machine_from_inactive_cpu(cpu_stop_fn_t fn, void *data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) const struct cpumask *cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) struct multi_stop_data msdata = { .fn = fn, .data = data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) .active_cpus = cpus };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) struct cpu_stop_done done;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) /* Local CPU must be inactive and CPU hotplug in progress. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) BUG_ON(cpu_active(raw_smp_processor_id()));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) msdata.num_threads = num_active_cpus() + 1; /* +1 for local */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) /* No proper task established and can't sleep - busy wait for lock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) while (!mutex_trylock(&stop_cpus_mutex))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) /* Schedule work on other CPUs and execute directly for local CPU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) set_state(&msdata, MULTI_STOP_PREPARE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) cpu_stop_init_done(&done, num_active_cpus());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) queue_stop_cpus_work(cpu_active_mask, multi_cpu_stop, &msdata,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) &done);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) ret = multi_cpu_stop(&msdata);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /* Busy wait for completion. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) while (!completion_done(&done.completion))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) mutex_unlock(&stop_cpus_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return ret ?: done.ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags