^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Split spinlock implementation out into its own file, so it can be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * compiled in a FTRACE-compatible way.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/atomic.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <asm/paravirt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <asm/qspinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <xen/events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include "xen-ops.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) static DEFINE_PER_CPU(int, lock_kicker_irq) = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) static DEFINE_PER_CPU(char *, irq_name);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) static DEFINE_PER_CPU(atomic_t, xen_qlock_wait_nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) static bool xen_pvspin = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) static void xen_qlock_kick(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) int irq = per_cpu(lock_kicker_irq, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) /* Don't kick if the target's kicker interrupt is not initialized. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) if (irq == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) xen_send_IPI_one(cpu, XEN_SPIN_UNLOCK_VECTOR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * Halt the current CPU & release it back to the host
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static void xen_qlock_wait(u8 *byte, u8 val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) int irq = __this_cpu_read(lock_kicker_irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) atomic_t *nest_cnt = this_cpu_ptr(&xen_qlock_wait_nest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) /* If kicker interrupts not initialized yet, just spin */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) if (irq == -1 || in_nmi())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) /* Detect reentry. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) atomic_inc(nest_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) /* If irq pending already and no nested call clear it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) if (atomic_read(nest_cnt) == 1 && xen_test_irq_pending(irq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) xen_clear_irq_pending(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) } else if (READ_ONCE(*byte) == val) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) /* Block until irq becomes pending (or a spurious wakeup) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) xen_poll_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) atomic_dec(nest_cnt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) static irqreturn_t dummy_handler(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) void xen_init_lock_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) char *name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) if (!xen_pvspin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) WARN(per_cpu(lock_kicker_irq, cpu) >= 0, "spinlock on CPU%d exists on IRQ%d!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) cpu, per_cpu(lock_kicker_irq, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) name = kasprintf(GFP_KERNEL, "spinlock%d", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) irq = bind_ipi_to_irqhandler(XEN_SPIN_UNLOCK_VECTOR,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) dummy_handler,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) IRQF_PERCPU|IRQF_NOBALANCING,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) name,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) if (irq >= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) disable_irq(irq); /* make sure it's never delivered */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) per_cpu(lock_kicker_irq, cpu) = irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) per_cpu(irq_name, cpu) = name;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) printk("cpu %d spinlock event irq %d\n", cpu, irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) void xen_uninit_lock_cpu(int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) if (!xen_pvspin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * When booting the kernel with 'mitigations=auto,nosmt', the secondary
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) * CPUs are not activated, and lock_kicker_irq is not initialized.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) irq = per_cpu(lock_kicker_irq, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) if (irq == -1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) unbind_from_irqhandler(irq, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) per_cpu(lock_kicker_irq, cpu) = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) kfree(per_cpu(irq_name, cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) per_cpu(irq_name, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) PV_CALLEE_SAVE_REGS_THUNK(xen_vcpu_stolen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) * Our init of PV spinlocks is split in two init functions due to us
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) * using paravirt patching and jump labels patching and having to do
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * all of this before SMP code is invoked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * The paravirt patching needs to be done _before_ the alternative asm code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * is started, otherwise we would not patch the core kernel code.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) void __init xen_init_spinlocks(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) /* Don't need to use pvqspinlock code if there is only 1 vCPU. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) if (num_possible_cpus() == 1 || nopvspin)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) xen_pvspin = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (!xen_pvspin) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) printk(KERN_DEBUG "xen: PV spinlocks disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static_branch_disable(&virt_spin_lock_key);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) printk(KERN_DEBUG "xen: PV spinlocks enabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) __pv_init_lock_hash();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) pv_ops.lock.queued_spin_lock_slowpath = __pv_queued_spin_lock_slowpath;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) pv_ops.lock.queued_spin_unlock =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) PV_CALLEE_SAVE(__pv_queued_spin_unlock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) pv_ops.lock.wait = xen_qlock_wait;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) pv_ops.lock.kick = xen_qlock_kick;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) pv_ops.lock.vcpu_is_preempted = PV_CALLEE_SAVE(xen_vcpu_stolen);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static __init int xen_parse_nopvspin(char *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) pr_notice("\"xen_nopvspin\" is deprecated, please use \"nopvspin\" instead\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) xen_pvspin = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) early_param("xen_nopvspin", xen_parse_nopvspin);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags