Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Xen event channels (2-level ABI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * Jeremy Fitzhardinge <jeremy@xensource.com>, XenSource Inc, 2007
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <asm/sync_bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <asm/xen/hypercall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <asm/xen/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <xen/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <xen/xen-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <xen/events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <xen/interface/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <xen/interface/event_channel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include "events_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * Note sizeof(xen_ulong_t) can be more than sizeof(unsigned long). Be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * careful to only use bitops which allow for this (e.g
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * test_bit/find_first_bit and friends but not __ffs) and to pass
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  * BITS_PER_EVTCHN_WORD as the bitmask length.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) #define BITS_PER_EVTCHN_WORD (sizeof(xen_ulong_t)*8)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  * Make a bitmask (i.e. unsigned long *) of a xen_ulong_t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * array. Primarily to avoid long lines (hence the terse name).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #define BM(x) (unsigned long *)(x)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) /* Find the first set bit in a evtchn mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #define EVTCHN_FIRST_BIT(w) find_first_bit(BM(&(w)), BITS_PER_EVTCHN_WORD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #define EVTCHN_MASK_SIZE (EVTCHN_2L_NR_CHANNELS/BITS_PER_EVTCHN_WORD)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) static DEFINE_PER_CPU(xen_ulong_t [EVTCHN_MASK_SIZE], cpu_evtchn_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) static unsigned evtchn_2l_max_channels(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	return EVTCHN_2L_NR_CHANNELS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) static void evtchn_2l_remove(evtchn_port_t evtchn, unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	clear_bit(evtchn, BM(per_cpu(cpu_evtchn_mask, cpu)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) static void evtchn_2l_bind_to_cpu(evtchn_port_t evtchn, unsigned int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 				  unsigned int old_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	clear_bit(evtchn, BM(per_cpu(cpu_evtchn_mask, old_cpu)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	set_bit(evtchn, BM(per_cpu(cpu_evtchn_mask, cpu)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) static void evtchn_2l_clear_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	sync_clear_bit(port, BM(&s->evtchn_pending[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) static void evtchn_2l_set_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	sync_set_bit(port, BM(&s->evtchn_pending[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) static bool evtchn_2l_is_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	return sync_test_bit(port, BM(&s->evtchn_pending[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) static void evtchn_2l_mask(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 	sync_set_bit(port, BM(&s->evtchn_mask[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) static void evtchn_2l_unmask(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	unsigned int cpu = get_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	int do_hypercall = 0, evtchn_pending = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	smp_wmb();	/* All writes before unmask must be visible. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 	if (unlikely((cpu != cpu_from_evtchn(port))))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 		do_hypercall = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 		 * Need to clear the mask before checking pending to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		 * avoid a race with an event becoming pending.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		 * EVTCHNOP_unmask will only trigger an upcall if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 		 * mask bit was set, so if a hypercall is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		 * remask the event.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		sync_clear_bit(port, BM(&s->evtchn_mask[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 		evtchn_pending = sync_test_bit(port, BM(&s->evtchn_pending[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 		if (unlikely(evtchn_pending && xen_hvm_domain())) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 			sync_set_bit(port, BM(&s->evtchn_mask[0]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 			do_hypercall = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	/* Slow path (hypercall) if this is a non-local port or if this is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	 * an hvm domain and an event is pending (hvm domains don't have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	 * their own implementation of irq_enable). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	if (do_hypercall) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 		struct evtchn_unmask unmask = { .port = port };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 		struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		 * The following is basically the equivalent of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 		 * 'hw_resend_irq'. Just like a real IO-APIC we 'lose
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		 * the interrupt edge' if the channel is masked.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 		if (evtchn_pending &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 		    !sync_test_and_set_bit(port / BITS_PER_EVTCHN_WORD,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 					   BM(&vcpu_info->evtchn_pending_sel)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 			vcpu_info->evtchn_upcall_pending = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	put_cpu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) static DEFINE_PER_CPU(unsigned int, current_word_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static DEFINE_PER_CPU(unsigned int, current_bit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)  * Mask out the i least significant bits of w
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #define MASK_LSBS(w, i) (w & ((~((xen_ulong_t)0UL)) << i))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static inline xen_ulong_t active_evtchns(unsigned int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 					 struct shared_info *sh,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 					 unsigned int idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	return sh->evtchn_pending[idx] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		per_cpu(cpu_evtchn_mask, cpu)[idx] &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		~sh->evtchn_mask[idx];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)  * Search the CPU's pending events bitmasks.  For each one found, map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)  * the event number to an irq, and feed it into do_IRQ() for handling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)  * Xen uses a two-level bitmap to speed searching.  The first level is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)  * a bitset of words which contain pending event bits.  The second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)  * level is a bitset of pending events themselves.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void evtchn_2l_handle_events(unsigned cpu, struct evtchn_loop_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	int irq;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	xen_ulong_t pending_words;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	xen_ulong_t pending_bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	int start_word_idx, start_bit_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	int word_idx, bit_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	struct shared_info *s = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	struct vcpu_info *vcpu_info = __this_cpu_read(xen_vcpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	/* Timer interrupt has highest priority. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	irq = irq_from_virq(cpu, VIRQ_TIMER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	if (irq != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		evtchn_port_t evtchn = evtchn_from_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		word_idx = evtchn / BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 		bit_idx = evtchn % BITS_PER_LONG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		if (active_evtchns(cpu, s, word_idx) & (1ULL << bit_idx))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 			generic_handle_irq(irq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	 * Master flag must be cleared /before/ clearing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	 * selector flag. xchg_xen_ulong must contain an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	 * appropriate barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 	pending_words = xchg_xen_ulong(&vcpu_info->evtchn_pending_sel, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	start_word_idx = __this_cpu_read(current_word_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	start_bit_idx = __this_cpu_read(current_bit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	word_idx = start_word_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	for (i = 0; pending_words != 0; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 		xen_ulong_t words;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 		words = MASK_LSBS(pending_words, word_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		 * If we masked out all events, wrap to beginning.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 		if (words == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 			word_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 			bit_idx = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 		word_idx = EVTCHN_FIRST_BIT(words);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		pending_bits = active_evtchns(cpu, s, word_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		bit_idx = 0; /* usually scan entire word from start */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 		 * We scan the starting word in two parts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		 * 1st time: start in the middle, scanning the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		 * upper bits.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 		 * 2nd time: scan the whole word (not just the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 		 * parts skipped in the first pass) -- if an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 		 * event in the previously scanned bits is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		 * pending again it would just be scanned on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 		 * the next loop anyway.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		if (word_idx == start_word_idx) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 			if (i == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 				bit_idx = start_bit_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			xen_ulong_t bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 			evtchn_port_t port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			bits = MASK_LSBS(pending_bits, bit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 			/* If we masked out all events, move on. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			if (bits == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 				break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			bit_idx = EVTCHN_FIRST_BIT(bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 			/* Process port. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 			port = (word_idx * BITS_PER_EVTCHN_WORD) + bit_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 			handle_irq_for_port(port, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 			bit_idx = (bit_idx + 1) % BITS_PER_EVTCHN_WORD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 			/* Next caller starts at last processed + 1 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 			__this_cpu_write(current_word_idx,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 					 bit_idx ? word_idx :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 					 (word_idx+1) % BITS_PER_EVTCHN_WORD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 			__this_cpu_write(current_bit_idx, bit_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 		} while (bit_idx != 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		/* Scan start_l1i twice; all others once. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 		if ((word_idx != start_word_idx) || (i != 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 			pending_words &= ~(1UL << word_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 		word_idx = (word_idx + 1) % BITS_PER_EVTCHN_WORD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) irqreturn_t xen_debug_interrupt(int irq, void *dev_id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	struct shared_info *sh = HYPERVISOR_shared_info;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	xen_ulong_t *cpu_evtchn = per_cpu(cpu_evtchn_mask, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	static DEFINE_SPINLOCK(debug_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	struct vcpu_info *v;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	spin_lock_irqsave(&debug_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	printk("\nvcpu %d\n  ", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	for_each_online_cpu(i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 		int pending;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 		v = per_cpu(xen_vcpu, i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 		pending = (get_irq_regs() && i == cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 			? xen_irqs_disabled(get_irq_regs())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 			: v->evtchn_upcall_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 		printk("%d: masked=%d pending=%d event_sel %0*"PRI_xen_ulong"\n  ", i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 		       pending, v->evtchn_upcall_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 		       (int)(sizeof(v->evtchn_pending_sel)*2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 		       v->evtchn_pending_sel);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	v = per_cpu(xen_vcpu, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 	printk("\npending:\n   ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 	for (i = ARRAY_SIZE(sh->evtchn_pending)-1; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 		printk("%0*"PRI_xen_ulong"%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 		       (int)sizeof(sh->evtchn_pending[0])*2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 		       sh->evtchn_pending[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 		       i % 8 == 0 ? "\n   " : " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	printk("\nglobal mask:\n   ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 		printk("%0*"PRI_xen_ulong"%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		       (int)(sizeof(sh->evtchn_mask[0])*2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 		       sh->evtchn_mask[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 		       i % 8 == 0 ? "\n   " : " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	printk("\nglobally unmasked:\n   ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 		printk("%0*"PRI_xen_ulong"%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 		       (int)(sizeof(sh->evtchn_mask[0])*2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 		       sh->evtchn_pending[i] & ~sh->evtchn_mask[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 		       i % 8 == 0 ? "\n   " : " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	printk("\nlocal cpu%d mask:\n   ", cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	for (i = (EVTCHN_2L_NR_CHANNELS/BITS_PER_EVTCHN_WORD)-1; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 		printk("%0*"PRI_xen_ulong"%s", (int)(sizeof(cpu_evtchn[0])*2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 		       cpu_evtchn[i],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		       i % 8 == 0 ? "\n   " : " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 	printk("\nlocally unmasked:\n   ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 	for (i = ARRAY_SIZE(sh->evtchn_mask)-1; i >= 0; i--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 		xen_ulong_t pending = sh->evtchn_pending[i]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 			& ~sh->evtchn_mask[i]
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 			& cpu_evtchn[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 		printk("%0*"PRI_xen_ulong"%s",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 		       (int)(sizeof(sh->evtchn_mask[0])*2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		       pending, i % 8 == 0 ? "\n   " : " ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 	printk("\npending list:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	for (i = 0; i < EVTCHN_2L_NR_CHANNELS; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 		if (sync_test_bit(i, BM(sh->evtchn_pending))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 			int word_idx = i / BITS_PER_EVTCHN_WORD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 			printk("  %d: event %d -> irq %d%s%s%s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 			       cpu_from_evtchn(i), i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 			       get_evtchn_to_irq(i),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 			       sync_test_bit(word_idx, BM(&v->evtchn_pending_sel))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) 			       ? "" : " l2-clear",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 			       !sync_test_bit(i, BM(sh->evtchn_mask))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) 			       ? "" : " globally-masked",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 			       sync_test_bit(i, BM(cpu_evtchn))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 			       ? "" : " locally-masked");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 	spin_unlock_irqrestore(&debug_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	return IRQ_HANDLED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static void evtchn_2l_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 	for_each_online_cpu(i)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 		memset(per_cpu(cpu_evtchn_mask, i), 0, sizeof(xen_ulong_t) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 				EVTCHN_2L_NR_CHANNELS/BITS_PER_EVTCHN_WORD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) static int evtchn_2l_percpu_deinit(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 	memset(per_cpu(cpu_evtchn_mask, cpu), 0, sizeof(xen_ulong_t) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 			EVTCHN_2L_NR_CHANNELS/BITS_PER_EVTCHN_WORD);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) static const struct evtchn_ops evtchn_ops_2l = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 	.max_channels      = evtchn_2l_max_channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	.nr_channels       = evtchn_2l_max_channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 	.remove            = evtchn_2l_remove,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	.bind_to_cpu       = evtchn_2l_bind_to_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	.clear_pending     = evtchn_2l_clear_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	.set_pending       = evtchn_2l_set_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	.is_pending        = evtchn_2l_is_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	.mask              = evtchn_2l_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	.unmask            = evtchn_2l_unmask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) 	.handle_events     = evtchn_2l_handle_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 	.resume	           = evtchn_2l_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 	.percpu_deinit     = evtchn_2l_percpu_deinit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) void __init xen_evtchn_2l_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	pr_info("Using 2-level ABI\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	evtchn_ops = &evtchn_ops_2l;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) }