Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  * Xen event channels (FIFO-based ABI)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Copyright (C) 2013 Citrix Systems R&D ltd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * This source code is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * modify it under the terms of the GNU General Public License as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * published by the Free Software Foundation; either version 2 of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * License, or (at your option) any later version.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * Or, when distributed separately from the Linux kernel or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * incorporated into other software packages, subject to the following
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * license:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * Permission is hereby granted, free of charge, to any person obtaining a copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * of this source file (the "Software"), to deal in the Software without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  * restriction, including without limitation the rights to use, copy, modify,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * merge, publish, distribute, sublicense, and/or sell copies of the Software,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * and to permit persons to whom the Software is furnished to do so, subject to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  * the following conditions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22)  * The above copyright notice and this permission notice shall be included in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23)  * all copies or substantial portions of the Software.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25)  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26)  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27)  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30)  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31)  * IN THE SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) #include <linux/linkage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) #include <linux/interrupt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) #include <linux/irq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) #include <linux/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) #include <asm/barrier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) #include <asm/sync_bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) #include <asm/xen/hypercall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) #include <asm/xen/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) #include <xen/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) #include <xen/xen-ops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) #include <xen/events.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) #include <xen/interface/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #include <xen/interface/event_channel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) #include <xen/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) #include "events_internal.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) #define EVENT_WORDS_PER_PAGE (XEN_PAGE_SIZE / sizeof(event_word_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #define MAX_EVENT_ARRAY_PAGES (EVTCHN_FIFO_NR_CHANNELS / EVENT_WORDS_PER_PAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) struct evtchn_fifo_queue {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 	uint32_t head[EVTCHN_FIFO_MAX_QUEUES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) static DEFINE_PER_CPU(struct evtchn_fifo_control_block *, cpu_control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) static DEFINE_PER_CPU(struct evtchn_fifo_queue, cpu_queue);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) static event_word_t *event_array[MAX_EVENT_ARRAY_PAGES] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) static unsigned event_array_pages __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  * sync_set_bit() and friends must be unsigned long aligned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #if BITS_PER_LONG > 32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) #define BM(w) (unsigned long *)((unsigned long)w & ~0x7UL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) #define EVTCHN_FIFO_BIT(b, w) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76)     (((unsigned long)w & 0x4UL) ? (EVTCHN_FIFO_ ##b + 32) : EVTCHN_FIFO_ ##b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) #define BM(w) ((unsigned long *)(w))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) #define EVTCHN_FIFO_BIT(b, w) EVTCHN_FIFO_ ##b
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) static inline event_word_t *event_word_from_port(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	unsigned i = port / EVENT_WORDS_PER_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	return event_array[i] + port % EVENT_WORDS_PER_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) static unsigned evtchn_fifo_max_channels(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 	return EVTCHN_FIFO_NR_CHANNELS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) static unsigned evtchn_fifo_nr_channels(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	return event_array_pages * EVENT_WORDS_PER_PAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) static int init_control_block(int cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)                               struct evtchn_fifo_control_block *control_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 	struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 	struct evtchn_init_control init_control;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 	unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	/* Reset the control block and the local HEADs. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	clear_page(control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 	for (i = 0; i < EVTCHN_FIFO_MAX_QUEUES; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 		q->head[i] = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	init_control.control_gfn = virt_to_gfn(control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	init_control.offset      = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	init_control.vcpu        = xen_vcpu_nr(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	return HYPERVISOR_event_channel_op(EVTCHNOP_init_control, &init_control);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) static void free_unused_array_pages(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	for (i = event_array_pages; i < MAX_EVENT_ARRAY_PAGES; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 		if (!event_array[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 			break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 		free_page((unsigned long)event_array[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 		event_array[i] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static void init_array_page(event_word_t *array_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	unsigned i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 	for (i = 0; i < EVENT_WORDS_PER_PAGE; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 		array_page[i] = 1 << EVTCHN_FIFO_MASKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) static int evtchn_fifo_setup(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	unsigned new_array_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	new_array_pages = port / EVENT_WORDS_PER_PAGE + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	if (new_array_pages > MAX_EVENT_ARRAY_PAGES)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	while (event_array_pages < new_array_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		void *array_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 		struct evtchn_expand_array expand_array;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		/* Might already have a page if we've resumed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 		array_page = event_array[event_array_pages];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 		if (!array_page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 			array_page = (void *)__get_free_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 			if (array_page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 				ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 				goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 			event_array[event_array_pages] = array_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 		/* Mask all events in this page before adding it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 		init_array_page(array_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 		expand_array.array_gfn = virt_to_gfn(array_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		ret = HYPERVISOR_event_channel_op(EVTCHNOP_expand_array, &expand_array);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 		if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 			goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		event_array_pages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)   error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	if (event_array_pages == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		panic("xen: unable to expand event array with initial page (%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		pr_err("unable to expand event array (%d)\n", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	free_unused_array_pages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) static void evtchn_fifo_bind_to_cpu(evtchn_port_t evtchn, unsigned int cpu, 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 				    unsigned int old_cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	/* no-op */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static void evtchn_fifo_clear_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	sync_clear_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) static void evtchn_fifo_set_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	sync_set_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) static bool evtchn_fifo_is_pending(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	return sync_test_bit(EVTCHN_FIFO_BIT(PENDING, word), BM(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static void evtchn_fifo_mask(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	sync_set_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static bool evtchn_fifo_is_masked(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	return sync_test_bit(EVTCHN_FIFO_BIT(MASKED, word), BM(word));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224)  * Clear MASKED if not PENDING, spinning if BUSY is set.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225)  * Return true if mask was cleared.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static bool clear_masked_cond(volatile event_word_t *word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	event_word_t new, old, w;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	w = *word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		if (!(w & (1 << EVTCHN_FIFO_MASKED)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		if (w & (1 << EVTCHN_FIFO_PENDING))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		old = w & ~(1 << EVTCHN_FIFO_BUSY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 		new = old & ~(1 << EVTCHN_FIFO_MASKED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		w = sync_cmpxchg(word, old, new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	} while (w != old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) static void evtchn_fifo_unmask(evtchn_port_t port)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 	event_word_t *word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 	BUG_ON(!irqs_disabled());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	if (!clear_masked_cond(word)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 		struct evtchn_unmask unmask = { .port = port };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 		(void)HYPERVISOR_event_channel_op(EVTCHNOP_unmask, &unmask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) static uint32_t clear_linked(volatile event_word_t *word)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 	event_word_t new, old, w;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	w = *word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 		old = w;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 		new = (w & ~((1 << EVTCHN_FIFO_LINKED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 			     | EVTCHN_FIFO_LINK_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	} while ((w = sync_cmpxchg(word, old, new)) != old);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 	return w & EVTCHN_FIFO_LINK_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) static void consume_one_event(unsigned cpu, struct evtchn_loop_ctrl *ctrl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 			      struct evtchn_fifo_control_block *control_block,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 			      unsigned priority, unsigned long *ready)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	struct evtchn_fifo_queue *q = &per_cpu(cpu_queue, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 	uint32_t head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	evtchn_port_t port;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 	event_word_t *word;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 	head = q->head[priority];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	 * Reached the tail last time?  Read the new HEAD from the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) 	 * control block.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) 	if (head == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) 		virt_rmb(); /* Ensure word is up-to-date before reading head. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) 		head = control_block->head[priority];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 	port = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) 	word = event_word_from_port(port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	head = clear_linked(word);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 	 * If the link is non-zero, there are more events in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) 	 * queue, otherwise the queue is empty.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	 * If the queue is empty, clear this priority from our local
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	 * copy of the ready word.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	if (head == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 		clear_bit(priority, ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	if (evtchn_fifo_is_pending(port) && !evtchn_fifo_is_masked(port)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 		if (unlikely(!ctrl))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 			pr_warn("Dropping pending event for port %u\n", port);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 			handle_irq_for_port(port, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 	q->head[priority] = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static void __evtchn_fifo_handle_events(unsigned cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 					struct evtchn_loop_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 	struct evtchn_fifo_control_block *control_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 	unsigned long ready;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	unsigned q;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	control_block = per_cpu(cpu_control_block, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	ready = xchg(&control_block->ready, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) 	while (ready) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 		q = find_first_bit(&ready, EVTCHN_FIFO_MAX_QUEUES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) 		consume_one_event(cpu, ctrl, control_block, q, &ready);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 		ready |= xchg(&control_block->ready, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static void evtchn_fifo_handle_events(unsigned cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) 				      struct evtchn_loop_ctrl *ctrl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) 	__evtchn_fifo_handle_events(cpu, ctrl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) static void evtchn_fifo_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) 	unsigned cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) 	for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) 		void *control_block = per_cpu(cpu_control_block, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) 		int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) 		if (!control_block)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) 		 * If this CPU is offline, take the opportunity to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) 		 * free the control block while it is not being
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) 		 * used.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) 		if (!cpu_online(cpu)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) 			free_page((unsigned long)control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 			per_cpu(cpu_control_block, cpu) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 			continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) 		ret = init_control_block(cpu, control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) 		BUG_ON(ret < 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) 	 * The event array starts out as empty again and is extended
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 	 * as normal when events are bound.  The existing pages will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) 	 * be reused.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) 	event_array_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static int evtchn_fifo_alloc_control_block(unsigned cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 	void *control_block = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) 	int ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) 	control_block = (void *)__get_free_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) 	if (control_block == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) 		goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) 	ret = init_control_block(cpu, control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) 	if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) 		goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) 	per_cpu(cpu_control_block, cpu) = control_block;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)   error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) 	free_page((unsigned long)control_block);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) static int evtchn_fifo_percpu_init(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) 	if (!per_cpu(cpu_control_block, cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) 		return evtchn_fifo_alloc_control_block(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) static int evtchn_fifo_percpu_deinit(unsigned int cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) 	__evtchn_fifo_handle_events(cpu, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) static const struct evtchn_ops evtchn_ops_fifo = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) 	.max_channels      = evtchn_fifo_max_channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) 	.nr_channels       = evtchn_fifo_nr_channels,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) 	.setup             = evtchn_fifo_setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) 	.bind_to_cpu       = evtchn_fifo_bind_to_cpu,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) 	.clear_pending     = evtchn_fifo_clear_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) 	.set_pending       = evtchn_fifo_set_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) 	.is_pending        = evtchn_fifo_is_pending,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) 	.mask              = evtchn_fifo_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) 	.unmask            = evtchn_fifo_unmask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) 	.handle_events     = evtchn_fifo_handle_events,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) 	.resume            = evtchn_fifo_resume,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) 	.percpu_init       = evtchn_fifo_percpu_init,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) 	.percpu_deinit     = evtchn_fifo_percpu_deinit,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) int __init xen_evtchn_fifo_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) 	int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) 	ret = evtchn_fifo_alloc_control_block(cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) 	if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) 		return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) 	pr_info("Using FIFO-based ABI\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) 	evtchn_ops = &evtchn_ops_fifo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) 	return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) }