^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /******************************************************************************
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * grant_table.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Granting foreign access to our memory reservation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2005-2006, Christopher Clark
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (c) 2004-2005, K A Fraser
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * modify it under the terms of the GNU General Public License version 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * as published by the Free Software Foundation; or, when distributed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * separately from the Linux kernel or incorporated into other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * software packages, subject to the following license:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Permission is hereby granted, free of charge, to any person obtaining a copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * of this source file (the "Software"), to deal in the Software without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * restriction, including without limitation the rights to use, copy, modify,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * merge, publish, distribute, sublicense, and/or sell copies of the Software,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * and to permit persons to whom the Software is furnished to do so, subject to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * the following conditions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * The above copyright notice and this permission notice shall be included in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * all copies or substantial portions of the Software.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * IN THE SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/io.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/workqueue.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/ratelimit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/dma-mapping.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <xen/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <xen/interface/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <xen/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <xen/grant_table.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <xen/interface/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <xen/hvc-console.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <xen/swiotlb-xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <xen/balloon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <asm/xen/cpuid.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <xen/mem-reservation.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <asm/xen/hypercall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) #include <asm/xen/interface.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <asm/sync_bitops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) /* External tools reserve first few grant table entries. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #define NR_RESERVED_ENTRIES 8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #define GNTTAB_LIST_END 0xffffffff
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) static grant_ref_t **gnttab_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) static unsigned int nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static int gnttab_free_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) static grant_ref_t gnttab_free_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) static DEFINE_SPINLOCK(gnttab_list_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) struct grant_frames xen_auto_xlat_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static unsigned int xen_gnttab_version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) module_param_named(version, xen_gnttab_version, uint, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct grant_entry_v1 *v1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) union grant_entry_v2 *v2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) void *addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) } gnttab_shared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) /*This is a structure of function pointers for grant table*/
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct gnttab_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * Version of the grant interface.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) unsigned int version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * Grant refs per grant frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) unsigned int grefs_per_grant_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * Mapping a list of frames for storing grant entries. Frames parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * is used to store grant table address when grant table being setup,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) * nr_gframes is the number of frames to map grant table. Returning
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) * GNTST_okay means success and negative value means failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) int (*map_frames)(xen_pfn_t *frames, unsigned int nr_gframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) * Release a list of frames which are mapped in map_frames for grant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) * entry status.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) void (*unmap_frames)(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) * Introducing a valid entry into the grant table, granting the frame of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) * this grant entry to domain for accessing or transfering. Ref
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) * parameter is reference of this introduced grant entry, domid is id of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) * granted domain, frame is the page frame to be granted, and flags is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) * status of the grant entry to be updated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) void (*update_entry)(grant_ref_t ref, domid_t domid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) unsigned long frame, unsigned flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) * Stop granting a grant entry to domain for accessing. Ref parameter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * reference of a grant entry whose grant access will be stopped,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * readonly is not in use in this function. If the grant entry is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) * currently mapped for reading or writing, just return failure(==0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) * directly and don't tear down the grant access. Otherwise, stop grant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) * access for this entry and return success(==1).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) int (*end_foreign_access_ref)(grant_ref_t ref, int readonly);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * Stop granting a grant entry to domain for transfer. Ref parameter is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * reference of a grant entry whose grant transfer will be stopped. If
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * tranfer has not started, just reclaim the grant entry and return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * failure(==0). Otherwise, wait for the transfer to complete and then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) * return the frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) unsigned long (*end_foreign_transfer_ref)(grant_ref_t ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) * Read the frame number related to a given grant reference.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) unsigned long (*read_frame)(grant_ref_t ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) struct unmap_refs_callback_data {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) struct completion completion;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) int result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) static const struct gnttab_ops *gnttab_interface;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) /* This reflects status of grant entries, so act as a global value. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static grant_status_t *grstatus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) static struct gnttab_free_callback *gnttab_free_callback_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static int gnttab_expand(unsigned int req_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define RPP (PAGE_SIZE / sizeof(grant_ref_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #define SPP (PAGE_SIZE / sizeof(grant_status_t))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static inline grant_ref_t *__gnttab_entry(grant_ref_t entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) return &gnttab_list[(entry) / RPP][(entry) % RPP];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* This can be used as an l-value */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #define gnttab_entry(entry) (*__gnttab_entry(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static int get_free_entries(unsigned count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) int ref, rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) grant_ref_t head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) if ((gnttab_free_count < count) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) ((rc = gnttab_expand(count - gnttab_free_count)) < 0)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) ref = head = gnttab_free_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) gnttab_free_count -= count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) while (count-- > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) head = gnttab_entry(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) gnttab_free_head = gnttab_entry(head);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) gnttab_entry(head) = GNTTAB_LIST_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) static void do_free_callbacks(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) struct gnttab_free_callback *callback, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) callback = gnttab_free_callback_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) gnttab_free_callback_list = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) while (callback != NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) next = callback->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) if (gnttab_free_count >= callback->count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) callback->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) callback->fn(callback->arg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) callback->next = gnttab_free_callback_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) gnttab_free_callback_list = callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) callback = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) static inline void check_free_callbacks(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (unlikely(gnttab_free_callback_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) do_free_callbacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static void put_free_entry(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) gnttab_entry(ref) = gnttab_free_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) gnttab_free_head = ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) gnttab_free_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) check_free_callbacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) * Following applies to gnttab_update_entry_v1 and gnttab_update_entry_v2.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) * Introducing a valid entry into the grant table:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) * 1. Write ent->domid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) * 2. Write ent->frame:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) * GTF_permit_access: Frame to which access is permitted.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * GTF_accept_transfer: Pseudo-phys frame slot being filled by new
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * frame, or zero if none.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * 3. Write memory barrier (WMB).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) * 4. Write ent->flags, inc. valid type.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static void gnttab_update_entry_v1(grant_ref_t ref, domid_t domid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) unsigned long frame, unsigned flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) gnttab_shared.v1[ref].domid = domid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) gnttab_shared.v1[ref].frame = frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) gnttab_shared.v1[ref].flags = flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) static void gnttab_update_entry_v2(grant_ref_t ref, domid_t domid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) unsigned long frame, unsigned int flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) gnttab_shared.v2[ref].hdr.domid = domid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) gnttab_shared.v2[ref].full_page.frame = frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) wmb(); /* Hypervisor concurrent accesses. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) gnttab_shared.v2[ref].hdr.flags = GTF_permit_access | flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * Public grant-issuing interface functions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) void gnttab_grant_foreign_access_ref(grant_ref_t ref, domid_t domid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) unsigned long frame, int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) gnttab_interface->update_entry(ref, domid, frame,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) GTF_permit_access | (readonly ? GTF_readonly : 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) int gnttab_grant_foreign_access(domid_t domid, unsigned long frame,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) int ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) ref = get_free_entries(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (unlikely(ref < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) gnttab_grant_foreign_access_ref(ref, domid, frame, readonly);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) return ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) EXPORT_SYMBOL_GPL(gnttab_grant_foreign_access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) static int gnttab_end_foreign_access_ref_v1(grant_ref_t ref, int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) u16 flags, nflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) u16 *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) pflags = &gnttab_shared.v1[ref].flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) nflags = *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) flags = nflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) if (flags & (GTF_reading|GTF_writing))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) } while ((nflags = sync_cmpxchg(pflags, flags, 0)) != flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static int gnttab_end_foreign_access_ref_v2(grant_ref_t ref, int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) gnttab_shared.v2[ref].hdr.flags = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) mb(); /* Concurrent access by hypervisor. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) if (grstatus[ref] & (GTF_reading|GTF_writing)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) * The read of grstatus needs to have acquire semantics.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * On x86, reads already have that, and we just need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * protect against compiler reorderings.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * On other architectures we may need a full barrier.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) barrier();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) static inline int _gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return gnttab_interface->end_foreign_access_ref(ref, readonly);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) int gnttab_end_foreign_access_ref(grant_ref_t ref, int readonly)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) if (_gnttab_end_foreign_access_ref(ref, readonly))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) pr_warn("WARNING: g.e. %#x still in use!\n", ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) EXPORT_SYMBOL_GPL(gnttab_end_foreign_access_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static unsigned long gnttab_read_frame_v1(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return gnttab_shared.v1[ref].frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static unsigned long gnttab_read_frame_v2(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) return gnttab_shared.v2[ref].full_page.frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) struct deferred_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) struct list_head list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) grant_ref_t ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) bool ro;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) uint16_t warn_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) static LIST_HEAD(deferred_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) static void gnttab_handle_deferred(struct timer_list *);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static DEFINE_TIMER(deferred_timer, gnttab_handle_deferred);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) static void gnttab_handle_deferred(struct timer_list *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) unsigned int nr = 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) struct deferred_entry *first = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) while (nr--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) struct deferred_entry *entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) = list_first_entry(&deferred_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) struct deferred_entry, list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (entry == first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) list_del(&entry->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) if (_gnttab_end_foreign_access_ref(entry->ref, entry->ro)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) put_free_entry(entry->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) pr_debug("freeing g.e. %#x (pfn %#lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) entry->ref, page_to_pfn(entry->page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) put_page(entry->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) kfree(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) entry = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) if (!--entry->warn_delay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) pr_info("g.e. %#x still pending\n", entry->ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) if (!first)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) first = entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) if (entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) list_add_tail(&entry->list, &deferred_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) else if (list_empty(&deferred_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) if (!list_empty(&deferred_list) && !timer_pending(&deferred_timer)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) deferred_timer.expires = jiffies + HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) add_timer(&deferred_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) static void gnttab_add_deferred(grant_ref_t ref, bool readonly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) struct deferred_entry *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) gfp_t gfp = (in_atomic() || irqs_disabled()) ? GFP_ATOMIC : GFP_KERNEL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) const char *what = KERN_WARNING "leaking";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) entry = kmalloc(sizeof(*entry), gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) unsigned long gfn = gnttab_interface->read_frame(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) page = pfn_to_page(gfn_to_pfn(gfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) get_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) entry->ref = ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) entry->ro = readonly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) entry->page = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) entry->warn_delay = 60;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) list_add_tail(&entry->list, &deferred_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (!timer_pending(&deferred_timer)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) deferred_timer.expires = jiffies + HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) add_timer(&deferred_timer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) what = KERN_DEBUG "deferring";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) printk("%s g.e. %#x (pfn %#lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) what, ref, page ? page_to_pfn(page) : -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) int gnttab_try_end_foreign_access(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) int ret = _gnttab_end_foreign_access_ref(ref, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) put_free_entry(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) EXPORT_SYMBOL_GPL(gnttab_try_end_foreign_access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) void gnttab_end_foreign_access(grant_ref_t ref, int readonly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) unsigned long page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (gnttab_try_end_foreign_access(ref)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) if (page != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) put_page(virt_to_page(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) gnttab_add_deferred(ref, readonly,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) page ? virt_to_page(page) : NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) EXPORT_SYMBOL_GPL(gnttab_end_foreign_access);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) int gnttab_grant_foreign_transfer(domid_t domid, unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) int ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) ref = get_free_entries(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (unlikely(ref < 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) gnttab_grant_foreign_transfer_ref(ref, domid, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) return ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) void gnttab_grant_foreign_transfer_ref(grant_ref_t ref, domid_t domid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) unsigned long pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) gnttab_interface->update_entry(ref, domid, pfn, GTF_accept_transfer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) EXPORT_SYMBOL_GPL(gnttab_grant_foreign_transfer_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) static unsigned long gnttab_end_foreign_transfer_ref_v1(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) unsigned long frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) u16 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) u16 *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) pflags = &gnttab_shared.v1[ref].flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * If a transfer is not even yet started, try to reclaim the grant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * reference and return failure (== 0).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) while (!((flags = *pflags) & GTF_transfer_committed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) if (sync_cmpxchg(pflags, flags, 0) == flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) /* If a transfer is in progress then wait until it is completed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) while (!(flags & GTF_transfer_completed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) flags = *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) rmb(); /* Read the frame number /after/ reading completion status. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) frame = gnttab_shared.v1[ref].frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) BUG_ON(frame == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) return frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) static unsigned long gnttab_end_foreign_transfer_ref_v2(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) unsigned long frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) u16 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) u16 *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) pflags = &gnttab_shared.v2[ref].hdr.flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) * If a transfer is not even yet started, try to reclaim the grant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) * reference and return failure (== 0).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) while (!((flags = *pflags) & GTF_transfer_committed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if (sync_cmpxchg(pflags, flags, 0) == flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) /* If a transfer is in progress then wait until it is completed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) while (!(flags & GTF_transfer_completed)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) flags = *pflags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) cpu_relax();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) rmb(); /* Read the frame number /after/ reading completion status. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) frame = gnttab_shared.v2[ref].full_page.frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) BUG_ON(frame == 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) return frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) unsigned long gnttab_end_foreign_transfer_ref(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return gnttab_interface->end_foreign_transfer_ref(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer_ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) unsigned long gnttab_end_foreign_transfer(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) unsigned long frame = gnttab_end_foreign_transfer_ref(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) put_free_entry(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) return frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) EXPORT_SYMBOL_GPL(gnttab_end_foreign_transfer);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) void gnttab_free_grant_reference(grant_ref_t ref)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) put_free_entry(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) EXPORT_SYMBOL_GPL(gnttab_free_grant_reference);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) void gnttab_free_grant_references(grant_ref_t head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) grant_ref_t ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) int count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if (head == GNTTAB_LIST_END)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) ref = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) while (gnttab_entry(ref) != GNTTAB_LIST_END) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) ref = gnttab_entry(ref);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) gnttab_entry(ref) = gnttab_free_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) gnttab_free_head = head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) gnttab_free_count += count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) check_free_callbacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) EXPORT_SYMBOL_GPL(gnttab_free_grant_references);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) int gnttab_alloc_grant_references(u16 count, grant_ref_t *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) int h = get_free_entries(count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) if (h < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) *head = h;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) EXPORT_SYMBOL_GPL(gnttab_alloc_grant_references);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) int gnttab_empty_grant_references(const grant_ref_t *private_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return (*private_head == GNTTAB_LIST_END);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) EXPORT_SYMBOL_GPL(gnttab_empty_grant_references);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) int gnttab_claim_grant_reference(grant_ref_t *private_head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) grant_ref_t g = *private_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) if (unlikely(g == GNTTAB_LIST_END))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) *private_head = gnttab_entry(g);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) return g;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) EXPORT_SYMBOL_GPL(gnttab_claim_grant_reference);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) void gnttab_release_grant_reference(grant_ref_t *private_head,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) grant_ref_t release)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) gnttab_entry(release) = *private_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) *private_head = release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) EXPORT_SYMBOL_GPL(gnttab_release_grant_reference);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) void gnttab_request_free_callback(struct gnttab_free_callback *callback,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) void (*fn)(void *), void *arg, u16 count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) struct gnttab_free_callback *cb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) /* Check if the callback is already on the list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) cb = gnttab_free_callback_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) while (cb) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) if (cb == callback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) cb = cb->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) callback->fn = fn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) callback->arg = arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) callback->count = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) callback->next = gnttab_free_callback_list;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) gnttab_free_callback_list = callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) check_free_callbacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) EXPORT_SYMBOL_GPL(gnttab_request_free_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) void gnttab_cancel_free_callback(struct gnttab_free_callback *callback)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) struct gnttab_free_callback **pcb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) spin_lock_irqsave(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) for (pcb = &gnttab_free_callback_list; *pcb; pcb = &(*pcb)->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (*pcb == callback) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) *pcb = callback->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) spin_unlock_irqrestore(&gnttab_list_lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) EXPORT_SYMBOL_GPL(gnttab_cancel_free_callback);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) static unsigned int gnttab_frames(unsigned int frames, unsigned int align)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) return (frames * gnttab_interface->grefs_per_grant_frame + align - 1) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) static int grow_gnttab_list(unsigned int more_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) unsigned int new_nr_grant_frames, extra_entries, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) unsigned int nr_glist_frames, new_nr_glist_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) unsigned int grefs_per_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) grefs_per_frame = gnttab_interface->grefs_per_grant_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) new_nr_grant_frames = nr_grant_frames + more_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) extra_entries = more_frames * grefs_per_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) new_nr_glist_frames = gnttab_frames(new_nr_grant_frames, RPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) for (i = nr_glist_frames; i < new_nr_glist_frames; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) if (!gnttab_list[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) goto grow_nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) for (i = grefs_per_frame * nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) i < grefs_per_frame * new_nr_grant_frames - 1; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) gnttab_entry(i) = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) gnttab_entry(i) = gnttab_free_head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) gnttab_free_head = grefs_per_frame * nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) gnttab_free_count += extra_entries;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) nr_grant_frames = new_nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) check_free_callbacks();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) grow_nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) while (i-- > nr_glist_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) free_page((unsigned long) gnttab_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static unsigned int __max_nr_grant_frames(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) struct gnttab_query_size query;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) query.dom = DOMID_SELF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) rc = HYPERVISOR_grant_table_op(GNTTABOP_query_size, &query, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) if ((rc < 0) || (query.status != GNTST_okay))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) return 4; /* Legacy max supported number of frames */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) return query.max_nr_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) unsigned int gnttab_max_grant_frames(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) unsigned int xen_max = __max_nr_grant_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) static unsigned int boot_max_nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) /* First time, initialize it properly. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) if (!boot_max_nr_grant_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) boot_max_nr_grant_frames = __max_nr_grant_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) if (xen_max > boot_max_nr_grant_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) return boot_max_nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return xen_max;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) EXPORT_SYMBOL_GPL(gnttab_max_grant_frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) int gnttab_setup_auto_xlat_frames(phys_addr_t addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) xen_pfn_t *pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) unsigned int max_nr_gframes = __max_nr_grant_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) void *vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) if (xen_auto_xlat_grant_frames.count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) vaddr = xen_remap(addr, XEN_PAGE_SIZE * max_nr_gframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) if (vaddr == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) pr_warn("Failed to ioremap gnttab share frames (addr=%pa)!\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) &addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) pfn = kcalloc(max_nr_gframes, sizeof(pfn[0]), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) if (!pfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) xen_unmap(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) for (i = 0; i < max_nr_gframes; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) pfn[i] = XEN_PFN_DOWN(addr) + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) xen_auto_xlat_grant_frames.vaddr = vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) xen_auto_xlat_grant_frames.pfn = pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) xen_auto_xlat_grant_frames.count = max_nr_gframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) EXPORT_SYMBOL_GPL(gnttab_setup_auto_xlat_frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) void gnttab_free_auto_xlat_frames(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) if (!xen_auto_xlat_grant_frames.count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) kfree(xen_auto_xlat_grant_frames.pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) xen_unmap(xen_auto_xlat_grant_frames.vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) xen_auto_xlat_grant_frames.pfn = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) xen_auto_xlat_grant_frames.count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) xen_auto_xlat_grant_frames.vaddr = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) EXPORT_SYMBOL_GPL(gnttab_free_auto_xlat_frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) int gnttab_pages_set_private(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) #if BITS_PER_LONG < 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) struct xen_page_foreign *foreign;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) foreign = kzalloc(sizeof(*foreign), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) if (!foreign)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) set_page_private(pages[i], (unsigned long)foreign);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) SetPagePrivate(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) EXPORT_SYMBOL_GPL(gnttab_pages_set_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) * gnttab_alloc_pages - alloc pages suitable for grant mapping into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) * @nr_pages: number of pages to alloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) * @pages: returns the pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) int gnttab_alloc_pages(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) ret = xen_alloc_unpopulated_pages(nr_pages, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) ret = gnttab_pages_set_private(nr_pages, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) gnttab_free_pages(nr_pages, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) EXPORT_SYMBOL_GPL(gnttab_alloc_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) #ifdef CONFIG_XEN_UNPOPULATED_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) static inline void cache_init(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) cache->pages = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) static inline bool cache_empty(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) return !cache->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) static inline struct page *cache_deq(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) page = cache->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) cache->pages = page->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) page->zone_device_data = cache->pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) cache->pages = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) static inline void cache_init(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) INIT_LIST_HEAD(&cache->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) static inline bool cache_empty(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return list_empty(&cache->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) static inline struct page *cache_deq(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) page = list_first_entry(&cache->pages, struct page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) static inline void cache_enq(struct gnttab_page_cache *cache, struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) list_add(&page->lru, &cache->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) void gnttab_page_cache_init(struct gnttab_page_cache *cache)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) spin_lock_init(&cache->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) cache_init(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) cache->num_pages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) EXPORT_SYMBOL_GPL(gnttab_page_cache_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) int gnttab_page_cache_get(struct gnttab_page_cache *cache, struct page **page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) spin_lock_irqsave(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) if (cache_empty(cache)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) return gnttab_alloc_pages(1, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) page[0] = cache_deq(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) cache->num_pages--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) EXPORT_SYMBOL_GPL(gnttab_page_cache_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) void gnttab_page_cache_put(struct gnttab_page_cache *cache, struct page **page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) unsigned int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) spin_lock_irqsave(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) for (i = 0; i < num; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) cache_enq(cache, page[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) cache->num_pages += num;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) EXPORT_SYMBOL_GPL(gnttab_page_cache_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) void gnttab_page_cache_shrink(struct gnttab_page_cache *cache, unsigned int num)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) struct page *page[10];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) unsigned int i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) spin_lock_irqsave(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) while (cache->num_pages > num) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) page[i] = cache_deq(cache);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) cache->num_pages--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (++i == ARRAY_SIZE(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) gnttab_free_pages(i, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) spin_lock_irqsave(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) spin_unlock_irqrestore(&cache->lock, flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) if (i != 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) gnttab_free_pages(i, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) EXPORT_SYMBOL_GPL(gnttab_page_cache_shrink);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) void gnttab_pages_clear_private(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) if (PagePrivate(pages[i])) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) #if BITS_PER_LONG < 64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) kfree((void *)page_private(pages[i]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) ClearPagePrivate(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) EXPORT_SYMBOL_GPL(gnttab_pages_clear_private);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) * gnttab_free_pages - free pages allocated by gnttab_alloc_pages()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) * @nr_pages; number of pages to free
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) * @pages: the pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) void gnttab_free_pages(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) gnttab_pages_clear_private(nr_pages, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) xen_free_unpopulated_pages(nr_pages, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) EXPORT_SYMBOL_GPL(gnttab_free_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) #ifdef CONFIG_XEN_GRANT_DMA_ALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) * gnttab_dma_alloc_pages - alloc DMAable pages suitable for grant mapping into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) * @args: arguments to the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) int gnttab_dma_alloc_pages(struct gnttab_dma_alloc_args *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) unsigned long pfn, start_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) size = args->nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) if (args->coherent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) args->vaddr = dma_alloc_coherent(args->dev, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) &args->dev_bus_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) GFP_KERNEL | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) args->vaddr = dma_alloc_wc(args->dev, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) &args->dev_bus_addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) GFP_KERNEL | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (!args->vaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) pr_debug("Failed to allocate DMA buffer of size %zu\n", size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) start_pfn = __phys_to_pfn(args->dev_bus_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) for (pfn = start_pfn, i = 0; pfn < start_pfn + args->nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) pfn++, i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) struct page *page = pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) args->pages[i] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) args->frames[i] = xen_page_to_gfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) xenmem_reservation_scrub_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) xenmem_reservation_va_mapping_reset(args->nr_pages, args->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) ret = xenmem_reservation_decrease(args->nr_pages, args->frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) if (ret != args->nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) pr_debug("Failed to decrease reservation for DMA buffer\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) ret = gnttab_pages_set_private(args->nr_pages, args->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) goto fail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) fail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) gnttab_dma_free_pages(args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) EXPORT_SYMBOL_GPL(gnttab_dma_alloc_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * gnttab_dma_free_pages - free DMAable pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * @args: arguments to the function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) int gnttab_dma_free_pages(struct gnttab_dma_alloc_args *args)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) size_t size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) gnttab_pages_clear_private(args->nr_pages, args->pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) for (i = 0; i < args->nr_pages; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) args->frames[i] = page_to_xen_pfn(args->pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) ret = xenmem_reservation_increase(args->nr_pages, args->frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) if (ret != args->nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) pr_debug("Failed to increase reservation for DMA buffer\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) xenmem_reservation_va_mapping_update(args->nr_pages, args->pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) args->frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) size = args->nr_pages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) if (args->coherent)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) dma_free_coherent(args->dev, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) args->vaddr, args->dev_bus_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) dma_free_wc(args->dev, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) args->vaddr, args->dev_bus_addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) EXPORT_SYMBOL_GPL(gnttab_dma_free_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) /* Handling of paged out grant targets (GNTST_eagain) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) #define MAX_DELAY 256
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) gnttab_retry_eagain_gop(unsigned int cmd, void *gop, int16_t *status,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) const char *func)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) unsigned delay = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) BUG_ON(HYPERVISOR_grant_table_op(cmd, gop, 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) if (*status == GNTST_eagain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) msleep(delay++);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) } while ((*status == GNTST_eagain) && (delay < MAX_DELAY));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (delay >= MAX_DELAY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) pr_err("%s: %s eagain grant\n", func, current->comm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) *status = GNTST_bad_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) void gnttab_batch_map(struct gnttab_map_grant_ref *batch, unsigned count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) struct gnttab_map_grant_ref *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) if (HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, batch, count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) for (op = batch; op < batch + count; op++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) if (op->status == GNTST_eagain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref, op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) &op->status, __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) EXPORT_SYMBOL_GPL(gnttab_batch_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) void gnttab_batch_copy(struct gnttab_copy *batch, unsigned count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) struct gnttab_copy *op;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) if (HYPERVISOR_grant_table_op(GNTTABOP_copy, batch, count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) for (op = batch; op < batch + count; op++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) if (op->status == GNTST_eagain)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) gnttab_retry_eagain_gop(GNTTABOP_copy, op,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) &op->status, __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) EXPORT_SYMBOL_GPL(gnttab_batch_copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) void gnttab_foreach_grant_in_range(struct page *page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) unsigned int offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) unsigned int len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) xen_grant_fn_t fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) unsigned int goffset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) unsigned int glen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) unsigned long xen_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) len = min_t(unsigned int, PAGE_SIZE - offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) goffset = xen_offset_in_page(offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) xen_pfn = page_to_xen_pfn(page) + XEN_PFN_DOWN(offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) glen = min_t(unsigned int, XEN_PAGE_SIZE - goffset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) fn(pfn_to_gfn(xen_pfn), goffset, glen, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) goffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) xen_pfn++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) len -= glen;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) EXPORT_SYMBOL_GPL(gnttab_foreach_grant_in_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) void gnttab_foreach_grant(struct page **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) unsigned int nr_grefs,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) xen_grant_fn_t fn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) unsigned int goffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) unsigned long xen_pfn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) for (i = 0; i < nr_grefs; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) if ((i % XEN_PFN_PER_PAGE) == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) xen_pfn = page_to_xen_pfn(pages[i / XEN_PFN_PER_PAGE]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) goffset = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) fn(pfn_to_gfn(xen_pfn), goffset, XEN_PAGE_SIZE, data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) goffset += XEN_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) xen_pfn++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) int gnttab_map_refs(struct gnttab_map_grant_ref *map_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) struct gnttab_map_grant_ref *kmap_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) struct page **pages, unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) int i, ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, map_ops, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) for (i = 0; i < count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) switch (map_ops[i].status) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) case GNTST_okay:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) struct xen_page_foreign *foreign;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) SetPageForeign(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) foreign = xen_page_foreign(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) foreign->domid = map_ops[i].dom;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) foreign->gref = map_ops[i].ref;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) case GNTST_no_device_space:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) pr_warn_ratelimited("maptrack limit reached, can't map all guest pages\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) case GNTST_eagain:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) /* Retry eagain maps */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) gnttab_retry_eagain_gop(GNTTABOP_map_grant_ref,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) map_ops + i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) &map_ops[i].status, __func__);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) /* Test status in next loop iteration. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) i--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) return set_foreign_p2m_mapping(map_ops, kmap_ops, pages, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) EXPORT_SYMBOL_GPL(gnttab_map_refs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) int gnttab_unmap_refs(struct gnttab_unmap_grant_ref *unmap_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) struct gnttab_unmap_grant_ref *kunmap_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) struct page **pages, unsigned int count)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, unmap_ops, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) for (i = 0; i < count; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) ClearPageForeign(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) return clear_foreign_p2m_mapping(unmap_ops, kunmap_ops, pages, count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) EXPORT_SYMBOL_GPL(gnttab_unmap_refs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) #define GNTTAB_UNMAP_REFS_DELAY 5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) static void gnttab_unmap_work(struct work_struct *work)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) struct gntab_unmap_queue_data
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) *unmap_data = container_of(work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) struct gntab_unmap_queue_data,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) gnttab_work.work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) if (unmap_data->age != UINT_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) unmap_data->age++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) __gnttab_unmap_refs_async(unmap_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) static void __gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) int pc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) for (pc = 0; pc < item->count; pc++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) if (page_count(item->pages[pc]) > 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) unsigned long delay = GNTTAB_UNMAP_REFS_DELAY * (item->age + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) schedule_delayed_work(&item->gnttab_work,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) msecs_to_jiffies(delay));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) ret = gnttab_unmap_refs(item->unmap_ops, item->kunmap_ops,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) item->pages, item->count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) item->done(ret, item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) void gnttab_unmap_refs_async(struct gntab_unmap_queue_data* item)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) INIT_DELAYED_WORK(&item->gnttab_work, gnttab_unmap_work);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) item->age = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262) __gnttab_unmap_refs_async(item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) EXPORT_SYMBOL_GPL(gnttab_unmap_refs_async);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) static void unmap_refs_callback(int result,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) struct gntab_unmap_queue_data *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) struct unmap_refs_callback_data *d = data->data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) d->result = result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) complete(&d->completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) int gnttab_unmap_refs_sync(struct gntab_unmap_queue_data *item)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) struct unmap_refs_callback_data data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) init_completion(&data.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) item->data = &data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) item->done = &unmap_refs_callback;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) gnttab_unmap_refs_async(item);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) wait_for_completion(&data.completion);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) return data.result;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) EXPORT_SYMBOL_GPL(gnttab_unmap_refs_sync);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) static unsigned int nr_status_frames(unsigned int nr_grant_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) return gnttab_frames(nr_grant_frames, SPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) static int gnttab_map_frames_v1(xen_pfn_t *frames, unsigned int nr_gframes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298) rc = arch_gnttab_map_shared(frames, nr_gframes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) gnttab_max_grant_frames(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) &gnttab_shared.addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) BUG_ON(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) static void gnttab_unmap_frames_v1(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) static int gnttab_map_frames_v2(xen_pfn_t *frames, unsigned int nr_gframes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) uint64_t *sframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) unsigned int nr_sframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) struct gnttab_get_status_frames getframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) nr_sframes = nr_status_frames(nr_gframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) /* No need for kzalloc as it is initialized in following hypercall
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) * GNTTABOP_get_status_frames.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) sframes = kmalloc_array(nr_sframes, sizeof(uint64_t), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) if (!sframes)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) getframes.dom = DOMID_SELF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) getframes.nr_frames = nr_sframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) set_xen_guest_handle(getframes.frame_list, sframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) rc = HYPERVISOR_grant_table_op(GNTTABOP_get_status_frames,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) &getframes, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) if (rc == -ENOSYS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) kfree(sframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) BUG_ON(rc || getframes.status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) rc = arch_gnttab_map_status(sframes, nr_sframes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) nr_status_frames(gnttab_max_grant_frames()),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) &grstatus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) BUG_ON(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) kfree(sframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) rc = arch_gnttab_map_shared(frames, nr_gframes,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) gnttab_max_grant_frames(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) &gnttab_shared.addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) BUG_ON(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) static void gnttab_unmap_frames_v2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) arch_gnttab_unmap(gnttab_shared.addr, nr_grant_frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) arch_gnttab_unmap(grstatus, nr_status_frames(nr_grant_frames));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) static int gnttab_map(unsigned int start_idx, unsigned int end_idx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) struct gnttab_setup_table setup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) xen_pfn_t *frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) unsigned int nr_gframes = end_idx + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) if (xen_feature(XENFEAT_auto_translated_physmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) struct xen_add_to_physmap xatp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) unsigned int i = end_idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) BUG_ON(xen_auto_xlat_grant_frames.count < nr_gframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) * Loop backwards, so that the first hypercall has the largest
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) * index, ensuring that the table will grow only once.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) xatp.domid = DOMID_SELF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) xatp.idx = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) xatp.space = XENMAPSPACE_grant_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) xatp.gpfn = xen_auto_xlat_grant_frames.pfn[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) rc = HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) if (rc != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) pr_warn("grant table add_to_physmap failed, err=%d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) } while (i-- > start_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) /* No need for kzalloc as it is initialized in following hypercall
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) * GNTTABOP_setup_table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) frames = kmalloc_array(nr_gframes, sizeof(unsigned long), GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) if (!frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) setup.dom = DOMID_SELF;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) setup.nr_frames = nr_gframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) set_xen_guest_handle(setup.frame_list, frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) rc = HYPERVISOR_grant_table_op(GNTTABOP_setup_table, &setup, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) if (rc == -ENOSYS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) kfree(frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) BUG_ON(rc || setup.status);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) rc = gnttab_interface->map_frames(frames, nr_gframes);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) kfree(frames);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) static const struct gnttab_ops gnttab_v1_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) .version = 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) .grefs_per_grant_frame = XEN_PAGE_SIZE /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) sizeof(struct grant_entry_v1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) .map_frames = gnttab_map_frames_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) .unmap_frames = gnttab_unmap_frames_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) .update_entry = gnttab_update_entry_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) .end_foreign_access_ref = gnttab_end_foreign_access_ref_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) .read_frame = gnttab_read_frame_v1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) static const struct gnttab_ops gnttab_v2_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) .version = 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) .grefs_per_grant_frame = XEN_PAGE_SIZE /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) sizeof(union grant_entry_v2),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) .map_frames = gnttab_map_frames_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) .unmap_frames = gnttab_unmap_frames_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) .update_entry = gnttab_update_entry_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) .end_foreign_access_ref = gnttab_end_foreign_access_ref_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) .end_foreign_transfer_ref = gnttab_end_foreign_transfer_ref_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) .read_frame = gnttab_read_frame_v2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) static bool gnttab_need_v2(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) #ifdef CONFIG_X86
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) uint32_t base, width;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) if (xen_pv_domain()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) base = xen_cpuid_base();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) if (cpuid_eax(base) < 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) return false; /* Information not available, use V1. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) width = cpuid_ebx(base + 5) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) XEN_CPUID_MACHINE_ADDRESS_WIDTH_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) return width > 32 + PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) return !!(max_possible_pfn >> 32);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) static void gnttab_request_version(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) long rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) struct gnttab_set_version gsv;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) if (gnttab_need_v2())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) gsv.version = 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) gsv.version = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) /* Boot parameter overrides automatic selection. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) if (xen_gnttab_version >= 1 && xen_gnttab_version <= 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) gsv.version = xen_gnttab_version;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) rc = HYPERVISOR_grant_table_op(GNTTABOP_set_version, &gsv, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) if (rc == 0 && gsv.version == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) gnttab_interface = &gnttab_v2_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) gnttab_interface = &gnttab_v1_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) pr_info("Grant tables using version %d layout\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) gnttab_interface->version);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) static int gnttab_setup(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) unsigned int max_nr_gframes;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) max_nr_gframes = gnttab_max_grant_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) if (max_nr_gframes < nr_grant_frames)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) return -ENOSYS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) if (xen_feature(XENFEAT_auto_translated_physmap) && gnttab_shared.addr == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) gnttab_shared.addr = xen_auto_xlat_grant_frames.vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) if (gnttab_shared.addr == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) pr_warn("gnttab share frames is not mapped!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) return gnttab_map(0, nr_grant_frames - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) int gnttab_resume(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) gnttab_request_version();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) return gnttab_setup();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) int gnttab_suspend(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) if (!xen_feature(XENFEAT_auto_translated_physmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) gnttab_interface->unmap_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) static int gnttab_expand(unsigned int req_entries)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) unsigned int cur, extra;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) cur = nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) extra = ((req_entries + gnttab_interface->grefs_per_grant_frame - 1) /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) gnttab_interface->grefs_per_grant_frame);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) if (cur + extra > gnttab_max_grant_frames()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) pr_warn_ratelimited("xen/grant-table: max_grant_frames reached"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) " cur=%u extra=%u limit=%u"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) " gnttab_free_count=%u req_entries=%u\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) cur, extra, gnttab_max_grant_frames(),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) gnttab_free_count, req_entries);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) return -ENOSPC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) rc = gnttab_map(cur, cur + extra - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) if (rc == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) rc = grow_gnttab_list(extra);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) int gnttab_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) unsigned long max_nr_grant_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) unsigned int max_nr_glist_frames, nr_glist_frames;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) unsigned int nr_init_grefs;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) gnttab_request_version();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) max_nr_grant_frames = gnttab_max_grant_frames();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) nr_grant_frames = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) /* Determine the maximum number of frames required for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * grant reference free list on the current hypervisor.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) max_nr_glist_frames = (max_nr_grant_frames *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) gnttab_interface->grefs_per_grant_frame / RPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) gnttab_list = kmalloc_array(max_nr_glist_frames,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) sizeof(grant_ref_t *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) if (gnttab_list == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) nr_glist_frames = gnttab_frames(nr_grant_frames, RPP);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) for (i = 0; i < nr_glist_frames; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) gnttab_list[i] = (grant_ref_t *)__get_free_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) if (gnttab_list[i] == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) ret = -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) goto ini_nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) ret = arch_gnttab_init(max_nr_grant_frames,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) nr_status_frames(max_nr_grant_frames));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) goto ini_nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) if (gnttab_setup() < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) ret = -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) goto ini_nomem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580) nr_init_grefs = nr_grant_frames *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) gnttab_interface->grefs_per_grant_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) for (i = NR_RESERVED_ENTRIES; i < nr_init_grefs - 1; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) gnttab_entry(i) = i + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) gnttab_entry(nr_init_grefs - 1) = GNTTAB_LIST_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) gnttab_free_count = nr_init_grefs - NR_RESERVED_ENTRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) gnttab_free_head = NR_RESERVED_ENTRIES;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) printk("Grant table initialized\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) ini_nomem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) for (i--; i >= 0; i--)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595) free_page((unsigned long)gnttab_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) kfree(gnttab_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) EXPORT_SYMBOL_GPL(gnttab_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) static int __gnttab_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) if (!xen_domain())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) /* Delay grant-table initialization in the PV on HVM case */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607) if (xen_hvm_domain() && !xen_pvh_domain())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) return gnttab_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) /* Starts after core_initcall so that xen_pvh_gnttab_setup can be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) * beforehand to initialize xen_auto_xlat_grant_frames. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) core_initcall_sync(__gnttab_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags