^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /******************************************************************************
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * Xen balloon driver - enables returning/claiming memory to/from Xen.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * Copyright (c) 2003, B Dragovic
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (c) 2003-2004, M Williamson, K Fraser
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (c) 2005 Dan M. Smith, IBM Corporation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (c) 2010 Daniel Kiper
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Memory hotplug support was written by Daniel Kiper. Work on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * it was sponsored by Google under Google Summer of Code 2010
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * program. Jeremy Fitzhardinge from Citrix was the mentor for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * this project.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * This program is free software; you can redistribute it and/or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * modify it under the terms of the GNU General Public License version 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * as published by the Free Software Foundation; or, when distributed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * separately from the Linux kernel or incorporated into other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * software packages, subject to the following license:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * Permission is hereby granted, free of charge, to any person obtaining a copy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * of this source file (the "Software"), to deal in the Software without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * restriction, including without limitation the rights to use, copy, modify,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * merge, publish, distribute, sublicense, and/or sell copies of the Software,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * and to permit persons to whom the Software is furnished to do so, subject to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * the following conditions:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * The above copyright notice and this permission notice shall be included in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * all copies or substantial portions of the Software.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * IN THE SOFTWARE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #define pr_fmt(fmt) "xen:" KBUILD_MODNAME ": " fmt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <linux/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <linux/cred.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <linux/freezer.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <linux/kthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <linux/list.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/notifier.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <linux/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <linux/memory_hotplug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <linux/percpu-defs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include <linux/sysctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) #include <linux/moduleparam.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #include <asm/xen/hypervisor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #include <asm/xen/hypercall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #include <xen/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) #include <xen/interface/xen.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) #include <xen/interface/memory.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #include <xen/balloon.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #include <xen/features.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #include <xen/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) #include <xen/mem-reservation.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #undef MODULE_PARAM_PREFIX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define MODULE_PARAM_PREFIX "xen."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static uint __read_mostly balloon_boot_timeout = 180;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) module_param(balloon_boot_timeout, uint, 0444);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static int xen_hotplug_unpopulated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static struct ctl_table balloon_table[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) .procname = "hotplug_unpopulated",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) .data = &xen_hotplug_unpopulated,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) .maxlen = sizeof(int),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) .mode = 0644,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) .proc_handler = proc_dointvec_minmax,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) .extra1 = SYSCTL_ZERO,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .extra2 = SYSCTL_ONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) static struct ctl_table balloon_root[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .procname = "balloon",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) .mode = 0555,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .child = balloon_table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) static struct ctl_table xen_root[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .procname = "xen",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) .mode = 0555,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) .child = balloon_root,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) * Use one extent per PAGE_SIZE to avoid to break down the page into
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) * multiple frame.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) #define EXTENT_ORDER (fls(XEN_PFN_PER_PAGE) - 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) * balloon_thread() state:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) * BP_DONE: done or nothing to do,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) * BP_WAIT: wait to be rescheduled,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) * BP_EAGAIN: error, go to sleep,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) * BP_ECANCELED: error, balloon operation canceled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static enum bp_state {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) BP_DONE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) BP_WAIT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) BP_EAGAIN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) BP_ECANCELED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) } balloon_state = BP_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /* Main waiting point for xen-balloon thread. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static DECLARE_WAIT_QUEUE_HEAD(balloon_thread_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) static DEFINE_MUTEX(balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) struct balloon_stats balloon_stats;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) EXPORT_SYMBOL_GPL(balloon_stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) /* We increase/decrease in batches which fit in a page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) static xen_pfn_t frame_list[PAGE_SIZE / sizeof(xen_pfn_t)];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) /* List of ballooned pages, threaded through the mem_map array. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) static LIST_HEAD(ballooned_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) static DECLARE_WAIT_QUEUE_HEAD(balloon_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /* When ballooning out (allocating memory to return to Xen) we don't really
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) want the kernel to try too hard since that can trigger the oom killer. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #define GFP_BALLOON \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) (GFP_HIGHUSER | __GFP_NOWARN | __GFP_NORETRY | __GFP_NOMEMALLOC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) /* balloon_append: add the given page to the balloon. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) static void balloon_append(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) __SetPageOffline(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) /* Lowmem is re-populated first, so highmem pages go at list tail. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) if (PageHighMem(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) list_add_tail(&page->lru, &ballooned_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) balloon_stats.balloon_high++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) list_add(&page->lru, &ballooned_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) balloon_stats.balloon_low++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) wake_up(&balloon_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) /* balloon_retrieve: rescue a page from the balloon, if it is not empty. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) static struct page *balloon_retrieve(bool require_lowmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (list_empty(&ballooned_pages))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) page = list_entry(ballooned_pages.next, struct page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (require_lowmem && PageHighMem(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) if (PageHighMem(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) balloon_stats.balloon_high--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) balloon_stats.balloon_low--;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) __ClearPageOffline(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) return page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) static struct page *balloon_next_page(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) struct list_head *next = page->lru.next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) if (next == &ballooned_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) return list_entry(next, struct page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) static void update_schedule(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) if (balloon_state == BP_WAIT || balloon_state == BP_ECANCELED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (balloon_state == BP_DONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) balloon_stats.schedule_delay = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) balloon_stats.retry_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) ++balloon_stats.retry_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) if (balloon_stats.max_retry_count != RETRY_UNLIMITED &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) balloon_stats.retry_count > balloon_stats.max_retry_count) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) balloon_stats.schedule_delay = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) balloon_stats.retry_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) balloon_state = BP_ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) balloon_stats.schedule_delay <<= 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) if (balloon_stats.schedule_delay > balloon_stats.max_schedule_delay)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) balloon_stats.schedule_delay = balloon_stats.max_schedule_delay;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) balloon_state = BP_EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) static void release_memory_resource(struct resource *resource)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) if (!resource)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) * No need to reset region to identity mapped since we now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) * know that no I/O can be in this region
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) release_resource(resource);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) kfree(resource);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) static struct resource *additional_memory_resource(phys_addr_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) res = kzalloc(sizeof(*res), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (!res)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) res->name = "System RAM";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) res->flags = IORESOURCE_SYSTEM_RAM | IORESOURCE_BUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) ret = allocate_resource(&iomem_resource, res,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) size, 0, -1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) PAGES_PER_SECTION * PAGE_SIZE, NULL, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) pr_err("Cannot allocate new System RAM resource\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) kfree(res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) return res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static enum bp_state reserve_additional_memory(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) long credit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct resource *resource;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) int nid, rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) unsigned long balloon_hotplug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) credit = balloon_stats.target_pages + balloon_stats.target_unpopulated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) - balloon_stats.total_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) * Already hotplugged enough pages? Wait for them to be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) * onlined.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (credit <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) return BP_WAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) balloon_hotplug = round_up(credit, PAGES_PER_SECTION);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) resource = additional_memory_resource(balloon_hotplug * PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) if (!resource)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) nid = memory_add_physaddr_to_nid(resource->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) #ifdef CONFIG_XEN_HAVE_PVMMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * We don't support PV MMU when Linux and Xen is using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) * different page granularity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) * add_memory() will build page tables for the new memory so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) * the p2m must contain invalid entries so the correct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) * non-present PTEs will be written.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) * If a failure occurs, the original (identity) p2m entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) * are not restored since this region is now known not to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) * conflict with any devices.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) if (!xen_feature(XENFEAT_auto_translated_physmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) unsigned long pfn, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) pfn = PFN_DOWN(resource->start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) for (i = 0; i < balloon_hotplug; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) if (!set_phys_to_machine(pfn + i, INVALID_P2M_ENTRY)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) pr_warn("set_phys_to_machine() failed, no memory added\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) * add_memory_resource() will call online_pages() which in its turn
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * will call xen_online_page() callback causing deadlock if we don't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * release balloon_mutex here. Unlocking here is safe because the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * callers drop the mutex before trying again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) /* add_memory_resource() requires the device_hotplug lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) lock_device_hotplug();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) rc = add_memory_resource(nid, resource, MEMHP_MERGE_RESOURCE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) unlock_device_hotplug();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) pr_warn("Cannot add additional memory (%i)\n", rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) goto err;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) balloon_stats.total_pages += balloon_hotplug;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) return BP_WAIT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) err:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) release_memory_resource(resource);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return BP_ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static void xen_online_page(struct page *page, unsigned int order)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) unsigned long i, size = (1 << order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) unsigned long start_pfn = page_to_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) struct page *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) pr_debug("Online %lu pages starting at pfn 0x%lx\n", size, start_pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) for (i = 0; i < size; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) p = pfn_to_page(start_pfn + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) balloon_append(p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) static int xen_memory_notifier(struct notifier_block *nb, unsigned long val, void *v)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (val == MEM_ONLINE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) wake_up(&balloon_thread_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) return NOTIFY_OK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static struct notifier_block xen_memory_nb = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) .notifier_call = xen_memory_notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) .priority = 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) static enum bp_state reserve_additional_memory(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) balloon_stats.target_pages = balloon_stats.current_pages +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) balloon_stats.target_unpopulated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) return BP_ECANCELED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) #endif /* CONFIG_XEN_BALLOON_MEMORY_HOTPLUG */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) static long current_credit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) return balloon_stats.target_pages - balloon_stats.current_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) static bool balloon_is_inflated(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) return balloon_stats.balloon_low || balloon_stats.balloon_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) static enum bp_state increase_reservation(unsigned long nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) if (nr_pages > ARRAY_SIZE(frame_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) nr_pages = ARRAY_SIZE(frame_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) page = list_first_entry_or_null(&ballooned_pages, struct page, lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) if (!page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) nr_pages = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) frame_list[i] = page_to_xen_pfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) page = balloon_next_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) rc = xenmem_reservation_increase(nr_pages, frame_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) if (rc <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) return BP_EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) for (i = 0; i < rc; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) page = balloon_retrieve(false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) BUG_ON(page == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) xenmem_reservation_va_mapping_update(1, &page, &frame_list[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) /* Relinquish the page back to the allocator. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) free_reserved_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) balloon_stats.current_pages += rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return BP_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) static enum bp_state decrease_reservation(unsigned long nr_pages, gfp_t gfp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) enum bp_state state = BP_DONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) struct page *page, *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) LIST_HEAD(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) if (nr_pages > ARRAY_SIZE(frame_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) nr_pages = ARRAY_SIZE(frame_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) page = alloc_page(gfp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (page == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) nr_pages = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) state = BP_EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) adjust_managed_page_count(page, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) xenmem_reservation_scrub_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) list_add(&page->lru, &pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) * Ensure that ballooned highmem pages don't have kmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * Do this before changing the p2m as kmap_flush_unused()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * reads PTEs to obtain pages (and hence needs the original
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * p2m entry).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) kmap_flush_unused();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) * Setup the frame, update direct mapping, invalidate P2M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) * and add to balloon.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) list_for_each_entry_safe(page, tmp, &pages, lru) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) frame_list[i++] = xen_page_to_gfn(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) xenmem_reservation_va_mapping_reset(1, &page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) balloon_append(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) ret = xenmem_reservation_decrease(nr_pages, frame_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) BUG_ON(ret != nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) balloon_stats.current_pages -= nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) return state;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) * Stop waiting if either state is BP_DONE and ballooning action is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) * needed, or if the credit has changed while state is not BP_DONE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) static bool balloon_thread_cond(long credit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) if (balloon_state == BP_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) credit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) return current_credit() != credit || kthread_should_stop();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) * As this is a kthread it is guaranteed to run as a single instance only.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) * We may of course race updates of the target counts (which are protected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) * by the balloon lock), or with changes to the Xen hard limit, but we will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) * recover from these in time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) static int balloon_thread(void *unused)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) long credit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) unsigned long timeout;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) set_freezable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) switch (balloon_state) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) case BP_DONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) case BP_ECANCELED:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) timeout = 3600 * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) case BP_EAGAIN:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) timeout = balloon_stats.schedule_delay * HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) case BP_WAIT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) timeout = HZ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) credit = current_credit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) wait_event_freezable_timeout(balloon_thread_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) balloon_thread_cond(credit), timeout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) if (kthread_should_stop())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) credit = current_credit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (credit > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) if (balloon_is_inflated())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) balloon_state = increase_reservation(credit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) balloon_state = reserve_additional_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if (credit < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) long n_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) n_pages = min(-credit, si_mem_available());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) balloon_state = decrease_reservation(n_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) GFP_BALLOON);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) if (balloon_state == BP_DONE && n_pages != -credit &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) n_pages < totalreserve_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) balloon_state = BP_EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) update_schedule();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) cond_resched();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /* Resets the Xen limit, sets new target, and kicks off processing. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) void balloon_set_new_target(unsigned long target)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) /* No need for lock. Not read-modify-write updates. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) balloon_stats.target_pages = target;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) wake_up(&balloon_thread_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) EXPORT_SYMBOL_GPL(balloon_set_new_target);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) static int add_ballooned_pages(int nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) enum bp_state st;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (xen_hotplug_unpopulated) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) st = reserve_additional_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (st != BP_ECANCELED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) rc = wait_event_interruptible(balloon_wq,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) !list_empty(&ballooned_pages));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) return rc ? -ENOMEM : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (si_mem_available() < nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) st = decrease_reservation(nr_pages, GFP_USER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) if (st != BP_DONE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) * alloc_xenballooned_pages - get pages that have been ballooned out
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) * @nr_pages: Number of pages to get
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) * @pages: pages returned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) * @return 0 on success, error otherwise
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) int alloc_xenballooned_pages(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) int pgno = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) balloon_stats.target_unpopulated += nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) while (pgno < nr_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) page = balloon_retrieve(true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (page) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) pages[pgno++] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) #ifdef CONFIG_XEN_HAVE_PVMMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) * We don't support PV MMU when Linux and Xen is using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * different page granularity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) BUILD_BUG_ON(XEN_PAGE_SIZE != PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) if (!xen_feature(XENFEAT_auto_translated_physmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) ret = xen_alloc_p2m_entry(page_to_pfn(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) goto out_undo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) ret = add_ballooned_pages(nr_pages - pgno);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) if (ret < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) goto out_undo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) out_undo:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) free_xenballooned_pages(pgno, pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) * NB: free_xenballooned_pages will only subtract pgno pages, but since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) * target_unpopulated is incremented with nr_pages at the start we need
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) * to remove the remaining ones also, or accounting will be screwed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) balloon_stats.target_unpopulated -= nr_pages - pgno;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) EXPORT_SYMBOL(alloc_xenballooned_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * free_xenballooned_pages - return pages retrieved with get_ballooned_pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) * @nr_pages: Number of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * @pages: pages to return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) void free_xenballooned_pages(int nr_pages, struct page **pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) mutex_lock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) if (pages[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) balloon_append(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) balloon_stats.target_unpopulated -= nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /* The balloon may be too large now. Shrink it if needed. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (current_credit())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) wake_up(&balloon_thread_wq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) mutex_unlock(&balloon_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) EXPORT_SYMBOL(free_xenballooned_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) #if defined(CONFIG_XEN_PV) && !defined(CONFIG_XEN_UNPOPULATED_ALLOC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) static void __init balloon_add_region(unsigned long start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) unsigned long pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) unsigned long pfn, extra_pfn_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) * If the amount of usable memory has been limited (e.g., with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) * the 'mem' command line parameter), don't add pages beyond
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) * this limit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) extra_pfn_end = min(max_pfn, start_pfn + pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) for (pfn = start_pfn; pfn < extra_pfn_end; pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) /* totalram_pages and totalhigh_pages do not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) include the boot-time balloon extension, so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) don't subtract from it. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) balloon_append(pfn_to_page(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) balloon_stats.total_pages += extra_pfn_end - start_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) static int __init balloon_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) struct task_struct *task;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (!xen_domain())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) pr_info("Initialising balloon driver\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) #ifdef CONFIG_XEN_PV
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) balloon_stats.current_pages = xen_pv_domain()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) ? min(xen_start_info->nr_pages - xen_released_pages, max_pfn)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) : get_num_physpages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) balloon_stats.current_pages = get_num_physpages();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) balloon_stats.target_pages = balloon_stats.current_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) balloon_stats.balloon_low = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) balloon_stats.balloon_high = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) balloon_stats.total_pages = balloon_stats.current_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) balloon_stats.schedule_delay = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) balloon_stats.max_schedule_delay = 32;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) balloon_stats.retry_count = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) balloon_stats.max_retry_count = 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) #ifdef CONFIG_XEN_BALLOON_MEMORY_HOTPLUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) set_online_page_callback(&xen_online_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) register_memory_notifier(&xen_memory_nb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) register_sysctl_table(xen_root);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) #if defined(CONFIG_XEN_PV) && !defined(CONFIG_XEN_UNPOPULATED_ALLOC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * Initialize the balloon with pages from the extra memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * regions (see arch/x86/xen/setup.c).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) for (i = 0; i < XEN_EXTRA_MEM_MAX_REGIONS; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (xen_extra_mem[i].n_pfns)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) balloon_add_region(xen_extra_mem[i].start_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) xen_extra_mem[i].n_pfns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) task = kthread_run(balloon_thread, NULL, "xen-balloon");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) if (IS_ERR(task)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) pr_err("xen-balloon thread could not be started, ballooning will not work!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return PTR_ERR(task);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) /* Init the xen-balloon driver. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) xen_balloon_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) subsys_initcall(balloon_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) static int __init balloon_wait_finish(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) long credit, last_credit = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) unsigned long last_changed = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) if (!xen_domain())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) return -ENODEV;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) /* PV guests don't need to wait. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (xen_pv_domain() || !current_credit())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) pr_notice("Waiting for initial ballooning down having finished.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) while ((credit = current_credit()) < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) if (credit != last_credit) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) last_changed = jiffies;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) last_credit = credit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) if (balloon_state == BP_ECANCELED) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) pr_warn_once("Initial ballooning failed, %ld pages need to be freed.\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) -credit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) if (jiffies - last_changed >= HZ * balloon_boot_timeout)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) panic("Initial ballooning failed!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) schedule_timeout_interruptible(HZ / 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) pr_notice("Initial ballooning down finished.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) late_initcall_sync(balloon_wait_finish);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags