^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * KVM guest address space mapping code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright IBM Corp. 2007, 2016, 2018
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * David Hildenbrand <david@redhat.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Janosch Frank <frankja@linux.vnet.ibm.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/pagewalk.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/ksm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/pgtable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/gmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #define GMAP_SHADOW_FAKE_TABLE 1ULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * gmap_alloc - allocate and initialize a guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * @mm: pointer to the parent mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * @limit: maximum address of the gmap address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * Returns a guest address space structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static struct gmap *gmap_alloc(unsigned long limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) unsigned long etype, atype;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) if (limit < _REGION3_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) limit = _REGION3_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) atype = _ASCE_TYPE_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) etype = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) } else if (limit < _REGION2_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) limit = _REGION2_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) atype = _ASCE_TYPE_REGION3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) etype = _REGION3_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) } else if (limit < _REGION1_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) limit = _REGION1_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) atype = _ASCE_TYPE_REGION2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) etype = _REGION2_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) limit = -1UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) atype = _ASCE_TYPE_REGION1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) etype = _REGION1_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) gmap = kzalloc(sizeof(struct gmap), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) if (!gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) INIT_LIST_HEAD(&gmap->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) INIT_LIST_HEAD(&gmap->children);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) INIT_LIST_HEAD(&gmap->pt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) INIT_RADIX_TREE(&gmap->guest_to_host, GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) INIT_RADIX_TREE(&gmap->host_to_guest, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) INIT_RADIX_TREE(&gmap->host_to_rmap, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) spin_lock_init(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) spin_lock_init(&gmap->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) refcount_set(&gmap->ref_count, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) page->index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) list_add(&page->lru, &gmap->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) table = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) crst_table_init(table, etype);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) gmap->table = table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) gmap->asce = atype | _ASCE_TABLE_LENGTH |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) _ASCE_USER_BITS | __pa(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) gmap->asce_end = limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) return gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) kfree(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) * gmap_create - create a guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) * @mm: pointer to the parent mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) * @limit: maximum size of the gmap address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * Returns a guest address space structure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) struct gmap *gmap_create(struct mm_struct *mm, unsigned long limit)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long gmap_asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) gmap = gmap_alloc(limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) if (!gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) gmap->mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) spin_lock(&mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) list_add_rcu(&gmap->list, &mm->context.gmap_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) if (list_is_singular(&mm->context.gmap_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) gmap_asce = gmap->asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) gmap_asce = -1UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) WRITE_ONCE(mm->context.gmap_asce, gmap_asce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) spin_unlock(&mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) return gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) EXPORT_SYMBOL_GPL(gmap_create);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) static void gmap_flush_tlb(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) if (MACHINE_HAS_IDTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) __tlb_flush_idte(gmap->asce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) __tlb_flush_global();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) static void gmap_radix_tree_free(struct radix_tree_root *root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) struct radix_tree_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) unsigned long indices[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) unsigned long index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) void __rcu **slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) int i, nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) /* A radix tree is freed by deleting all of its entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) radix_tree_for_each_slot(slot, root, &iter, index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) indices[nr] = iter.index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) if (++nr == 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) radix_tree_delete(root, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) } while (nr > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) static void gmap_rmap_radix_tree_free(struct radix_tree_root *root)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) struct gmap_rmap *rmap, *rnext, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) struct radix_tree_iter iter;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) unsigned long indices[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) unsigned long index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) void __rcu **slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) int i, nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) /* A radix tree is freed by deleting all of its entries */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) index = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) do {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) radix_tree_for_each_slot(slot, root, &iter, index) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) indices[nr] = iter.index;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) if (++nr == 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) for (i = 0; i < nr; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) index = indices[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) head = radix_tree_delete(root, index);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) gmap_for_each_rmap_safe(rmap, rnext, head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) kfree(rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) } while (nr > 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) * gmap_free - free a guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * No locks required. There are no references to this gmap anymore.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) static void gmap_free(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) struct page *page, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) /* Flush tlb of all gmaps (if not already done for shadows) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (!(gmap_is_shadow(gmap) && gmap->removed))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) gmap_flush_tlb(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* Free all segment & region tables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) list_for_each_entry_safe(page, next, &gmap->crst_list, lru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) gmap_radix_tree_free(&gmap->guest_to_host);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) gmap_radix_tree_free(&gmap->host_to_guest);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) /* Free additional data for a shadow gmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (gmap_is_shadow(gmap)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) /* Free all page tables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) list_for_each_entry_safe(page, next, &gmap->pt_list, lru)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) page_table_free_pgste(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) gmap_rmap_radix_tree_free(&gmap->host_to_rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) /* Release reference to the parent */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) gmap_put(gmap->parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) kfree(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) * gmap_get - increase reference counter for guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) * Returns the gmap pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) struct gmap *gmap_get(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) refcount_inc(&gmap->ref_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) return gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) EXPORT_SYMBOL_GPL(gmap_get);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * gmap_put - decrease reference counter for guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) * If the reference counter reaches zero the guest address space is freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) void gmap_put(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) if (refcount_dec_and_test(&gmap->ref_count))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) gmap_free(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) EXPORT_SYMBOL_GPL(gmap_put);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * gmap_remove - remove a guest address space but do not free it yet
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) void gmap_remove(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) struct gmap *sg, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) unsigned long gmap_asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) /* Remove all shadow gmaps linked to this gmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) if (!list_empty(&gmap->children)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) spin_lock(&gmap->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) list_for_each_entry_safe(sg, next, &gmap->children, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) list_del(&sg->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) gmap_put(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) spin_unlock(&gmap->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) /* Remove gmap from the pre-mm list */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) spin_lock(&gmap->mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) list_del_rcu(&gmap->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) if (list_empty(&gmap->mm->context.gmap_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) gmap_asce = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) else if (list_is_singular(&gmap->mm->context.gmap_list))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) gmap_asce = list_first_entry(&gmap->mm->context.gmap_list,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) struct gmap, list)->asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) gmap_asce = -1UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) WRITE_ONCE(gmap->mm->context.gmap_asce, gmap_asce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) spin_unlock(&gmap->mm->context.lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) /* Put reference */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) gmap_put(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) EXPORT_SYMBOL_GPL(gmap_remove);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) * gmap_enable - switch primary space to the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) void gmap_enable(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) S390_lowcore.gmap = (unsigned long) gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) EXPORT_SYMBOL_GPL(gmap_enable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) * gmap_disable - switch back to the standard primary address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) void gmap_disable(struct gmap *gmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) S390_lowcore.gmap = 0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) EXPORT_SYMBOL_GPL(gmap_disable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) * gmap_get_enabled - get a pointer to the currently enabled gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * Returns a pointer to the currently enabled gmap. 0 if none is enabled.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) struct gmap *gmap_get_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return (struct gmap *) S390_lowcore.gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) EXPORT_SYMBOL_GPL(gmap_get_enabled);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) * gmap_alloc_table is assumed to be called with mmap_lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static int gmap_alloc_table(struct gmap *gmap, unsigned long *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) unsigned long init, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) unsigned long *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) /* since we dont free the gmap table until gmap_free we can unlock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) new = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) crst_table_init(new, init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) if (*table & _REGION_ENTRY_INVALID) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) list_add(&page->lru, &gmap->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) *table = (unsigned long) new | _REGION_ENTRY_LENGTH |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) (*table & _REGION_ENTRY_TYPE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) page->index = gaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) page = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * __gmap_segment_gaddr - find virtual address from segment pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * @entry: pointer to a segment table entry in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) * Returns the virtual address in the guest address space for the segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) static unsigned long __gmap_segment_gaddr(unsigned long *entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) unsigned long offset, mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) offset = (unsigned long) entry / sizeof(unsigned long);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) offset = (offset & (PTRS_PER_PMD - 1)) * PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) page = virt_to_page((void *)((unsigned long) entry & mask));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) return page->index + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) * __gmap_unlink_by_vmaddr - unlink a single segment via a host address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) * @vmaddr: address in the host process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) * Returns 1 if a TLB flush is required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) static int __gmap_unlink_by_vmaddr(struct gmap *gmap, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) unsigned long *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) int flush = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) entry = radix_tree_delete(&gmap->host_to_guest, vmaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) flush = (*entry != _SEGMENT_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) *entry = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) return flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) * __gmap_unmap_by_gaddr - unmap a single segment via a guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * @gaddr: address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * Returns 1 if a TLB flush is required
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) static int __gmap_unmap_by_gaddr(struct gmap *gmap, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) unsigned long vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) vmaddr = (unsigned long) radix_tree_delete(&gmap->guest_to_host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) gaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) return vmaddr ? __gmap_unlink_by_vmaddr(gmap, vmaddr) : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) * gmap_unmap_segment - unmap segment from the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) * @to: address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) * @len: length of the memory area to unmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) * Returns 0 if the unmap succeeded, -EINVAL if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) int gmap_unmap_segment(struct gmap *gmap, unsigned long to, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) unsigned long off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) int flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) if ((to | len) & (PMD_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) if (len == 0 || to + len < to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) flush = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) mmap_write_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) for (off = 0; off < len; off += PMD_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) flush |= __gmap_unmap_by_gaddr(gmap, to + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) mmap_write_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) gmap_flush_tlb(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) EXPORT_SYMBOL_GPL(gmap_unmap_segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) * gmap_map_segment - map a segment to the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) * @from: source address in the parent address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) * @to: target address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) * @len: length of the memory area to map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) * Returns 0 if the mmap succeeded, -EINVAL or -ENOMEM if not.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) int gmap_map_segment(struct gmap *gmap, unsigned long from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) unsigned long to, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) unsigned long off;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) int flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) if ((from | to | len) & (PMD_SIZE - 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (len == 0 || from + len < from || to + len < to ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) from + len - 1 > TASK_SIZE_MAX || to + len - 1 > gmap->asce_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) flush = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) mmap_write_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) for (off = 0; off < len; off += PMD_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) /* Remove old translation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) flush |= __gmap_unmap_by_gaddr(gmap, to + off);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) /* Store new translation */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) if (radix_tree_insert(&gmap->guest_to_host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) (to + off) >> PMD_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) (void *) from + off))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) mmap_write_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) gmap_flush_tlb(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) if (off >= len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) gmap_unmap_segment(gmap, to, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) EXPORT_SYMBOL_GPL(gmap_map_segment);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) * __gmap_translate - translate a guest address to a user space address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) * @gaddr: guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) * Returns user space address which corresponds to the guest address or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) * -EFAULT if no such mapping exists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) * This function does not establish potentially missing page table entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) * The mmap_lock of the mm that belongs to the address space must be held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) * when this function gets called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) * Note: Can also be called for shadow gmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) unsigned long __gmap_translate(struct gmap *gmap, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) unsigned long vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) vmaddr = (unsigned long)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) radix_tree_lookup(&gmap->guest_to_host, gaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) /* Note: guest_to_host is empty for a shadow gmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) return vmaddr ? (vmaddr | (gaddr & ~PMD_MASK)) : -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) EXPORT_SYMBOL_GPL(__gmap_translate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * gmap_translate - translate a guest address to a user space address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) * @gaddr: guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) * Returns user space address which corresponds to the guest address or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) * -EFAULT if no such mapping exists.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) * This function does not establish potentially missing page table entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) unsigned long gmap_translate(struct gmap *gmap, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) unsigned long rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) mmap_read_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) rc = __gmap_translate(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) mmap_read_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) EXPORT_SYMBOL_GPL(gmap_translate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * gmap_unlink - disconnect a page table from the gmap shadow tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * @table: pointer to the host page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * @vmaddr: vm address associated with the host page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) void gmap_unlink(struct mm_struct *mm, unsigned long *table,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) int flush;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) flush = __gmap_unlink_by_vmaddr(gmap, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) if (flush)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) gmap_flush_tlb(gmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *old, pmd_t new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) unsigned long gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * gmap_link - set up shadow page tables to connect a host to a guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) * @gaddr: guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) * @vmaddr: vm address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) * if the vm address is already mapped to a different guest segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) * The mmap_lock of the mm that belongs to the address space must be held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) * when this function gets called.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) int __gmap_link(struct gmap *gmap, unsigned long gaddr, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) struct mm_struct *mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) u64 unprot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Create higher level tables in the gmap page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) table = gmap->table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) if ((*table & _REGION_ENTRY_INVALID) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) gmap_alloc_table(gmap, table, _REGION2_ENTRY_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) gaddr & _REGION1_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) if ((*table & _REGION_ENTRY_INVALID) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) gmap_alloc_table(gmap, table, _REGION3_ENTRY_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) gaddr & _REGION2_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) if ((gmap->asce & _ASCE_TYPE_MASK) >= _ASCE_TYPE_REGION3) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) if ((*table & _REGION_ENTRY_INVALID) &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) gmap_alloc_table(gmap, table, _SEGMENT_ENTRY_EMPTY,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) gaddr & _REGION3_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) /* Walk the parent mm page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) mm = gmap->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) pgd = pgd_offset(mm, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) VM_BUG_ON(pgd_none(*pgd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) p4d = p4d_offset(pgd, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) VM_BUG_ON(p4d_none(*p4d));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) pud = pud_offset(p4d, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) VM_BUG_ON(pud_none(*pud));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) /* large puds cannot yet be handled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) if (pud_large(*pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) pmd = pmd_offset(pud, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) VM_BUG_ON(pmd_none(*pmd));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) /* Are we allowed to use huge pages? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) if (pmd_large(*pmd) && !gmap->mm->context.allow_gmap_hpage_1m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /* Link gmap segment table entry location to page table. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) rc = radix_tree_preload(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) ptl = pmd_lock(mm, pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (*table == _SEGMENT_ENTRY_EMPTY) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) rc = radix_tree_insert(&gmap->host_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) vmaddr >> PMD_SHIFT, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) if (pmd_large(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) *table = (pmd_val(*pmd) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) _SEGMENT_ENTRY_HARDWARE_BITS_LARGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) | _SEGMENT_ENTRY_GMAP_UC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) *table = pmd_val(*pmd) &
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) _SEGMENT_ENTRY_HARDWARE_BITS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) } else if (*table & _SEGMENT_ENTRY_PROTECT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) !(pmd_val(*pmd) & _SEGMENT_ENTRY_PROTECT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) unprot = (u64)*table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) unprot &= ~_SEGMENT_ENTRY_PROTECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) unprot |= _SEGMENT_ENTRY_GMAP_UC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) gmap_pmdp_xchg(gmap, (pmd_t *)table, __pmd(unprot), gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) radix_tree_preload_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) * gmap_fault - resolve a fault on a guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) * @gaddr: guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) * @fault_flags: flags to pass down to handle_mm_fault()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) * Returns 0 on success, -ENOMEM for out of memory conditions, and -EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) * if the vm address is already mapped to a different guest segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) int gmap_fault(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) unsigned int fault_flags)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) unsigned long vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) bool unlocked;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) mmap_read_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) retry:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) unlocked = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) vmaddr = __gmap_translate(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) if (IS_ERR_VALUE(vmaddr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) rc = vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (fixup_user_fault(gmap->mm, vmaddr, fault_flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) &unlocked)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) rc = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) * In the case that fixup_user_fault unlocked the mmap_lock during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * faultin redo __gmap_translate to not race with a map/unmap_segment.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) if (unlocked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) goto retry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) rc = __gmap_link(gmap, gaddr, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) out_up:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) mmap_read_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) EXPORT_SYMBOL_GPL(gmap_fault);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) * this function is assumed to be called with mmap_lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) void __gmap_zap(struct gmap *gmap, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) unsigned long vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* Find the vm address for the guest address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) vmaddr = (unsigned long) radix_tree_lookup(&gmap->guest_to_host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) gaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) if (vmaddr) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) vmaddr |= gaddr & ~PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /* Get pointer to the page table entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) ptep = get_locked_pte(gmap->mm, vmaddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (likely(ptep)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) ptep_zap_unused(gmap->mm, vmaddr, ptep, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) pte_unmap_unlock(ptep, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) EXPORT_SYMBOL_GPL(__gmap_zap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) void gmap_discard(struct gmap *gmap, unsigned long from, unsigned long to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) unsigned long gaddr, vmaddr, size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) mmap_read_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) for (gaddr = from; gaddr < to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) gaddr = (gaddr + PMD_SIZE) & PMD_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) /* Find the vm address for the guest address */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) vmaddr = (unsigned long)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) radix_tree_lookup(&gmap->guest_to_host,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) gaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (!vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) vmaddr |= gaddr & ~PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) /* Find vma in the parent mm */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) vma = find_vma(gmap->mm, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (!vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * We do not discard pages that are backed by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * hugetlbfs, so we don't have to refault them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (is_vm_hugetlb_page(vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) size = min(to - gaddr, PMD_SIZE - (gaddr & ~PMD_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) zap_page_range(vma, vmaddr, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) mmap_read_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) EXPORT_SYMBOL_GPL(gmap_discard);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) static LIST_HEAD(gmap_notifier_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) static DEFINE_SPINLOCK(gmap_notifier_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) * gmap_register_pte_notifier - register a pte invalidation callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) * @nb: pointer to the gmap notifier block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) void gmap_register_pte_notifier(struct gmap_notifier *nb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) spin_lock(&gmap_notifier_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) list_add_rcu(&nb->list, &gmap_notifier_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) spin_unlock(&gmap_notifier_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) EXPORT_SYMBOL_GPL(gmap_register_pte_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * gmap_unregister_pte_notifier - remove a pte invalidation callback
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * @nb: pointer to the gmap notifier block
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) void gmap_unregister_pte_notifier(struct gmap_notifier *nb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) spin_lock(&gmap_notifier_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) list_del_rcu(&nb->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) spin_unlock(&gmap_notifier_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) synchronize_rcu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) EXPORT_SYMBOL_GPL(gmap_unregister_pte_notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * gmap_call_notifier - call all registered invalidation callbacks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) * @start: start virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) * @end: end virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) static void gmap_call_notifier(struct gmap *gmap, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) struct gmap_notifier *nb;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) list_for_each_entry(nb, &gmap_notifier_list, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) nb->notifier_call(gmap, start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) * gmap_table_walk - walk the gmap page tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) * @level: page table level to stop at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) * Returns a table entry pointer for the given guest address and @level
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) * @level=0 : returns a pointer to a page table table entry (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) * @level=1 : returns a pointer to a segment table entry (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) * @level=2 : returns a pointer to a region-3 table entry (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) * @level=3 : returns a pointer to a region-2 table entry (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * @level=4 : returns a pointer to a region-1 table entry (or NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * Returns NULL if the gmap page tables could not be walked to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) * requested level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) * Note: Can also be called for shadow gmaps.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) static inline unsigned long *gmap_table_walk(struct gmap *gmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) unsigned long gaddr, int level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) const int asce_type = gmap->asce & _ASCE_TYPE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) unsigned long *table = gmap->table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) if (gmap_is_shadow(gmap) && gmap->removed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) if (WARN_ON_ONCE(level > (asce_type >> 2) + 1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) if (asce_type != _ASCE_TYPE_REGION1 &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) gaddr & (-1UL << (31 + (asce_type >> 2) * 11)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) switch (asce_type) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) case _ASCE_TYPE_REGION1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) table += (gaddr & _REGION1_INDEX) >> _REGION1_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) if (level == 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) if (*table & _REGION_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) case _ASCE_TYPE_REGION2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) table += (gaddr & _REGION2_INDEX) >> _REGION2_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) if (level == 3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (*table & _REGION_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) case _ASCE_TYPE_REGION3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) table += (gaddr & _REGION3_INDEX) >> _REGION3_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (level == 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) if (*table & _REGION_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) table = (unsigned long *)(*table & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) fallthrough;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) case _ASCE_TYPE_SEGMENT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830) table += (gaddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) if (level == 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) if (*table & _REGION_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) table = (unsigned long *)(*table & _SEGMENT_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) table += (gaddr & _PAGE_INDEX) >> _PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) return table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) * gmap_pte_op_walk - walk the gmap page table, get the page table lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) * and return the pte pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) * @ptl: pointer to the spinlock pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) * Returns a pointer to the locked pte for a guest address, or NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) static pte_t *gmap_pte_op_walk(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) spinlock_t **ptl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) /* Walk the gmap page table, lock and get pte pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) table = gmap_table_walk(gmap, gaddr, 1); /* get segment pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (!table || *table & _SEGMENT_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) return pte_alloc_map_lock(gmap->mm, (pmd_t *) table, gaddr, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) * gmap_pte_op_fixup - force a page in and connect the gmap page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) * @vmaddr: address in the host process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) * Returns 0 if the caller can retry __gmap_translate (might fail again),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) * -ENOMEM if out of memory and -EFAULT if anything goes wrong while fixing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) * up or connecting the gmap page table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) static int gmap_pte_op_fixup(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) unsigned long vmaddr, int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) struct mm_struct *mm = gmap->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) unsigned int fault_flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) bool unlocked = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) fault_flags = (prot == PROT_WRITE) ? FAULT_FLAG_WRITE : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) if (fixup_user_fault(mm, vmaddr, fault_flags, &unlocked))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) if (unlocked)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) /* lost mmap_lock, caller has to retry __gmap_translate */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) /* Connect the page tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) return __gmap_link(gmap, gaddr, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * gmap_pte_op_end - release the page table lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * @ptl: pointer to the spinlock pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) static void gmap_pte_op_end(spinlock_t *ptl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (ptl)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) * gmap_pmd_op_walk - walk the gmap tables, get the guest table lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) * and return the pmd pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) * Returns a pointer to the pmd for a guest address, or NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) static inline pmd_t *gmap_pmd_op_walk(struct gmap *gmap, unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) pmdp = (pmd_t *) gmap_table_walk(gmap, gaddr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) if (!pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) /* without huge pages, there is no need to take the table lock */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) if (!gmap->mm->context.allow_gmap_hpage_1m)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) return pmd_none(*pmdp) ? NULL : pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) if (pmd_none(*pmdp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) /* 4k page table entries are locked via the pte (pte_alloc_map_lock). */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930) if (!pmd_large(*pmdp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) return pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) * gmap_pmd_op_end - release the guest_table_lock if needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) * @gmap: pointer to the guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) * @pmdp: pointer to the pmd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) static inline void gmap_pmd_op_end(struct gmap *gmap, pmd_t *pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) if (pmd_large(*pmdp))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) * gmap_protect_pmd - remove access rights to memory and set pmd notification bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) * @pmdp: pointer to the pmd to be protected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) * @bits: notification bits to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) * Returns:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) * 0 if successfully protected
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) * -EAGAIN if a fixup is needed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) * -EINVAL if unsupported notifier bits have been specified
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) * Expected to be called with sg->mm->mmap_lock in read and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * guest_table_lock held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) static int gmap_protect_pmd(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) pmd_t *pmdp, int prot, unsigned long bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) int pmd_i = pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) int pmd_p = pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) pmd_t new = *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) /* Fixup needed */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) if ((pmd_i && (prot != PROT_NONE)) || (pmd_p && (prot == PROT_WRITE)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) if (prot == PROT_NONE && !pmd_i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) pmd_val(new) |= _SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) if (prot == PROT_READ && !pmd_p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) pmd_val(new) &= ~_SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) pmd_val(new) |= _SEGMENT_ENTRY_PROTECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) gmap_pmdp_xchg(gmap, pmdp, new, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) if (bits & GMAP_NOTIFY_MPROT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) pmd_val(*pmdp) |= _SEGMENT_ENTRY_GMAP_IN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) /* Shadow GMAP protection needs split PMDs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) if (bits & GMAP_NOTIFY_SHADOW)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) * gmap_protect_pte - remove access rights to memory and set pgste bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) * @pmdp: pointer to the pmd associated with the pte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) * @bits: notification bits to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) * Returns 0 if successfully protected, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) * -EAGAIN if a fixup is needed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) * Expected to be called with sg->mm->mmap_lock in read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) static int gmap_protect_pte(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) pmd_t *pmdp, int prot, unsigned long bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) spinlock_t *ptl = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) unsigned long pbits = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) return -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) ptep = pte_alloc_map_lock(gmap->mm, pmdp, gaddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) if (!ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) pbits |= (bits & GMAP_NOTIFY_MPROT) ? PGSTE_IN_BIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) pbits |= (bits & GMAP_NOTIFY_SHADOW) ? PGSTE_VSIE_BIT : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) /* Protect and unlock. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) rc = ptep_force_prot(gmap->mm, gaddr, ptep, prot, pbits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) gmap_pte_op_end(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) * gmap_protect_range - remove access rights to memory and set pgste bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) * @len: size of area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) * @bits: pgste notification bits to set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) * Returns 0 if successfully protected, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) * -EFAULT if gaddr is invalid (or mapping for shadows is missing).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) static int gmap_protect_range(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) unsigned long len, int prot, unsigned long bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) unsigned long vmaddr, dist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) BUG_ON(gmap_is_shadow(gmap));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) rc = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) pmdp = gmap_pmd_op_walk(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) if (pmdp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) if (!pmd_large(*pmdp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) rc = gmap_protect_pte(gmap, gaddr, pmdp, prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) len -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) gaddr += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) rc = gmap_protect_pmd(gmap, gaddr, pmdp, prot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) bits);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) dist = HPAGE_SIZE - (gaddr & ~HPAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065) len = len < dist ? 0 : len - dist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) gaddr = (gaddr & HPAGE_MASK) + HPAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) gmap_pmd_op_end(gmap, pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) if (rc == -EINVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) /* -EAGAIN, fixup of userspace mm and gmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) vmaddr = __gmap_translate(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) if (IS_ERR_VALUE(vmaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) return vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) * gmap_mprotect_notify - change access rights for a range of ptes and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) * call the notifier if any pte changes again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) * @len: size of area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) * @prot: indicates access rights: PROT_NONE, PROT_READ or PROT_WRITE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) * Returns 0 if for each page in the given range a gmap mapping exists,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) * the new access rights could be set and the notifier could be armed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) * If the gmap mapping is missing for one or more pages -EFAULT is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) * returned. If no memory could be allocated -ENOMEM is returned.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) * This function establishes missing page table entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) int gmap_mprotect_notify(struct gmap *gmap, unsigned long gaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) unsigned long len, int prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) if ((gaddr & ~PAGE_MASK) || (len & ~PAGE_MASK) || gmap_is_shadow(gmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) if (!MACHINE_HAS_ESOP && prot == PROT_READ)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) mmap_read_lock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) rc = gmap_protect_range(gmap, gaddr, len, prot, GMAP_NOTIFY_MPROT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) mmap_read_unlock(gmap->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) EXPORT_SYMBOL_GPL(gmap_mprotect_notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) * gmap_read_table - get an unsigned long value from a guest page table using
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) * absolute addressing, without marking the page referenced.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) * @gmap: pointer to guest mapping meta data structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) * @val: pointer to the unsigned long value to return
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) * Returns 0 if the value was read, -ENOMEM if out of memory and -EFAULT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) * if reading using the virtual address failed. -EINVAL if called on a gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) * shadow.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) * Called with gmap->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) int gmap_read_table(struct gmap *gmap, unsigned long gaddr, unsigned long *val)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) unsigned long address, vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) pte_t *ptep, pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (gmap_is_shadow(gmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141) rc = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) ptep = gmap_pte_op_walk(gmap, gaddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) if (ptep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) pte = *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) if (pte_present(pte) && (pte_val(pte) & _PAGE_READ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) address = pte_val(pte) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) address += gaddr & ~PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) *val = *(unsigned long *) address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) pte_val(*ptep) |= _PAGE_YOUNG;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) /* Do *NOT* clear the _PAGE_INVALID bit! */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) gmap_pte_op_end(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155) if (!rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) vmaddr = __gmap_translate(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) if (IS_ERR_VALUE(vmaddr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) rc = vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) rc = gmap_pte_op_fixup(gmap, gaddr, vmaddr, PROT_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) EXPORT_SYMBOL_GPL(gmap_read_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) * gmap_insert_rmap - add a rmap to the host_to_rmap radix tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) * @vmaddr: vm address associated with the rmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) * @rmap: pointer to the rmap structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178) static inline void gmap_insert_rmap(struct gmap *sg, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) struct gmap_rmap *rmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) void __rcu **slot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) slot = radix_tree_lookup_slot(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) if (slot) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) rmap->next = radix_tree_deref_slot_protected(slot,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187) &sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) radix_tree_replace_slot(&sg->host_to_rmap, slot, rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190) rmap->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) radix_tree_insert(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) * gmap_protect_rmap - restrict access rights to memory (RO) and create an rmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) * @raddr: rmap address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) * @paddr: address in the parent guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) * @len: length of the memory area to protect
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) * Returns 0 if successfully protected and the rmap was created, -ENOMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) * if out of memory and -EFAULT if paddr is invalid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) static int gmap_protect_rmap(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) unsigned long paddr, unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) struct gmap *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) struct gmap_rmap *rmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) unsigned long vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217) parent = sg->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) while (len) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) vmaddr = __gmap_translate(parent, paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220) if (IS_ERR_VALUE(vmaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) return vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) rmap = kzalloc(sizeof(*rmap), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223) if (!rmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) rmap->raddr = raddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) rc = radix_tree_preload(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) kfree(rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) rc = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) ptep = gmap_pte_op_walk(parent, paddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) if (ptep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) rc = ptep_force_prot(parent->mm, paddr, ptep, PROT_READ,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) PGSTE_VSIE_BIT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) if (!rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) gmap_insert_rmap(sg, vmaddr, rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) gmap_pte_op_end(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) radix_tree_preload_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) kfree(rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245) rc = gmap_pte_op_fixup(parent, paddr, vmaddr, PROT_READ);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250) paddr += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) len -= PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) #define _SHADOW_RMAP_MASK 0x7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) #define _SHADOW_RMAP_REGION1 0x5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) #define _SHADOW_RMAP_REGION2 0x4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259) #define _SHADOW_RMAP_REGION3 0x3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) #define _SHADOW_RMAP_SEGMENT 0x2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) #define _SHADOW_RMAP_PGTABLE 0x1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) * gmap_idte_one - invalidate a single region or segment table entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * @asce: region or segment table *origin* + table-type bits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * @vaddr: virtual address to identify the table entry to flush
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) * The invalid bit of a single region or segment table entry is set
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269) * and the associated TLB entries depending on the entry are flushed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) * The table-type of the @asce identifies the portion of the @vaddr
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) * that is used as the invalidation index.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) static inline void gmap_idte_one(unsigned long asce, unsigned long vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) asm volatile(
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) " .insn rrf,0xb98e0000,%0,%1,0,0"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) : : "a" (asce), "a" (vaddr) : "cc", "memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) * gmap_unshadow_page - remove a page from a shadow page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287) static void gmap_unshadow_page(struct gmap *sg, unsigned long raddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) table = gmap_table_walk(sg, raddr, 0); /* get page table pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) if (!table || *table & _PAGE_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) gmap_call_notifier(sg, raddr, raddr + _PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) ptep_unshadow_pte(sg->mm, raddr, (pte_t *) table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300) * __gmap_unshadow_pgt - remove all entries from a shadow page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) * @pgt: pointer to the start of a shadow page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307) static void __gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) unsigned long *pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313) for (i = 0; i < _PAGE_ENTRIES; i++, raddr += _PAGE_SIZE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) pgt[i] = _PAGE_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) * gmap_unshadow_pgt - remove a shadow page table from a segment entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) * @raddr: address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) static void gmap_unshadow_pgt(struct gmap *sg, unsigned long raddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) unsigned long sto, *ste, *pgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330) ste = gmap_table_walk(sg, raddr, 1); /* get segment pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) if (!ste || !(*ste & _SEGMENT_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) gmap_call_notifier(sg, raddr, raddr + _SEGMENT_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) sto = (unsigned long) (ste - ((raddr & _SEGMENT_INDEX) >> _SEGMENT_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) gmap_idte_one(sto | _ASCE_TYPE_SEGMENT, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) pgt = (unsigned long *)(*ste & _SEGMENT_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) *ste = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) __gmap_unshadow_pgt(sg, raddr, pgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) /* Free page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) page = pfn_to_page(__pa(pgt) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) page_table_free_pgste(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) * __gmap_unshadow_sgt - remove all entries from a shadow segment table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) * @sgt: pointer to the start of a shadow segment table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) static void __gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) unsigned long *sgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) unsigned long *pgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) for (i = 0; i < _CRST_ENTRIES; i++, raddr += _SEGMENT_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) if (!(sgt[i] & _SEGMENT_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) pgt = (unsigned long *)(sgt[i] & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) sgt[i] = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) __gmap_unshadow_pgt(sg, raddr, pgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) /* Free page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) page = pfn_to_page(__pa(pgt) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) page_table_free_pgste(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) * gmap_unshadow_sgt - remove a shadow segment table from a region-3 entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) * Called with the shadow->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) static void gmap_unshadow_sgt(struct gmap *sg, unsigned long raddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) unsigned long r3o, *r3e, *sgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387) r3e = gmap_table_walk(sg, raddr, 2); /* get region-3 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) if (!r3e || !(*r3e & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) gmap_call_notifier(sg, raddr, raddr + _REGION3_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) r3o = (unsigned long) (r3e - ((raddr & _REGION3_INDEX) >> _REGION3_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) gmap_idte_one(r3o | _ASCE_TYPE_REGION3, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) sgt = (unsigned long *)(*r3e & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) *r3e = _REGION3_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) __gmap_unshadow_sgt(sg, raddr, sgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) /* Free segment table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397) page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) * __gmap_unshadow_r3t - remove all entries from a shadow region-3 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405) * @raddr: address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) * @r3t: pointer to the start of a shadow region-3 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410) static void __gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) unsigned long *r3t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) unsigned long *sgt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION3_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) if (!(r3t[i] & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421) sgt = (unsigned long *)(r3t[i] & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) r3t[i] = _REGION3_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) __gmap_unshadow_sgt(sg, raddr, sgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) /* Free segment table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) page = pfn_to_page(__pa(sgt) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) * gmap_unshadow_r3t - remove a shadow region-3 table from a region-2 entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) static void gmap_unshadow_r3t(struct gmap *sg, unsigned long raddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440) unsigned long r2o, *r2e, *r3t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) r2e = gmap_table_walk(sg, raddr, 3); /* get region-2 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445) if (!r2e || !(*r2e & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) gmap_call_notifier(sg, raddr, raddr + _REGION2_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) r2o = (unsigned long) (r2e - ((raddr & _REGION2_INDEX) >> _REGION2_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) gmap_idte_one(r2o | _ASCE_TYPE_REGION2, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) r3t = (unsigned long *)(*r2e & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) *r2e = _REGION2_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) __gmap_unshadow_r3t(sg, raddr, r3t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) /* Free region 3 table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) * __gmap_unshadow_r2t - remove all entries from a shadow region-2 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) * @r2t: pointer to the start of a shadow region-2 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) static void __gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) unsigned long *r2t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470) unsigned long *r3t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475) for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION2_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (!(r2t[i] & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) r3t = (unsigned long *)(r2t[i] & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) r2t[i] = _REGION2_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) __gmap_unshadow_r3t(sg, raddr, r3t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) /* Free region 3 table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482) page = pfn_to_page(__pa(r3t) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) * gmap_unshadow_r2t - remove a shadow region-2 table from a region-1 entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) * Called with the sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) static void gmap_unshadow_r2t(struct gmap *sg, unsigned long raddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) unsigned long r1o, *r1e, *r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) r1e = gmap_table_walk(sg, raddr, 4); /* get region-1 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502) if (!r1e || !(*r1e & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) gmap_call_notifier(sg, raddr, raddr + _REGION1_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) r1o = (unsigned long) (r1e - ((raddr & _REGION1_INDEX) >> _REGION1_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) gmap_idte_one(r1o | _ASCE_TYPE_REGION1, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) r2t = (unsigned long *)(*r1e & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) *r1e = _REGION1_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) __gmap_unshadow_r2t(sg, raddr, r2t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510) /* Free region 2 table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517) * __gmap_unshadow_r1t - remove all entries from a shadow region-1 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) * @raddr: rmap address in the shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) * @r1t: pointer to the start of a shadow region-1 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) * Called with the shadow->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) static void __gmap_unshadow_r1t(struct gmap *sg, unsigned long raddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) unsigned long *r1t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) unsigned long asce, *r2t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) asce = (unsigned long) r1t | _ASCE_TYPE_REGION1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) for (i = 0; i < _CRST_ENTRIES; i++, raddr += _REGION1_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) if (!(r1t[i] & _REGION_ENTRY_ORIGIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) r2t = (unsigned long *)(r1t[i] & _REGION_ENTRY_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) __gmap_unshadow_r2t(sg, raddr, r2t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) /* Clear entry and flush translation r1t -> r2t */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) gmap_idte_one(asce, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) r1t[i] = _REGION1_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541) /* Free region 2 table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) page = pfn_to_page(__pa(r2t) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543) list_del(&page->lru);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) * gmap_unshadow - remove a shadow page table completely
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) * Called with sg->guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1553) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1554) static void gmap_unshadow(struct gmap *sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1555) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1556) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1557)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1558) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1559) if (sg->removed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1560) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1561) sg->removed = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1562) gmap_call_notifier(sg, 0, -1UL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1563) gmap_flush_tlb(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1564) table = (unsigned long *)(sg->asce & _ASCE_ORIGIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1565) switch (sg->asce & _ASCE_TYPE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1566) case _ASCE_TYPE_REGION1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1567) __gmap_unshadow_r1t(sg, 0, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1568) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1569) case _ASCE_TYPE_REGION2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1570) __gmap_unshadow_r2t(sg, 0, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1571) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1572) case _ASCE_TYPE_REGION3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1573) __gmap_unshadow_r3t(sg, 0, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1574) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1575) case _ASCE_TYPE_SEGMENT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1576) __gmap_unshadow_sgt(sg, 0, table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1577) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1578) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1579) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1581) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1582) * gmap_find_shadow - find a specific asce in the list of shadow tables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1583) * @parent: pointer to the parent gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1584) * @asce: ASCE for which the shadow table is created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1585) * @edat_level: edat level to be used for the shadow translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1586) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1587) * Returns the pointer to a gmap if a shadow table with the given asce is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1588) * already available, ERR_PTR(-EAGAIN) if another one is just being created,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1589) * otherwise NULL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1591) static struct gmap *gmap_find_shadow(struct gmap *parent, unsigned long asce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1592) int edat_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1594) struct gmap *sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1596) list_for_each_entry(sg, &parent->children, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1597) if (sg->orig_asce != asce || sg->edat_level != edat_level ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1598) sg->removed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1599) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1600) if (!sg->initialized)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1601) return ERR_PTR(-EAGAIN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1602) refcount_inc(&sg->ref_count);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1603) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1604) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1605) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1606) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1607)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1608) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1609) * gmap_shadow_valid - check if a shadow guest address space matches the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1610) * given properties and is still valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1611) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1612) * @asce: ASCE for which the shadow table is requested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1613) * @edat_level: edat level to be used for the shadow translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1614) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1615) * Returns 1 if the gmap shadow is still valid and matches the given
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1616) * properties, the caller can continue using it. Returns 0 otherwise, the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1617) * caller has to request a new shadow gmap in this case.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1618) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1619) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1620) int gmap_shadow_valid(struct gmap *sg, unsigned long asce, int edat_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1621) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1622) if (sg->removed)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1623) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1624) return sg->orig_asce == asce && sg->edat_level == edat_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1626) EXPORT_SYMBOL_GPL(gmap_shadow_valid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1627)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1628) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1629) * gmap_shadow - create/find a shadow guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1630) * @parent: pointer to the parent gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1631) * @asce: ASCE for which the shadow table is created
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1632) * @edat_level: edat level to be used for the shadow translation
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1633) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1634) * The pages of the top level page table referred by the asce parameter
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1635) * will be set to read-only and marked in the PGSTEs of the kvm process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1636) * The shadow table will be removed automatically on any change to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1637) * PTE mapping for the source table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1638) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1639) * Returns a guest address space structure, ERR_PTR(-ENOMEM) if out of memory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1640) * ERR_PTR(-EAGAIN) if the caller has to retry and ERR_PTR(-EFAULT) if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1641) * parent gmap table could not be protected.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1642) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1643) struct gmap *gmap_shadow(struct gmap *parent, unsigned long asce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1644) int edat_level)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1645) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1646) struct gmap *sg, *new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1647) unsigned long limit;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1648) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1650) BUG_ON(parent->mm->context.allow_gmap_hpage_1m);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1651) BUG_ON(gmap_is_shadow(parent));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1652) spin_lock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1653) sg = gmap_find_shadow(parent, asce, edat_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1654) spin_unlock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1655) if (sg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1656) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1657) /* Create a new shadow gmap */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1658) limit = -1UL >> (33 - (((asce & _ASCE_TYPE_MASK) >> 2) * 11));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1659) if (asce & _ASCE_REAL_SPACE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1660) limit = -1UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1661) new = gmap_alloc(limit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1662) if (!new)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1663) return ERR_PTR(-ENOMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1664) new->mm = parent->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1665) new->parent = gmap_get(parent);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1666) new->orig_asce = asce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1667) new->edat_level = edat_level;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1668) new->initialized = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1669) spin_lock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1670) /* Recheck if another CPU created the same shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1671) sg = gmap_find_shadow(parent, asce, edat_level);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1672) if (sg) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1673) spin_unlock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1674) gmap_free(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1675) return sg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1676) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1677) if (asce & _ASCE_REAL_SPACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1678) /* only allow one real-space gmap shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1679) list_for_each_entry(sg, &parent->children, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1680) if (sg->orig_asce & _ASCE_REAL_SPACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1681) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1682) gmap_unshadow(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1683) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1684) list_del(&sg->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1685) gmap_put(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1686) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1687) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1688) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1690) refcount_set(&new->ref_count, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1691) list_add(&new->list, &parent->children);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1692) if (asce & _ASCE_REAL_SPACE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1693) /* nothing to protect, return right away */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1694) new->initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1695) spin_unlock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1696) return new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1697) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1698) spin_unlock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1699) /* protect after insertion, so it will get properly invalidated */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1700) mmap_read_lock(parent->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1701) rc = gmap_protect_range(parent, asce & _ASCE_ORIGIN,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1702) ((asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1703) PROT_READ, GMAP_NOTIFY_SHADOW);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1704) mmap_read_unlock(parent->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1705) spin_lock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1706) new->initialized = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1707) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1708) list_del(&new->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1709) gmap_free(new);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1710) new = ERR_PTR(rc);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1711) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1712) spin_unlock(&parent->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1713) return new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1715) EXPORT_SYMBOL_GPL(gmap_shadow);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1716)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1717) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1718) * gmap_shadow_r2t - create an empty shadow region 2 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1719) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1720) * @saddr: faulting address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1721) * @r2t: parent gmap address of the region 2 table to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1722) * @fake: r2t references contiguous guest memory block, not a r2t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1723) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1724) * The r2t parameter specifies the address of the source table. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1725) * four pages of the source table are made read-only in the parent gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1726) * address space. A write to the source table area @r2t will automatically
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1727) * remove the shadow r2 table and all of its decendents.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1728) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1729) * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1730) * shadow table structure is incomplete, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1731) * -EFAULT if an address in the parent gmap could not be resolved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1732) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1733) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1734) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1735) int gmap_shadow_r2t(struct gmap *sg, unsigned long saddr, unsigned long r2t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1736) int fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1737) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1738) unsigned long raddr, origin, offset, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1739) unsigned long *s_r2t, *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1740) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1741) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1742)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1743) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1744) /* Allocate a shadow region second table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1745) page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1746) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1747) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1748) page->index = r2t & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1749) if (fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1750) page->index |= GMAP_SHADOW_FAKE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1751) s_r2t = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1752) /* Install shadow region second table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1753) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1754) table = gmap_table_walk(sg, saddr, 4); /* get region-1 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1755) if (!table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1756) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1757) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1758) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1759) if (!(*table & _REGION_ENTRY_INVALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1760) rc = 0; /* Already established */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1761) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1762) } else if (*table & _REGION_ENTRY_ORIGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1763) rc = -EAGAIN; /* Race with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1764) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1766) crst_table_init(s_r2t, _REGION2_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1767) /* mark as invalid as long as the parent table is not protected */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1768) *table = (unsigned long) s_r2t | _REGION_ENTRY_LENGTH |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1769) _REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1770) if (sg->edat_level >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1771) *table |= (r2t & _REGION_ENTRY_PROTECT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1772) list_add(&page->lru, &sg->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1773) if (fake) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1774) /* nothing to protect for fake tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1775) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1776) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1777) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1778) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1779) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1780) /* Make r2t read-only in parent gmap page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1781) raddr = (saddr & _REGION1_MASK) | _SHADOW_RMAP_REGION1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1782) origin = r2t & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1783) offset = ((r2t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1784) len = ((r2t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1785) rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1786) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1787) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1788) table = gmap_table_walk(sg, saddr, 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1789) if (!table || (*table & _REGION_ENTRY_ORIGIN) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1790) (unsigned long) s_r2t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1791) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1792) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1793) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1794) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1795) gmap_unshadow_r2t(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1797) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1798) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1799) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1800) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1801) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1802) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1803) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1804) EXPORT_SYMBOL_GPL(gmap_shadow_r2t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1806) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1807) * gmap_shadow_r3t - create a shadow region 3 table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1808) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1809) * @saddr: faulting address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1810) * @r3t: parent gmap address of the region 3 table to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1811) * @fake: r3t references contiguous guest memory block, not a r3t
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1812) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1813) * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1814) * shadow table structure is incomplete, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1815) * -EFAULT if an address in the parent gmap could not be resolved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1816) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1817) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1818) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1819) int gmap_shadow_r3t(struct gmap *sg, unsigned long saddr, unsigned long r3t,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1820) int fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1821) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1822) unsigned long raddr, origin, offset, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1823) unsigned long *s_r3t, *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1824) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1825) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1826)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1827) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1828) /* Allocate a shadow region second table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1829) page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1830) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1831) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1832) page->index = r3t & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1833) if (fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1834) page->index |= GMAP_SHADOW_FAKE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1835) s_r3t = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1836) /* Install shadow region second table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1837) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1838) table = gmap_table_walk(sg, saddr, 3); /* get region-2 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1839) if (!table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1840) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1841) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1842) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1843) if (!(*table & _REGION_ENTRY_INVALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1844) rc = 0; /* Already established */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1845) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1846) } else if (*table & _REGION_ENTRY_ORIGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1847) rc = -EAGAIN; /* Race with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1848) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1849) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1850) crst_table_init(s_r3t, _REGION3_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1851) /* mark as invalid as long as the parent table is not protected */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1852) *table = (unsigned long) s_r3t | _REGION_ENTRY_LENGTH |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1853) _REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1854) if (sg->edat_level >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1855) *table |= (r3t & _REGION_ENTRY_PROTECT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1856) list_add(&page->lru, &sg->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1857) if (fake) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1858) /* nothing to protect for fake tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1859) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1860) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1861) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1862) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1863) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1864) /* Make r3t read-only in parent gmap page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1865) raddr = (saddr & _REGION2_MASK) | _SHADOW_RMAP_REGION2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1866) origin = r3t & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1867) offset = ((r3t & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1868) len = ((r3t & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1869) rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1870) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1871) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1872) table = gmap_table_walk(sg, saddr, 3);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1873) if (!table || (*table & _REGION_ENTRY_ORIGIN) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1874) (unsigned long) s_r3t)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1875) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1876) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1877) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1878) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1879) gmap_unshadow_r3t(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1880) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1881) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1882) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1883) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1884) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1885) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1886) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1887) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1888) EXPORT_SYMBOL_GPL(gmap_shadow_r3t);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1889)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1890) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1891) * gmap_shadow_sgt - create a shadow segment table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1892) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1893) * @saddr: faulting address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1894) * @sgt: parent gmap address of the segment table to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1895) * @fake: sgt references contiguous guest memory block, not a sgt
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1896) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1897) * Returns: 0 if successfully shadowed or already shadowed, -EAGAIN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1898) * shadow table structure is incomplete, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1899) * -EFAULT if an address in the parent gmap could not be resolved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1900) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1901) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1902) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1903) int gmap_shadow_sgt(struct gmap *sg, unsigned long saddr, unsigned long sgt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1904) int fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1905) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1906) unsigned long raddr, origin, offset, len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1907) unsigned long *s_sgt, *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1908) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1909) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1910)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1911) BUG_ON(!gmap_is_shadow(sg) || (sgt & _REGION3_ENTRY_LARGE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1912) /* Allocate a shadow segment table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1913) page = alloc_pages(GFP_KERNEL, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1914) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1915) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1916) page->index = sgt & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1917) if (fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1918) page->index |= GMAP_SHADOW_FAKE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1919) s_sgt = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1920) /* Install shadow region second table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1921) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1922) table = gmap_table_walk(sg, saddr, 2); /* get region-3 pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1923) if (!table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1924) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1925) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1926) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1927) if (!(*table & _REGION_ENTRY_INVALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1928) rc = 0; /* Already established */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1929) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1930) } else if (*table & _REGION_ENTRY_ORIGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1931) rc = -EAGAIN; /* Race with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1932) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1933) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1934) crst_table_init(s_sgt, _SEGMENT_ENTRY_EMPTY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1935) /* mark as invalid as long as the parent table is not protected */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1936) *table = (unsigned long) s_sgt | _REGION_ENTRY_LENGTH |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1937) _REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1938) if (sg->edat_level >= 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1939) *table |= sgt & _REGION_ENTRY_PROTECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1940) list_add(&page->lru, &sg->crst_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1941) if (fake) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1942) /* nothing to protect for fake tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1943) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1944) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1945) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1946) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1947) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1948) /* Make sgt read-only in parent gmap page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1949) raddr = (saddr & _REGION3_MASK) | _SHADOW_RMAP_REGION3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1950) origin = sgt & _REGION_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1951) offset = ((sgt & _REGION_ENTRY_OFFSET) >> 6) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1952) len = ((sgt & _REGION_ENTRY_LENGTH) + 1) * PAGE_SIZE - offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1953) rc = gmap_protect_rmap(sg, raddr, origin + offset, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1954) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1955) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1956) table = gmap_table_walk(sg, saddr, 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1957) if (!table || (*table & _REGION_ENTRY_ORIGIN) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1958) (unsigned long) s_sgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1959) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1960) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1961) *table &= ~_REGION_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1962) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1963) gmap_unshadow_sgt(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1964) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1965) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1966) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1967) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1968) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1969) __free_pages(page, CRST_ALLOC_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1970) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1971) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1972) EXPORT_SYMBOL_GPL(gmap_shadow_sgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1973)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1974) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1975) * gmap_shadow_lookup_pgtable - find a shadow page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1976) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1977) * @saddr: the address in the shadow aguest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1978) * @pgt: parent gmap address of the page table to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1979) * @dat_protection: if the pgtable is marked as protected by dat
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1980) * @fake: pgt references contiguous guest memory block, not a pgtable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1981) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1982) * Returns 0 if the shadow page table was found and -EAGAIN if the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1983) * table was not found.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1984) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1985) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1987) int gmap_shadow_pgt_lookup(struct gmap *sg, unsigned long saddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1988) unsigned long *pgt, int *dat_protection,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1989) int *fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1990) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1991) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1992) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1993) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1994)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1995) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1996) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1997) table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1998) if (table && !(*table & _SEGMENT_ENTRY_INVALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1999) /* Shadow page tables are full pages (pte+pgste) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2000) page = pfn_to_page(*table >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2001) *pgt = page->index & ~GMAP_SHADOW_FAKE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2002) *dat_protection = !!(*table & _SEGMENT_ENTRY_PROTECT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2003) *fake = !!(page->index & GMAP_SHADOW_FAKE_TABLE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2004) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2005) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2006) rc = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2007) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2008) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2009) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2011) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2012) EXPORT_SYMBOL_GPL(gmap_shadow_pgt_lookup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2013)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2014) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2015) * gmap_shadow_pgt - instantiate a shadow page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2016) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2017) * @saddr: faulting address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2018) * @pgt: parent gmap address of the page table to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2019) * @fake: pgt references contiguous guest memory block, not a pgtable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2020) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2021) * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2022) * shadow table structure is incomplete, -ENOMEM if out of memory,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2023) * -EFAULT if an address in the parent gmap could not be resolved and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2024) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2025) * Called with gmap->mm->mmap_lock in read
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2026) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2027) int gmap_shadow_pgt(struct gmap *sg, unsigned long saddr, unsigned long pgt,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2028) int fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2029) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2030) unsigned long raddr, origin;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2031) unsigned long *s_pgt, *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2032) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2033) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2035) BUG_ON(!gmap_is_shadow(sg) || (pgt & _SEGMENT_ENTRY_LARGE));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2036) /* Allocate a shadow page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2037) page = page_table_alloc_pgste(sg->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2038) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2039) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2040) page->index = pgt & _SEGMENT_ENTRY_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2041) if (fake)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2042) page->index |= GMAP_SHADOW_FAKE_TABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2043) s_pgt = (unsigned long *) page_to_phys(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2044) /* Install shadow page table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2045) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2046) table = gmap_table_walk(sg, saddr, 1); /* get segment pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2047) if (!table) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2048) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2049) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2050) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2051) if (!(*table & _SEGMENT_ENTRY_INVALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2052) rc = 0; /* Already established */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2053) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2054) } else if (*table & _SEGMENT_ENTRY_ORIGIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2055) rc = -EAGAIN; /* Race with shadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2056) goto out_free;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2057) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2058) /* mark as invalid as long as the parent table is not protected */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2059) *table = (unsigned long) s_pgt | _SEGMENT_ENTRY |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2060) (pgt & _SEGMENT_ENTRY_PROTECT) | _SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2061) list_add(&page->lru, &sg->pt_list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2062) if (fake) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2063) /* nothing to protect for fake tables */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2064) *table &= ~_SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2065) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2066) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2067) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2068) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2069) /* Make pgt read-only in parent gmap page table (not the pgste) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2070) raddr = (saddr & _SEGMENT_MASK) | _SHADOW_RMAP_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2071) origin = pgt & _SEGMENT_ENTRY_ORIGIN & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2072) rc = gmap_protect_rmap(sg, raddr, origin, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2073) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2074) if (!rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2075) table = gmap_table_walk(sg, saddr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2076) if (!table || (*table & _SEGMENT_ENTRY_ORIGIN) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2077) (unsigned long) s_pgt)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2078) rc = -EAGAIN; /* Race with unshadow */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2079) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2080) *table &= ~_SEGMENT_ENTRY_INVALID;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2081) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2082) gmap_unshadow_pgt(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2083) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2084) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2085) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2086) out_free:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2087) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2088) page_table_free_pgste(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2089) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2091) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2092) EXPORT_SYMBOL_GPL(gmap_shadow_pgt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2093)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2094) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2095) * gmap_shadow_page - create a shadow page mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2096) * @sg: pointer to the shadow guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2097) * @saddr: faulting address in the shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2098) * @pte: pte in parent gmap address space to get shadowed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2099) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2100) * Returns 0 if successfully shadowed or already shadowed, -EAGAIN if the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2101) * shadow table structure is incomplete, -ENOMEM if out of memory and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2102) * -EFAULT if an address in the parent gmap could not be resolved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2103) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2104) * Called with sg->mm->mmap_lock in read.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2105) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2106) int gmap_shadow_page(struct gmap *sg, unsigned long saddr, pte_t pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2108) struct gmap *parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2109) struct gmap_rmap *rmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2110) unsigned long vmaddr, paddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2111) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2112) pte_t *sptep, *tptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2113) int prot;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2114) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2115)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2116) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2117) parent = sg->parent;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2118) prot = (pte_val(pte) & _PAGE_PROTECT) ? PROT_READ : PROT_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2120) rmap = kzalloc(sizeof(*rmap), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2121) if (!rmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2122) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2123) rmap->raddr = (saddr & PAGE_MASK) | _SHADOW_RMAP_PGTABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2125) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2126) paddr = pte_val(pte) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2127) vmaddr = __gmap_translate(parent, paddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2128) if (IS_ERR_VALUE(vmaddr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2129) rc = vmaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2130) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2131) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2132) rc = radix_tree_preload(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2133) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2134) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2135) rc = -EAGAIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2136) sptep = gmap_pte_op_walk(parent, paddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2137) if (sptep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2138) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2139) /* Get page table pointer */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2140) tptep = (pte_t *) gmap_table_walk(sg, saddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2141) if (!tptep) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2142) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2143) gmap_pte_op_end(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2144) radix_tree_preload_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2145) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2147) rc = ptep_shadow_pte(sg->mm, saddr, sptep, tptep, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2148) if (rc > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2149) /* Success and a new mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2150) gmap_insert_rmap(sg, vmaddr, rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2151) rmap = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2152) rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2153) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2154) gmap_pte_op_end(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2155) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2157) radix_tree_preload_end();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2158) if (!rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2159) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2160) rc = gmap_pte_op_fixup(parent, paddr, vmaddr, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2161) if (rc)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2162) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2164) kfree(rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2165) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2166) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2167) EXPORT_SYMBOL_GPL(gmap_shadow_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2169) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2170) * gmap_shadow_notify - handle notifications for shadow gmap
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2171) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2172) * Called with sg->parent->shadow_lock.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2173) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2174) static void gmap_shadow_notify(struct gmap *sg, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2175) unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2176) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2177) struct gmap_rmap *rmap, *rnext, *head;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2178) unsigned long start, end, bits, raddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2179)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2180) BUG_ON(!gmap_is_shadow(sg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2182) spin_lock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2183) if (sg->removed) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2184) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2185) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2186) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2187) /* Check for top level table */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2188) start = sg->orig_asce & _ASCE_ORIGIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2189) end = start + ((sg->orig_asce & _ASCE_TABLE_LENGTH) + 1) * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2190) if (!(sg->orig_asce & _ASCE_REAL_SPACE) && gaddr >= start &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2191) gaddr < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2192) /* The complete shadow table has to go */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2193) gmap_unshadow(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2194) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2195) list_del(&sg->list);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2196) gmap_put(sg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2197) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2198) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2199) /* Remove the page table tree from on specific entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2200) head = radix_tree_delete(&sg->host_to_rmap, vmaddr >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2201) gmap_for_each_rmap_safe(rmap, rnext, head) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2202) bits = rmap->raddr & _SHADOW_RMAP_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2203) raddr = rmap->raddr ^ bits;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2204) switch (bits) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2205) case _SHADOW_RMAP_REGION1:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2206) gmap_unshadow_r2t(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2207) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2208) case _SHADOW_RMAP_REGION2:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2209) gmap_unshadow_r3t(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2210) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2211) case _SHADOW_RMAP_REGION3:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2212) gmap_unshadow_sgt(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2213) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2214) case _SHADOW_RMAP_SEGMENT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2215) gmap_unshadow_pgt(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2216) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2217) case _SHADOW_RMAP_PGTABLE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2218) gmap_unshadow_page(sg, raddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2219) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2221) kfree(rmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2223) spin_unlock(&sg->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2225)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2226) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2227) * ptep_notify - call all invalidation callbacks for a specific pte.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2228) * @mm: pointer to the process mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2229) * @addr: virtual address in the process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2230) * @pte: pointer to the page table entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2231) * @bits: bits from the pgste that caused the notify call
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2232) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2233) * This function is assumed to be called with the page table lock held
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2234) * for the pte to notify.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2235) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2236) void ptep_notify(struct mm_struct *mm, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2237) pte_t *pte, unsigned long bits)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2238) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2239) unsigned long offset, gaddr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2240) unsigned long *table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2241) struct gmap *gmap, *sg, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2243) offset = ((unsigned long) pte) & (255 * sizeof(pte_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2244) offset = offset * (PAGE_SIZE / sizeof(pte_t));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2245) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2246) list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2247) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2248) table = radix_tree_lookup(&gmap->host_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2249) vmaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2250) if (table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2251) gaddr = __gmap_segment_gaddr(table) + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2252) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2253) if (!table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2254) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2255)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2256) if (!list_empty(&gmap->children) && (bits & PGSTE_VSIE_BIT)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2257) spin_lock(&gmap->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2258) list_for_each_entry_safe(sg, next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2259) &gmap->children, list)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2260) gmap_shadow_notify(sg, vmaddr, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2261) spin_unlock(&gmap->shadow_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2263) if (bits & PGSTE_IN_BIT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2264) gmap_call_notifier(gmap, gaddr, gaddr + PAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2266) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2268) EXPORT_SYMBOL_GPL(ptep_notify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2270) static void pmdp_notify_gmap(struct gmap *gmap, pmd_t *pmdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2271) unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2272) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2273) pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_IN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2274) gmap_call_notifier(gmap, gaddr, gaddr + HPAGE_SIZE - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2275) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2276)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2277) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2278) * gmap_pmdp_xchg - exchange a gmap pmd with another
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2279) * @gmap: pointer to the guest address space structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2280) * @pmdp: pointer to the pmd entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2281) * @new: replacement entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2282) * @gaddr: the affected guest address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2283) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2284) * This function is assumed to be called with the guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2285) * held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2286) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2287) static void gmap_pmdp_xchg(struct gmap *gmap, pmd_t *pmdp, pmd_t new,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2288) unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2290) gaddr &= HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2291) pmdp_notify_gmap(gmap, pmdp, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2292) pmd_val(new) &= ~_SEGMENT_ENTRY_GMAP_IN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2293) if (MACHINE_HAS_TLB_GUEST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2294) __pmdp_idte(gaddr, (pmd_t *)pmdp, IDTE_GUEST_ASCE, gmap->asce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2295) IDTE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2296) else if (MACHINE_HAS_IDTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2297) __pmdp_idte(gaddr, (pmd_t *)pmdp, 0, 0, IDTE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2298) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2299) __pmdp_csp(pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2300) *pmdp = new;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2301) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2302)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2303) static void gmap_pmdp_clear(struct mm_struct *mm, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2304) int purge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2306) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2307) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2308) unsigned long gaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2309)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2310) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2311) list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2312) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2313) pmdp = (pmd_t *)radix_tree_delete(&gmap->host_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2314) vmaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2315) if (pmdp) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2316) gaddr = __gmap_segment_gaddr((unsigned long *)pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2317) pmdp_notify_gmap(gmap, pmdp, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2318) WARN_ON(pmd_val(*pmdp) & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2319) _SEGMENT_ENTRY_GMAP_UC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2320) if (purge)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2321) __pmdp_csp(pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2322) pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2323) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2324) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2326) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2327) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2328)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2329) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2330) * gmap_pmdp_invalidate - invalidate all affected guest pmd entries without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2331) * flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2332) * @mm: pointer to the process mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2333) * @vmaddr: virtual address in the process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2334) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2335) void gmap_pmdp_invalidate(struct mm_struct *mm, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2336) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2337) gmap_pmdp_clear(mm, vmaddr, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2338) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2339) EXPORT_SYMBOL_GPL(gmap_pmdp_invalidate);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2340)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2341) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2342) * gmap_pmdp_csp - csp all affected guest pmd entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2343) * @mm: pointer to the process mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2344) * @vmaddr: virtual address in the process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2345) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2346) void gmap_pmdp_csp(struct mm_struct *mm, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2347) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2348) gmap_pmdp_clear(mm, vmaddr, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2350) EXPORT_SYMBOL_GPL(gmap_pmdp_csp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2352) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2353) * gmap_pmdp_idte_local - invalidate and clear a guest pmd entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2354) * @mm: pointer to the process mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2355) * @vmaddr: virtual address in the process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2356) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2357) void gmap_pmdp_idte_local(struct mm_struct *mm, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2358) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2359) unsigned long *entry, gaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2360) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2361) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2362)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2363) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2364) list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2365) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2366) entry = radix_tree_delete(&gmap->host_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2367) vmaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2368) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2369) pmdp = (pmd_t *)entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2370) gaddr = __gmap_segment_gaddr(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2371) pmdp_notify_gmap(gmap, pmdp, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2372) WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2373) _SEGMENT_ENTRY_GMAP_UC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2374) if (MACHINE_HAS_TLB_GUEST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2375) __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2376) gmap->asce, IDTE_LOCAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2377) else if (MACHINE_HAS_IDTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2378) __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_LOCAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2379) *entry = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2381) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2382) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2383) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2384) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2385) EXPORT_SYMBOL_GPL(gmap_pmdp_idte_local);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2386)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2387) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2388) * gmap_pmdp_idte_global - invalidate and clear a guest pmd entry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2389) * @mm: pointer to the process mm_struct
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2390) * @vmaddr: virtual address in the process address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2391) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2392) void gmap_pmdp_idte_global(struct mm_struct *mm, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2393) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2394) unsigned long *entry, gaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2395) struct gmap *gmap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2396) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2398) rcu_read_lock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2399) list_for_each_entry_rcu(gmap, &mm->context.gmap_list, list) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2400) spin_lock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2401) entry = radix_tree_delete(&gmap->host_to_guest,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2402) vmaddr >> PMD_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2403) if (entry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2404) pmdp = (pmd_t *)entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2405) gaddr = __gmap_segment_gaddr(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2406) pmdp_notify_gmap(gmap, pmdp, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2407) WARN_ON(*entry & ~(_SEGMENT_ENTRY_HARDWARE_BITS_LARGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2408) _SEGMENT_ENTRY_GMAP_UC));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2409) if (MACHINE_HAS_TLB_GUEST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2410) __pmdp_idte(gaddr, pmdp, IDTE_GUEST_ASCE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2411) gmap->asce, IDTE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2412) else if (MACHINE_HAS_IDTE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2413) __pmdp_idte(gaddr, pmdp, 0, 0, IDTE_GLOBAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2414) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2415) __pmdp_csp(pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2416) *entry = _SEGMENT_ENTRY_EMPTY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2417) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2418) spin_unlock(&gmap->guest_table_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2419) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2420) rcu_read_unlock();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2421) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2422) EXPORT_SYMBOL_GPL(gmap_pmdp_idte_global);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2423)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2424) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2425) * gmap_test_and_clear_dirty_pmd - test and reset segment dirty status
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2426) * @gmap: pointer to guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2427) * @pmdp: pointer to the pmd to be tested
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2428) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2429) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2430) * This function is assumed to be called with the guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2431) * held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2432) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2433) static bool gmap_test_and_clear_dirty_pmd(struct gmap *gmap, pmd_t *pmdp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2434) unsigned long gaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2436) if (pmd_val(*pmdp) & _SEGMENT_ENTRY_INVALID)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2437) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2439) /* Already protected memory, which did not change is clean */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2440) if (pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2441) !(pmd_val(*pmdp) & _SEGMENT_ENTRY_GMAP_UC))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2442) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2444) /* Clear UC indication and reset protection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2445) pmd_val(*pmdp) &= ~_SEGMENT_ENTRY_GMAP_UC;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2446) gmap_protect_pmd(gmap, gaddr, pmdp, PROT_READ, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2447) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2449)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2450) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2451) * gmap_sync_dirty_log_pmd - set bitmap based on dirty status of segment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2452) * @gmap: pointer to guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2453) * @bitmap: dirty bitmap for this pmd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2454) * @gaddr: virtual address in the guest address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2455) * @vmaddr: virtual address in the host address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2456) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2457) * This function is assumed to be called with the guest_table_lock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2458) * held.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2459) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2460) void gmap_sync_dirty_log_pmd(struct gmap *gmap, unsigned long bitmap[4],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2461) unsigned long gaddr, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2462) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2463) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2464) pmd_t *pmdp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2465) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2466) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2467)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2468) pmdp = gmap_pmd_op_walk(gmap, gaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2469) if (!pmdp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2470) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2472) if (pmd_large(*pmdp)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2473) if (gmap_test_and_clear_dirty_pmd(gmap, pmdp, gaddr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2474) bitmap_fill(bitmap, _PAGE_ENTRIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2475) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2476) for (i = 0; i < _PAGE_ENTRIES; i++, vmaddr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2477) ptep = pte_alloc_map_lock(gmap->mm, pmdp, vmaddr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2478) if (!ptep)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2479) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2480) if (ptep_test_and_clear_uc(gmap->mm, vmaddr, ptep))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2481) set_bit(i, bitmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2482) spin_unlock(ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2484) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2485) gmap_pmd_op_end(gmap, pmdp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2486) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2487) EXPORT_SYMBOL_GPL(gmap_sync_dirty_log_pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2489) #ifdef CONFIG_TRANSPARENT_HUGEPAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2490) static int thp_split_walk_pmd_entry(pmd_t *pmd, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2491) unsigned long end, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2492) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2493) struct vm_area_struct *vma = walk->vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2495) split_huge_pmd(vma, pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2496) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2497) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2499) static const struct mm_walk_ops thp_split_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2500) .pmd_entry = thp_split_walk_pmd_entry,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2501) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2503) static inline void thp_split_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2504) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2505) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2506)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2507) for (vma = mm->mmap; vma != NULL; vma = vma->vm_next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2508) vma->vm_flags &= ~VM_HUGEPAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2509) vma->vm_flags |= VM_NOHUGEPAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2510) walk_page_vma(vma, &thp_split_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2511) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2512) mm->def_flags |= VM_NOHUGEPAGE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2513) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2514) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2515) static inline void thp_split_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2516) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2517) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2518) #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2520) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2521) * Remove all empty zero pages from the mapping for lazy refaulting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2522) * - This must be called after mm->context.has_pgste is set, to avoid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2523) * future creation of zero pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2524) * - This must be called after THP was enabled
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2525) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2526) static int __zap_zero_pages(pmd_t *pmd, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2527) unsigned long end, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2528) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2529) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2530)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2531) for (addr = start; addr != end; addr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2532) pte_t *ptep;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2533) spinlock_t *ptl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2534)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2535) ptep = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2536) if (is_zero_pfn(pte_pfn(*ptep)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2537) ptep_xchg_direct(walk->mm, addr, ptep, __pte(_PAGE_INVALID));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2538) pte_unmap_unlock(ptep, ptl);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2539) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2540) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2541) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2542)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2543) static const struct mm_walk_ops zap_zero_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2544) .pmd_entry = __zap_zero_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2545) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2546)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2547) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2548) * switch on pgstes for its userspace process (for kvm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2549) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2550) int s390_enable_sie(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2551) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2552) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2553)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2554) /* Do we have pgstes? if yes, we are done */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2555) if (mm_has_pgste(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2556) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2557) /* Fail if the page tables are 2K */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2558) if (!mm_alloc_pgste(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2559) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2560) mmap_write_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2561) mm->context.has_pgste = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2562) /* split thp mappings and disable thp for future mappings */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2563) thp_split_mm(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2564) walk_page_range(mm, 0, TASK_SIZE, &zap_zero_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2565) mmap_write_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2566) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2568) EXPORT_SYMBOL_GPL(s390_enable_sie);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2569)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2570) int gmap_mark_unmergeable(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2572) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2573) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2574) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2575)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2576) for (vma = mm->mmap; vma; vma = vma->vm_next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2577) ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2578) MADV_UNMERGEABLE, &vma->vm_flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2579) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2580) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2581) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2582) mm->def_flags &= ~VM_MERGEABLE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2583) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2585) EXPORT_SYMBOL_GPL(gmap_mark_unmergeable);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2587) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2588) * Enable storage key handling from now on and initialize the storage
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2589) * keys with the default key.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2590) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2591) static int __s390_enable_skey_pte(pte_t *pte, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2592) unsigned long next, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2593) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2594) /* Clear storage key */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2595) ptep_zap_key(walk->mm, addr, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2596) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2597) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2598)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2599) static int __s390_enable_skey_hugetlb(pte_t *pte, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2600) unsigned long hmask, unsigned long next,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2601) struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2602) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2603) pmd_t *pmd = (pmd_t *)pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2604) unsigned long start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2605) struct page *page = pmd_page(*pmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2606)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2607) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2608) * The write check makes sure we do not set a key on shared
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2609) * memory. This is needed as the walker does not differentiate
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2610) * between actual guest memory and the process executable or
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2611) * shared libraries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2612) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2613) if (pmd_val(*pmd) & _SEGMENT_ENTRY_INVALID ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2614) !(pmd_val(*pmd) & _SEGMENT_ENTRY_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2615) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2617) start = pmd_val(*pmd) & HPAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2618) end = start + HPAGE_SIZE - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2619) __storage_key_init_range(start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2620) set_bit(PG_arch_1, &page->flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2621) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2622) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2623)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2624) static const struct mm_walk_ops enable_skey_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2625) .hugetlb_entry = __s390_enable_skey_hugetlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2626) .pte_entry = __s390_enable_skey_pte,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2627) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2628)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2629) int s390_enable_skey(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2630) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2631) struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2632) int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2634) mmap_write_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2635) if (mm_uses_skeys(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2636) goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2637)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2638) mm->context.uses_skeys = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2639) rc = gmap_mark_unmergeable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2640) if (rc) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2641) mm->context.uses_skeys = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2642) goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2643) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2644) walk_page_range(mm, 0, TASK_SIZE, &enable_skey_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2646) out_up:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2647) mmap_write_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2648) return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2649) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2650) EXPORT_SYMBOL_GPL(s390_enable_skey);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2652) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2653) * Reset CMMA state, make all pages stable again.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2654) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2655) static int __s390_reset_cmma(pte_t *pte, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2656) unsigned long next, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2658) ptep_zap_unused(walk->mm, addr, pte, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2659) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2660) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2661)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2662) static const struct mm_walk_ops reset_cmma_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2663) .pte_entry = __s390_reset_cmma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2664) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2666) void s390_reset_cmma(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2667) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2668) mmap_write_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2669) walk_page_range(mm, 0, TASK_SIZE, &reset_cmma_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2670) mmap_write_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2671) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2672) EXPORT_SYMBOL_GPL(s390_reset_cmma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2673)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2674) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2675) * make inaccessible pages accessible again
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2676) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2677) static int __s390_reset_acc(pte_t *ptep, unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2678) unsigned long next, struct mm_walk *walk)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2679) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2680) pte_t pte = READ_ONCE(*ptep);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2681)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2682) if (pte_present(pte))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2683) WARN_ON_ONCE(uv_destroy_page(pte_val(pte) & PAGE_MASK));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2684) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2685) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2687) static const struct mm_walk_ops reset_acc_walk_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2688) .pte_entry = __s390_reset_acc,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2689) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2690)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2691) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2692) void s390_reset_acc(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2694) if (!mm_is_protected(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2695) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2696) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2697) * we might be called during
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2698) * reset: we walk the pages and clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2699) * close of all kvm file descriptors: we walk the pages and clear
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2700) * exit of process on fd closure: vma already gone, do nothing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2701) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2702) if (!mmget_not_zero(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2703) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2704) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2705) walk_page_range(mm, 0, TASK_SIZE, &reset_acc_walk_ops, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2706) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2707) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2708) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2709) EXPORT_SYMBOL_GPL(s390_reset_acc);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags