^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * This is a module to test the HMM (Heterogeneous Memory Management)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * mirror and zone device private memory migration APIs of the kernel.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Userspace programs can register with the driver to mirror their own address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * space and can use the device to read/write any valid virtual address.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #include <linux/fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/module.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/cdev.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/mutex.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/rwsem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/delay.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/hmm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/swapops.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/sched/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/platform_device.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include "test_hmm_uapi.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #define DMIRROR_NDEVICES 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #define DMIRROR_RANGE_FAULT_TIMEOUT 1000
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #define DEVMEM_CHUNKS_RESERVE 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) static const struct dev_pagemap_ops dmirror_devmem_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) static const struct mmu_interval_notifier_ops dmirror_min_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) static dev_t dmirror_dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) struct dmirror_device;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) struct dmirror_bounce {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) unsigned long size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) unsigned long cpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define DPT_XA_TAG_WRITE 3UL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * Data structure to track address ranges and register for mmu interval
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * notifier updates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) struct dmirror_interval {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) struct mmu_interval_notifier notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) struct dmirror *dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) * Data attached to the open device file.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) * Note that it might be shared after a fork().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct dmirror {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) struct dmirror_device *mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) struct xarray pt;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) struct mmu_interval_notifier notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct mutex mutex;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) * ZONE_DEVICE pages for migration and simulating device memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) struct dmirror_chunk {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) struct dev_pagemap pagemap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) struct dmirror_device *mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) * Per device data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) struct dmirror_device {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) struct cdev cdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) struct hmm_devmem *devmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) unsigned int devmem_capacity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned int devmem_count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) struct dmirror_chunk **devmem_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) struct mutex devmem_lock; /* protects the above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) unsigned long calloc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) unsigned long cfree;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) struct page *free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) spinlock_t lock; /* protects the above */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) static int dmirror_bounce_init(struct dmirror_bounce *bounce,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long addr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) unsigned long size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) bounce->addr = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) bounce->size = size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) bounce->cpages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) bounce->ptr = vmalloc(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) if (!bounce->ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) vfree(bounce->ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) static int dmirror_fops_open(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) struct cdev *cdev = inode->i_cdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) struct dmirror *dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /* Mirror this process address space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) if (dmirror == NULL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) mutex_init(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) xa_init(&dmirror->pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) kfree(dmirror);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) filp->private_data = dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) static int dmirror_fops_release(struct inode *inode, struct file *filp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) struct dmirror *dmirror = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) mmu_interval_notifier_remove(&dmirror->notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) xa_destroy(&dmirror->pt);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) kfree(dmirror);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) static struct dmirror_device *dmirror_page_to_device(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) return container_of(page->pgmap, struct dmirror_chunk,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) pagemap)->mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) unsigned long *pfns = range->hmm_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) for (pfn = (range->start >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pfn < (range->end >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) pfn++, pfns++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) * Since we asked for hmm_range_fault() to populate pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) * it shouldn't return an error entry on success.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) WARN_ON(*pfns & HMM_PFN_ERROR);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) WARN_ON(!(*pfns & HMM_PFN_VALID));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) page = hmm_pfn_to_page(*pfns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) WARN_ON(!page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) entry = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) if (*pfns & HMM_PFN_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) if (xa_is_err(entry))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) return xa_err(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * The XArray doesn't hold references to pages since it relies on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * the mmu notifier to clear page pointers when they become stale.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * Therefore, it is OK to just clear the entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) end >> PAGE_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) xa_erase(&dmirror->pt, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) const struct mmu_notifier_range *range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) unsigned long cur_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) * Ignore invalidation callbacks for device private pages since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) * the invalidation is handled as part of the migration process.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) if (range->event == MMU_NOTIFY_MIGRATE &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) range->migrate_pgmap_owner == dmirror->mdevice)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (mmu_notifier_range_blockable(range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) else if (!mutex_trylock(&dmirror->mutex))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) mmu_interval_set_seq(mni, cur_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) dmirror_do_update(dmirror, range->start, range->end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static const struct mmu_interval_notifier_ops dmirror_min_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) .invalidate = dmirror_interval_invalidate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) static int dmirror_range_fault(struct dmirror *dmirror,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) struct hmm_range *range)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) struct mm_struct *mm = dmirror->notifier.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) unsigned long timeout =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) while (true) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) if (time_after(jiffies, timeout)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) range->notifier_seq = mmu_interval_read_begin(range->notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) ret = hmm_range_fault(range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (ret == -EBUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (mmu_interval_read_retry(range->notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) range->notifier_seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) ret = dmirror_do_fault(dmirror, range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) unsigned long end, bool write)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) struct mm_struct *mm = dmirror->notifier.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) unsigned long pfns[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) struct hmm_range range = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) .notifier = &dmirror->notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) .hmm_pfns = pfns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) .pfn_flags_mask = 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) .default_flags =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) .dev_private_owner = dmirror->mdevice,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) /* Since the mm is for the mirrored process, get a reference first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (!mmget_not_zero(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) for (addr = start; addr < end; addr = range.end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) range.start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) ret = dmirror_range_fault(dmirror, &range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) unsigned long end, struct dmirror_bounce *bounce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) void *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) entry = xa_load(&dmirror->pt, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) page = xa_untag_pointer(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) tmp = kmap(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) memcpy(ptr, tmp, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) kunmap(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) ptr += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) bounce->cpages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) struct dmirror_bounce bounce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) unsigned long start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) unsigned long size = cmd->npages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) start = cmd->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) end = start + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (end < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) ret = dmirror_bounce_init(&bounce, start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) ret = dmirror_do_read(dmirror, start, end, &bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (ret != -ENOENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) ret = dmirror_fault(dmirror, start, end, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) cmd->faults++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) bounce.size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) cmd->cpages = bounce.cpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) dmirror_bounce_fini(&bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) unsigned long end, struct dmirror_bounce *bounce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) void *tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) entry = xa_load(&dmirror->pt, pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) page = xa_untag_pointer(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) return -ENOENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) tmp = kmap(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) memcpy(tmp, ptr, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) kunmap(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) ptr += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) bounce->cpages++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) struct dmirror_bounce bounce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) unsigned long start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) unsigned long size = cmd->npages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) start = cmd->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) end = start + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (end < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) ret = dmirror_bounce_init(&bounce, start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) bounce.size)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) goto fini;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) while (1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) ret = dmirror_do_write(dmirror, start, end, &bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) if (ret != -ENOENT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) ret = dmirror_fault(dmirror, start, end, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) cmd->faults++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) fini:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) cmd->cpages = bounce.cpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) dmirror_bounce_fini(&bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) static bool dmirror_allocate_chunk(struct dmirror_device *mdevice,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct page **ppage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) struct dmirror_chunk *devmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) struct resource *res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) unsigned long pfn_first;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) unsigned long pfn_last;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) if (!devmem)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) "hmm_dmirror");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) if (IS_ERR(res))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) goto err_devmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) devmem->pagemap.range.start = res->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) devmem->pagemap.range.end = res->end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) devmem->pagemap.nr_range = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) devmem->pagemap.ops = &dmirror_devmem_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) devmem->pagemap.owner = mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) mutex_lock(&mdevice->devmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if (mdevice->devmem_count == mdevice->devmem_capacity) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct dmirror_chunk **new_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) unsigned int new_capacity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) new_capacity = mdevice->devmem_capacity +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) DEVMEM_CHUNKS_RESERVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) new_chunks = krealloc(mdevice->devmem_chunks,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) sizeof(new_chunks[0]) * new_capacity,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (!new_chunks)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) goto err_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) mdevice->devmem_capacity = new_capacity;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) mdevice->devmem_chunks = new_chunks;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) ptr = memremap_pages(&devmem->pagemap, numa_node_id());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) if (IS_ERR(ptr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) goto err_release;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) devmem->mdevice = mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) mutex_unlock(&mdevice->devmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) DEVMEM_CHUNK_SIZE / (1024 * 1024),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) mdevice->devmem_count,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) pfn_first, pfn_last);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) spin_lock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) for (pfn = pfn_first; pfn < pfn_last; pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) struct page *page = pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) page->zone_device_data = mdevice->free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) mdevice->free_pages = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) if (ppage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) *ppage = mdevice->free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) mdevice->free_pages = (*ppage)->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) mdevice->calloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) spin_unlock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) err_release:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) mutex_unlock(&mdevice->devmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) err_devmem:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) kfree(devmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) struct page *dpage = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) struct page *rpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) * This is a fake device so we alloc real system memory to store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) * our device memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) rpage = alloc_page(GFP_HIGHUSER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (!rpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) spin_lock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (mdevice->free_pages) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) dpage = mdevice->free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) mdevice->free_pages = dpage->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) mdevice->calloc++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) spin_unlock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) spin_unlock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) if (!dmirror_allocate_chunk(mdevice, &dpage))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) goto error;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) dpage->zone_device_data = rpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) get_page(dpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) lock_page(dpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return dpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) error:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) __free_page(rpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) struct dmirror *dmirror)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) struct dmirror_device *mdevice = dmirror->mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) const unsigned long *src = args->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) unsigned long *dst = args->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) src++, dst++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) struct page *spage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) struct page *dpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) struct page *rpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) if (!(*src & MIGRATE_PFN_MIGRATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) * Note that spage might be NULL which is OK since it is an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) * unallocated pte_none() or read-only zero page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) spage = migrate_pfn_to_page(*src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) dpage = dmirror_devmem_alloc_page(mdevice);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) if (!dpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) rpage = dpage->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) if (spage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) copy_highpage(rpage, spage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) clear_highpage(rpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) * Normally, a device would use the page->zone_device_data to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) * point to the mirror but here we use it to hold the page for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) * the simulated device memory and that page holds the pointer
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) * to the mirror.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) rpage->zone_device_data = dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) *dst = migrate_pfn(page_to_pfn(dpage)) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) MIGRATE_PFN_LOCKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) if ((*src & MIGRATE_PFN_WRITE) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) (!spage && args->vma->vm_flags & VM_WRITE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) *dst |= MIGRATE_PFN_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) struct dmirror *dmirror)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) unsigned long start = args->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626) unsigned long end = args->end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) const unsigned long *src = args->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) const unsigned long *dst = args->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) /* Map the migrated pages into the device's page tables. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) src++, dst++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) struct page *dpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) void *entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) if (!(*src & MIGRATE_PFN_MIGRATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) dpage = migrate_pfn_to_page(*dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643) if (!dpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) * Store the page that holds the data so the page table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) * doesn't have to deal with ZONE_DEVICE private pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) entry = dpage->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) if (*dst & MIGRATE_PFN_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) if (xa_is_err(entry)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) return xa_err(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) static int dmirror_migrate(struct dmirror *dmirror,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) struct hmm_dmirror_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) unsigned long start, end, addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) unsigned long size = cmd->npages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) struct mm_struct *mm = dmirror->notifier.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) unsigned long src_pfns[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) unsigned long dst_pfns[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) struct dmirror_bounce bounce;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) struct migrate_vma args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) start = cmd->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) end = start + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) if (end < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) /* Since the mm is for the mirrored process, get a reference first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) if (!mmget_not_zero(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) for (addr = start; addr < end; addr = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) vma = find_vma(mm, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (!vma || addr < vma->vm_start ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) !(vma->vm_flags & VM_READ)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) if (next > vma->vm_end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) next = vma->vm_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) args.vma = vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) args.src = src_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) args.dst = dst_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) args.start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) args.end = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) args.pgmap_owner = dmirror->mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) args.flags = MIGRATE_VMA_SELECT_SYSTEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706) ret = migrate_vma_setup(&args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) dmirror_migrate_alloc_and_copy(&args, dmirror);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) migrate_vma_pages(&args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) dmirror_migrate_finalize_and_map(&args, dmirror);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) migrate_vma_finalize(&args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) /* Return the migrated data for verification. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) ret = dmirror_bounce_init(&bounce, start, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) ret = dmirror_do_read(dmirror, start, end, &bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) if (ret == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) bounce.size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) cmd->cpages = bounce.cpages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) dmirror_bounce_fini(&bounce);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) unsigned char *perm, unsigned long entry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) if (entry & HMM_PFN_ERROR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) *perm = HMM_DMIRROR_PROT_ERROR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) if (!(entry & HMM_PFN_VALID)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) *perm = HMM_DMIRROR_PROT_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) page = hmm_pfn_to_page(entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) if (is_device_private_page(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /* Is the page migrated to this device or some other? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) if (dmirror->mdevice == dmirror_page_to_device(page))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) } else if (is_zero_pfn(page_to_pfn(page)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) *perm = HMM_DMIRROR_PROT_ZERO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) *perm = HMM_DMIRROR_PROT_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) if (entry & HMM_PFN_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) *perm |= HMM_DMIRROR_PROT_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) *perm |= HMM_DMIRROR_PROT_READ;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) *perm |= HMM_DMIRROR_PROT_PMD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) *perm |= HMM_DMIRROR_PROT_PUD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) const struct mmu_notifier_range *range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) unsigned long cur_seq)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) struct dmirror_interval *dmi =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) container_of(mni, struct dmirror_interval, notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) struct dmirror *dmirror = dmi->dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) if (mmu_notifier_range_blockable(range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) else if (!mutex_trylock(&dmirror->mutex))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) * Snapshots only need to set the sequence number since any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * invalidation in the interval invalidates the whole snapshot.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) mmu_interval_set_seq(mni, cur_seq);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795) return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798) static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) .invalidate = dmirror_snapshot_invalidate,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) static int dmirror_range_snapshot(struct dmirror *dmirror,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) struct hmm_range *range,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) unsigned char *perm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) struct mm_struct *mm = dmirror->notifier.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) struct dmirror_interval notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) unsigned long timeout =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) unsigned long n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) notifier.dmirror = dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) range->notifier = ¬ifier.notifier;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) ret = mmu_interval_notifier_insert(range->notifier, mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) range->start, range->end - range->start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) &dmirror_mrn_ops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) while (true) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) if (time_after(jiffies, timeout)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) ret = -EBUSY;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) range->notifier_seq = mmu_interval_read_begin(range->notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) mmap_read_lock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) ret = hmm_range_fault(range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) mmap_read_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) if (ret == -EBUSY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) mutex_lock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841) if (mmu_interval_read_retry(range->notifier,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) range->notifier_seq)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) n = (range->end - range->start) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) for (i = 0; i < n; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) mutex_unlock(&dmirror->mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) mmu_interval_notifier_remove(range->notifier);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) static int dmirror_snapshot(struct dmirror *dmirror,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) struct hmm_dmirror_cmd *cmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862) struct mm_struct *mm = dmirror->notifier.mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) unsigned long start, end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) unsigned long size = cmd->npages << PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) unsigned long next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867) unsigned long pfns[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) unsigned char perm[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) char __user *uptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) struct hmm_range range = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) .hmm_pfns = pfns,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) .dev_private_owner = dmirror->mdevice,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876) start = cmd->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) end = start + size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) if (end < start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) /* Since the mm is for the mirrored process, get a reference first. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) if (!mmget_not_zero(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * Register a temporary notifier to detect invalidations even if it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * overlaps with other mmu_interval_notifiers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) uptr = u64_to_user_ptr(cmd->ptr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) for (addr = start; addr < end; addr = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) unsigned long n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) range.start = addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) range.end = next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) ret = dmirror_range_snapshot(dmirror, &range, perm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) n = (range.end - range.start) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) if (copy_to_user(uptr, perm, n)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) ret = -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) cmd->cpages += n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) uptr += n;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) mmput(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) static long dmirror_fops_unlocked_ioctl(struct file *filp,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) unsigned int command,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) void __user *uarg = (void __user *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) struct hmm_dmirror_cmd cmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921) struct dmirror *dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) dmirror = filp->private_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925) if (!dmirror)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (copy_from_user(&cmd, uarg, sizeof(cmd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) if (cmd.addr & ~PAGE_MASK)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) cmd.cpages = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) cmd.faults = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) switch (command) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) case HMM_DMIRROR_READ:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) ret = dmirror_read(dmirror, &cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) case HMM_DMIRROR_WRITE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) ret = dmirror_write(dmirror, &cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) case HMM_DMIRROR_MIGRATE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) ret = dmirror_migrate(dmirror, &cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952) case HMM_DMIRROR_SNAPSHOT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) ret = dmirror_snapshot(dmirror, &cmd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) if (copy_to_user(uarg, &cmd, sizeof(cmd)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976) page = alloc_page(GFP_KERNEL | __GFP_ZERO);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) if (!page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) ret = vm_insert_page(vma, addr, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) __free_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) put_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991) static const struct file_operations dmirror_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) .open = dmirror_fops_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) .release = dmirror_fops_release,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) .mmap = dmirror_fops_mmap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995) .unlocked_ioctl = dmirror_fops_unlocked_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) .llseek = default_llseek,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) .owner = THIS_MODULE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) static void dmirror_devmem_free(struct page *page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) struct page *rpage = page->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) struct dmirror_device *mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) if (rpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) __free_page(rpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) mdevice = dmirror_page_to_device(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) spin_lock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) mdevice->cfree++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) page->zone_device_data = mdevice->free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) mdevice->free_pages = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) spin_unlock(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) struct dmirror *dmirror)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) const unsigned long *src = args->src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) unsigned long *dst = args->dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022) unsigned long start = args->start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) unsigned long end = args->end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) for (addr = start; addr < end; addr += PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) src++, dst++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) struct page *dpage, *spage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) spage = migrate_pfn_to_page(*src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) spage = spage->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) if (!dpage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) lock_page(dpage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) copy_highpage(dpage, spage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) if (*src & MIGRATE_PFN_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) *dst |= MIGRATE_PFN_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051) struct migrate_vma args;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) unsigned long src_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) unsigned long dst_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054) struct page *rpage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) struct dmirror *dmirror;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) vm_fault_t ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059) * Normally, a device would use the page->zone_device_data to point to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) * the mirror but here we use it to hold the page for the simulated
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) * device memory and that page holds the pointer to the mirror.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) rpage = vmf->page->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) dmirror = rpage->zone_device_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) /* FIXME demonstrate how we can adjust migrate range */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) args.vma = vmf->vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068) args.start = vmf->address;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) args.end = args.start + PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070) args.src = &src_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) args.dst = &dst_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) args.pgmap_owner = dmirror->mdevice;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) if (migrate_vma_setup(&args))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) return VM_FAULT_SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078) ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) migrate_vma_pages(&args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) * No device finalize step is needed since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) * dmirror_devmem_fault_alloc_and_copy() will have already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) * invalidated the device page table.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) migrate_vma_finalize(&args);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) static const struct dev_pagemap_ops dmirror_devmem_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) .page_free = dmirror_devmem_free,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) .migrate_to_ram = dmirror_devmem_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) static int dmirror_device_init(struct dmirror_device *mdevice, int id)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) dev_t dev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) dev = MKDEV(MAJOR(dmirror_dev), id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) mutex_init(&mdevice->devmem_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103) spin_lock_init(&mdevice->lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) cdev_init(&mdevice->cdevice, &dmirror_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) mdevice->cdevice.owner = THIS_MODULE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107) ret = cdev_add(&mdevice->cdevice, dev, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) /* Build a list of free ZONE_DEVICE private struct pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) dmirror_allocate_chunk(mdevice, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117) static void dmirror_device_remove(struct dmirror_device *mdevice)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) unsigned int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121) if (mdevice->devmem_chunks) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) for (i = 0; i < mdevice->devmem_count; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) struct dmirror_chunk *devmem =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124) mdevice->devmem_chunks[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) memunmap_pages(&devmem->pagemap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) release_mem_region(devmem->pagemap.range.start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128) range_len(&devmem->pagemap.range));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) kfree(devmem);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131) kfree(mdevice->devmem_chunks);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) cdev_del(&mdevice->cdevice);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) static int __init hmm_dmirror_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) int id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) "HMM_DMIRROR");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) goto err_unreg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) for (id = 0; id < DMIRROR_NDEVICES; id++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) ret = dmirror_device_init(dmirror_devices + id, id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) goto err_chrdev;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) pr_info("HMM test module loaded. This is only for testing HMM.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) err_chrdev:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) while (--id >= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158) dmirror_device_remove(dmirror_devices + id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) err_unreg:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) static void __exit hmm_dmirror_exit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) int id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) for (id = 0; id < DMIRROR_NDEVICES; id++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) dmirror_device_remove(dmirror_devices + id);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) module_init(hmm_dmirror_init);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) module_exit(hmm_dmirror_exit);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175) MODULE_LICENSE("GPL");
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags