^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) #include <linux/uaccess.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) #include <linux/ktime.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) #include <linux/debugfs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) #define GUP_FAST_BENCHMARK _IOWR('g', 1, struct gup_benchmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) #define GUP_BENCHMARK _IOWR('g', 2, struct gup_benchmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) #define PIN_FAST_BENCHMARK _IOWR('g', 3, struct gup_benchmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #define PIN_BENCHMARK _IOWR('g', 4, struct gup_benchmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #define PIN_LONGTERM_BENCHMARK _IOWR('g', 5, struct gup_benchmark)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) struct gup_benchmark {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) __u64 get_delta_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) __u64 put_delta_usec;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) __u64 addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) __u64 size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) __u32 nr_pages_per_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) __u32 flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) __u64 expansion[10]; /* For future use */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) static void put_back_pages(unsigned int cmd, struct page **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) unsigned long nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) case GUP_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) case GUP_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) for (i = 0; i < nr_pages; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) put_page(pages[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) case PIN_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) case PIN_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) case PIN_LONGTERM_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) unpin_user_pages(pages, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) static void verify_dma_pinned(unsigned int cmd, struct page **pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) unsigned long nr_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) case PIN_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) case PIN_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) case PIN_LONGTERM_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) for (i = 0; i < nr_pages; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) page = pages[i];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) if (WARN(!page_maybe_dma_pinned(page),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) "pages[%lu] is NOT dma-pinned\n", i)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) dump_page(page, "gup_benchmark failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) static int __gup_benchmark_ioctl(unsigned int cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) struct gup_benchmark *gup)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) ktime_t start_time, end_time;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) unsigned long i, nr_pages, addr, next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) int nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) struct page **pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) int ret = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) bool needs_mmap_lock =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) cmd != GUP_FAST_BENCHMARK && cmd != PIN_FAST_BENCHMARK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) if (gup->size > ULONG_MAX)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) nr_pages = gup->size / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) pages = kvcalloc(nr_pages, sizeof(void *), GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) if (!pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) if (needs_mmap_lock && mmap_read_lock_killable(current->mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) ret = -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) goto free_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) i = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) nr = gup->nr_pages_per_call;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) start_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) for (addr = gup->addr; addr < gup->addr + gup->size; addr = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) if (nr != gup->nr_pages_per_call)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) next = addr + nr * PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) if (next > gup->addr + gup->size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) next = gup->addr + gup->size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) nr = (next - addr) / PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) /* Filter out most gup flags: only allow a tiny subset here: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) gup->flags &= FOLL_WRITE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) case GUP_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) nr = get_user_pages_fast(addr, nr, gup->flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) pages + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) case GUP_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) nr = get_user_pages(addr, nr, gup->flags, pages + i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) case PIN_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) nr = pin_user_pages_fast(addr, nr, gup->flags,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) pages + i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) case PIN_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) nr = pin_user_pages(addr, nr, gup->flags, pages + i,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) case PIN_LONGTERM_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) nr = pin_user_pages(addr, nr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) gup->flags | FOLL_LONGTERM,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) pages + i, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ret = -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) goto unlock;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) if (nr <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) i += nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) end_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) /* Shifting the meaning of nr_pages: now it is actual number pinned: */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) nr_pages = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) gup->get_delta_usec = ktime_us_delta(end_time, start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) gup->size = addr - gup->addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * Take an un-benchmark-timed moment to verify DMA pinned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * state: print a warning if any non-dma-pinned pages are found:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) verify_dma_pinned(cmd, pages, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) start_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) put_back_pages(cmd, pages, nr_pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) end_time = ktime_get();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) gup->put_delta_usec = ktime_us_delta(end_time, start_time);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unlock:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) if (needs_mmap_lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) mmap_read_unlock(current->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) free_pages:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) kvfree(pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static long gup_benchmark_ioctl(struct file *filep, unsigned int cmd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) unsigned long arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) struct gup_benchmark gup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) switch (cmd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) case GUP_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) case GUP_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) case PIN_FAST_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) case PIN_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) case PIN_LONGTERM_BENCHMARK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) if (copy_from_user(&gup, (void __user *)arg, sizeof(gup)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) ret = __gup_benchmark_ioctl(cmd, &gup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (ret)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) return ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) if (copy_to_user((void __user *)arg, &gup, sizeof(gup)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) static const struct file_operations gup_benchmark_fops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) .open = nonseekable_open,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) .unlocked_ioctl = gup_benchmark_ioctl,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static int gup_benchmark_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) debugfs_create_file_unsafe("gup_benchmark", 0600, NULL, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) &gup_benchmark_fops);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) late_initcall(gup_benchmark_init);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags