^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-only
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * Stress userfaultfd syscall.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * Copyright (C) 2015 Red Hat, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * This test allocates two virtual areas and bounces the physical
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * memory across the two virtual areas (from area_src to area_dst)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * using userfaultfd.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * There are three threads running per CPU:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) * 1) one per-CPU thread takes a per-page pthread_mutex in a random
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * page of the area_dst (while the physical page may still be in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * area_src), and increments a per-page counter in the same page,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * and checks its value against a verification region.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * 2) another per-CPU thread handles the userfaults generated by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * thread 1 above. userfaultfd blocking reads or poll() modes are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * exercised interleaved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * 3) one last per-CPU thread transfers the memory in the background
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * at maximum bandwidth (if not already transferred by thread
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) * 2). Each cpu thread takes cares of transferring a portion of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) * area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * When all threads of type 3 completed the transfer, one bounce is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * complete. area_src and area_dst are then swapped. All threads are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) * respawned and so the bounce is immediately restarted in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * opposite direction.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * per-CPU threads 1 by triggering userfaults inside
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * pthread_mutex_lock will also verify the atomicity of the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) * transfer (UFFDIO_COPY).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #define _GNU_SOURCE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <stdio.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <stdlib.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <sys/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <sys/stat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <fcntl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <time.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) #include <signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) #include <poll.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #include <string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) #include <sys/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #include <sys/syscall.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #include <sys/ioctl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #include <sys/wait.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) #include <pthread.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) #include <linux/userfaultfd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) #include <setjmp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) #include <stdbool.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) #include <assert.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) #include "../kselftest.h"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #ifdef __NR_userfaultfd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) static unsigned long nr_cpus, nr_pages, nr_pages_per_cpu, page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) #define BOUNCE_RANDOM (1<<0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) #define BOUNCE_RACINGFAULTS (1<<1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) #define BOUNCE_VERIFY (1<<2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) #define BOUNCE_POLL (1<<3)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) static int bounces;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) #define TEST_ANON 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) #define TEST_HUGETLB 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) #define TEST_SHMEM 3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) static int test_type;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) /* exercise the test_uffdio_*_eexist every ALARM_INTERVAL_SECS */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) #define ALARM_INTERVAL_SECS 10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) static volatile bool test_uffdio_copy_eexist = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) static volatile bool test_uffdio_zeropage_eexist = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) /* Whether to test uffd write-protection */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static bool test_uffdio_wp = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) /* Whether to test uffd minor faults */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) static bool test_uffdio_minor = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) static bool map_shared;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) static int shm_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) static int huge_fd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) static char *huge_fd_off0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) static unsigned long long *count_verify;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) static int uffd = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) static int uffd_flags, finished, *pipefd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) static char *area_src, *area_src_alias, *area_dst, *area_dst_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) static char *zeropage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) pthread_attr_t attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) /* Userfaultfd test statistics */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) struct uffd_stats {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) unsigned long missing_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) unsigned long wp_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) unsigned long minor_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) /* pthread_mutex_t starts at page offset 0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define area_mutex(___area, ___nr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) ((pthread_mutex_t *) ((___area) + (___nr)*page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) * count is placed in the page after pthread_mutex_t naturally aligned
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) * to avoid non alignment faults on non-x86 archs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) #define area_count(___area, ___nr) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) ((volatile unsigned long long *) ((unsigned long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) ((___area) + (___nr)*page_size + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) sizeof(pthread_mutex_t) + \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) sizeof(unsigned long long) - 1) & \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) ~(unsigned long)(sizeof(unsigned long long) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) - 1)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) const char *examples =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) "# Run anonymous memory test on 100MiB region with 99999 bounces:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) "./userfaultfd anon 100 99999\n\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) "# Run share memory test on 1GiB region with 99 bounces:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) "./userfaultfd shmem 1000 99\n\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) "# Run hugetlb memory test on 256MiB region with 50 bounces (using /dev/hugepages/hugefile):\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) "./userfaultfd hugetlb 256 50 /dev/hugepages/hugefile\n\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) "# Run the same hugetlb test but using shmem:\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) "./userfaultfd hugetlb_shared 256 50 /dev/hugepages/hugefile\n\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) "# 10MiB-~6GiB 999 bounces anonymous test, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) "continue forever unless an error triggers\n"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) "while ./userfaultfd anon $[RANDOM % 6000 + 10] 999; do true; done\n\n";
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) static void usage(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) fprintf(stderr, "\nUsage: ./userfaultfd <test type> <MiB> <bounces> "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) "[hugetlbfs_file]\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) fprintf(stderr, "Supported <test type>: anon, hugetlb, "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) "hugetlb_shared, shmem\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) fprintf(stderr, "Examples:\n\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) fprintf(stderr, "%s", examples);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) exit(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #define _err(fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) int ret = errno; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) fprintf(stderr, "ERROR: " fmt, ##__VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) fprintf(stderr, " (errno=%d, line=%d)\n", \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) ret, __LINE__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) #define err(fmt, ...) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) do { \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) _err(fmt, ##__VA_ARGS__); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) exit(1); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) static void uffd_stats_reset(struct uffd_stats *uffd_stats,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) unsigned long n_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) for (i = 0; i < n_cpus; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) uffd_stats[i].cpu = i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) uffd_stats[i].missing_faults = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) uffd_stats[i].wp_faults = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) uffd_stats[i].minor_faults = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) static void uffd_stats_report(struct uffd_stats *stats, int n_cpus)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) unsigned long long miss_total = 0, wp_total = 0, minor_total = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) for (i = 0; i < n_cpus; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) miss_total += stats[i].missing_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) wp_total += stats[i].wp_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) minor_total += stats[i].minor_faults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) printf("userfaults: ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) if (miss_total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) printf("%llu missing (", miss_total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) for (i = 0; i < n_cpus; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) printf("%lu+", stats[i].missing_faults);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) printf("\b) ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) if (wp_total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) printf("%llu wp (", wp_total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) for (i = 0; i < n_cpus; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) printf("%lu+", stats[i].wp_faults);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) printf("\b) ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) if (minor_total) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) printf("%llu minor (", minor_total);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) for (i = 0; i < n_cpus; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) printf("%lu+", stats[i].minor_faults);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) printf("\b)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) printf("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static void anon_release_pages(char *rel_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) if (madvise(rel_area, nr_pages * page_size, MADV_DONTNEED))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) err("madvise(MADV_DONTNEED) failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) static void anon_allocate_area(void **alloc_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) if (*alloc_area == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) err("posix_memalign() failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static void noop_alias_mapping(__u64 *start, size_t len, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static void hugetlb_release_pages(char *rel_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) if (fallocate(huge_fd, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) rel_area == huge_fd_off0 ? 0 : nr_pages * page_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) nr_pages * page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) err("fallocate() failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) static void hugetlb_allocate_area(void **alloc_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) void *area_alias = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) char **alloc_area_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) (map_shared ? MAP_SHARED : MAP_PRIVATE) |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) MAP_HUGETLB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) huge_fd, *alloc_area == area_src ? 0 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) nr_pages * page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) if (*alloc_area == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) err("mmap of hugetlbfs file failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) if (map_shared) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) MAP_SHARED | MAP_HUGETLB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) huge_fd, *alloc_area == area_src ? 0 :
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) nr_pages * page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) if (area_alias == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) err("mmap of hugetlb file alias failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) if (*alloc_area == area_src) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) huge_fd_off0 = *alloc_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) alloc_area_alias = &area_src_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) alloc_area_alias = &area_dst_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) if (area_alias)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) *alloc_area_alias = area_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) static void hugetlb_alias_mapping(__u64 *start, size_t len, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) if (!map_shared)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) * We can't zap just the pagetable with hugetlbfs because
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * MADV_DONTEED won't work. So exercise -EEXIST on a alias
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) * mapping where the pagetables are not established initially,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) * this way we'll exercise the -EEXEC at the fs level.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) *start = (unsigned long) area_dst_alias + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static void shmem_release_pages(char *rel_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) if (madvise(rel_area, nr_pages * page_size, MADV_REMOVE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) err("madvise(MADV_REMOVE) failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) static void shmem_allocate_area(void **alloc_area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) void *area_alias = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) bool is_src = alloc_area == (void **)&area_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) unsigned long offset = is_src ? 0 : nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) *alloc_area = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) MAP_SHARED, shm_fd, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) if (*alloc_area == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) err("mmap of memfd failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) area_alias = mmap(NULL, nr_pages * page_size, PROT_READ | PROT_WRITE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) MAP_SHARED, shm_fd, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) if (area_alias == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) err("mmap of memfd alias failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) if (is_src)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) area_src_alias = area_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) area_dst_alias = area_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) static void shmem_alias_mapping(__u64 *start, size_t len, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) *start = (unsigned long)area_dst_alias + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) struct uffd_test_ops {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) void (*allocate_area)(void **alloc_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) void (*release_pages)(char *rel_area);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) void (*alias_mapping)(__u64 *start, size_t len, unsigned long offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #define SHMEM_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) (1 << _UFFDIO_COPY) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) (1 << _UFFDIO_ZEROPAGE))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) #define ANON_EXPECTED_IOCTLS ((1 << _UFFDIO_WAKE) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) (1 << _UFFDIO_COPY) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) (1 << _UFFDIO_ZEROPAGE) | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) (1 << _UFFDIO_WRITEPROTECT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) static struct uffd_test_ops anon_uffd_test_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) .expected_ioctls = ANON_EXPECTED_IOCTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) .allocate_area = anon_allocate_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) .release_pages = anon_release_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) .alias_mapping = noop_alias_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) static struct uffd_test_ops shmem_uffd_test_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) .expected_ioctls = SHMEM_EXPECTED_IOCTLS,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) .allocate_area = shmem_allocate_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) .release_pages = shmem_release_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) .alias_mapping = shmem_alias_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) static struct uffd_test_ops hugetlb_uffd_test_ops = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) .expected_ioctls = UFFD_API_RANGE_IOCTLS_BASIC & ~(1 << _UFFDIO_CONTINUE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) .allocate_area = hugetlb_allocate_area,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) .release_pages = hugetlb_release_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) .alias_mapping = hugetlb_alias_mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) static struct uffd_test_ops *uffd_test_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) static void userfaultfd_open(uint64_t *features)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) struct uffdio_api uffdio_api;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) uffd = syscall(__NR_userfaultfd, O_CLOEXEC | O_NONBLOCK | UFFD_USER_MODE_ONLY);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) if (uffd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) err("userfaultfd syscall not available in this kernel");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) uffd_flags = fcntl(uffd, F_GETFD, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) uffdio_api.api = UFFD_API;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) uffdio_api.features = *features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) if (ioctl(uffd, UFFDIO_API, &uffdio_api))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) err("UFFDIO_API failed.\nPlease make sure to "
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) "run with either root or ptrace capability.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) if (uffdio_api.api != UFFD_API)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) err("UFFDIO_API error: %" PRIu64, (uint64_t)uffdio_api.api);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) *features = uffdio_api.features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) static inline void munmap_area(void **area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) if (*area)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) if (munmap(*area, nr_pages * page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) err("munmap");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) *area = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) static void uffd_test_ctx_clear(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) size_t i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) if (pipefd) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) for (i = 0; i < nr_cpus * 2; ++i) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (close(pipefd[i]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) err("close pipefd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) free(pipefd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) pipefd = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) if (count_verify) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) free(count_verify);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) count_verify = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) if (uffd != -1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) if (close(uffd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) err("close uffd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) uffd = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) huge_fd_off0 = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) munmap_area((void **)&area_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) munmap_area((void **)&area_src_alias);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) munmap_area((void **)&area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) munmap_area((void **)&area_dst_alias);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) static void uffd_test_ctx_init_ext(uint64_t *features)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) unsigned long nr, cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) uffd_test_ctx_clear();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) uffd_test_ops->allocate_area((void **)&area_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) uffd_test_ops->allocate_area((void **)&area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) userfaultfd_open(features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) count_verify = malloc(nr_pages * sizeof(unsigned long long));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) if (!count_verify)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) err("count_verify");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) for (nr = 0; nr < nr_pages; nr++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) *area_mutex(area_src, nr) =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) (pthread_mutex_t)PTHREAD_MUTEX_INITIALIZER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) count_verify[nr] = *area_count(area_src, nr) = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) * In the transition between 255 to 256, powerpc will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) * read out of order in my_bcmp and see both bytes as
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) * zero, so leave a placeholder below always non-zero
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) * after the count, to avoid my_bcmp to trigger false
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) * positives.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) *(area_count(area_src, nr) + 1) = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) * After initialization of area_src, we must explicitly release pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) * for area_dst to make sure it's fully empty. Otherwise we could have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440) * some area_dst pages be errornously initialized with zero pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) * hence we could hit memory corruption later in the test.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) * One example is when THP is globally enabled, above allocate_area()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) * calls could have the two areas merged into a single VMA (as they
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) * will have the same VMA flags so they're mergeable). When we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) * initialize the area_src above, it's possible that some part of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) * area_dst could have been faulted in via one huge THP that will be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) * shared between area_src and area_dst. It could cause some of the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) * area_dst won't be trapped by missing userfaults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451) * This release_pages() will guarantee even if that happened, we'll
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) * proactively split the thp and drop any accidentally initialized
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) * pages within area_dst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) uffd_test_ops->release_pages(area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) pipefd = malloc(sizeof(int) * nr_cpus * 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) if (!pipefd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) err("pipefd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) for (cpu = 0; cpu < nr_cpus; cpu++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) if (pipe2(&pipefd[cpu * 2], O_CLOEXEC | O_NONBLOCK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) err("pipe");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) static inline void uffd_test_ctx_init(uint64_t features)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) uffd_test_ctx_init_ext(&features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) static int my_bcmp(char *str1, char *str2, size_t n)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) unsigned long i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) for (i = 0; i < n; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) if (str1[i] != str2[i])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) static void wp_range(int ufd, __u64 start, __u64 len, bool wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) struct uffdio_writeprotect prms = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /* Write protection page faults */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) prms.range.start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) prms.range.len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) /* Undo write-protect, do wakeup after that */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) prms.mode = wp ? UFFDIO_WRITEPROTECT_MODE_WP : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) if (ioctl(ufd, UFFDIO_WRITEPROTECT, &prms))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) err("clear WP failed: address=0x%"PRIx64, (uint64_t)start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493) static void continue_range(int ufd, __u64 start, __u64 len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) struct uffdio_continue req;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) req.range.start = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499) req.range.len = len;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) req.mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) if (ioctl(ufd, UFFDIO_CONTINUE, &req))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) err("UFFDIO_CONTINUE failed for address 0x%" PRIx64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) (uint64_t)start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * Error handling within the kernel for continue is subtly different
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * from copy or zeropage, so it may be a source of bugs. Trigger an
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) * error (-EEXIST) on purpose, to verify doing so doesn't cause a BUG.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) req.mapped = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) ret = ioctl(ufd, UFFDIO_CONTINUE, &req);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) if (ret >= 0 || req.mapped != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) err("failed to exercise UFFDIO_CONTINUE error handling, ret=%d, mapped=%" PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) ret, (int64_t) req.mapped);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) static void *locking_thread(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) unsigned long cpu = (unsigned long) arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) struct random_data rand;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) unsigned long page_nr = *(&(page_nr)); /* uninitialized warning */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) int32_t rand_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) unsigned long long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) char randstate[64];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) unsigned int seed;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) if (bounces & BOUNCE_RANDOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) seed = (unsigned int) time(NULL) - bounces;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) if (!(bounces & BOUNCE_RACINGFAULTS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) seed += cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) bzero(&rand, sizeof(rand));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) bzero(&randstate, sizeof(randstate));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) if (initstate_r(seed, randstate, sizeof(randstate), &rand))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) err("initstate_r failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) page_nr = -bounces;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) if (!(bounces & BOUNCE_RACINGFAULTS))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) page_nr += cpu * nr_pages_per_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) while (!finished) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) if (bounces & BOUNCE_RANDOM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) if (random_r(&rand, &rand_nr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) err("random_r failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) page_nr = rand_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (sizeof(page_nr) > sizeof(rand_nr)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) if (random_r(&rand, &rand_nr))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) err("random_r failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) page_nr |= (((unsigned long) rand_nr) << 16) <<
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) 16;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) page_nr += 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) page_nr %= nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) pthread_mutex_lock(area_mutex(area_dst, page_nr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) count = *area_count(area_dst, page_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) if (count != count_verify[page_nr])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) err("page_nr %lu memory corruption %llu %llu",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) page_nr, count, count_verify[page_nr]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) *area_count(area_dst, page_nr) = count_verify[page_nr] = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) pthread_mutex_unlock(area_mutex(area_dst, page_nr));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) static void retry_copy_page(int ufd, struct uffdio_copy *uffdio_copy,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) uffd_test_ops->alias_mapping(&uffdio_copy->dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) uffdio_copy->len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) if (ioctl(ufd, UFFDIO_COPY, uffdio_copy)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) /* real retval in ufdio_copy.copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) if (uffdio_copy->copy != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) err("UFFDIO_COPY retry error: %"PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) (int64_t)uffdio_copy->copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) err("UFFDIO_COPY retry unexpected: %"PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) (int64_t)uffdio_copy->copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) static int __copy_page(int ufd, unsigned long offset, bool retry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) struct uffdio_copy uffdio_copy;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) if (offset >= nr_pages * page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) err("unexpected offset %lu\n", offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) uffdio_copy.dst = (unsigned long) area_dst + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) uffdio_copy.src = (unsigned long) area_src + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) uffdio_copy.len = page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) uffdio_copy.mode = UFFDIO_COPY_MODE_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) uffdio_copy.mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) uffdio_copy.copy = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) if (ioctl(ufd, UFFDIO_COPY, &uffdio_copy)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) /* real retval in ufdio_copy.copy */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602) if (uffdio_copy.copy != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) err("UFFDIO_COPY error: %"PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604) (int64_t)uffdio_copy.copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) } else if (uffdio_copy.copy != page_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) err("UFFDIO_COPY error: %"PRId64, (int64_t)uffdio_copy.copy);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) if (test_uffdio_copy_eexist && retry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) test_uffdio_copy_eexist = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) retry_copy_page(ufd, &uffdio_copy, offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) static int copy_page_retry(int ufd, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619) return __copy_page(ufd, offset, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) static int copy_page(int ufd, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) return __copy_page(ufd, offset, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) static int uffd_read_msg(int ufd, struct uffd_msg *msg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) int ret = read(uffd, msg, sizeof(*msg));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631) if (ret != sizeof(*msg)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) if (ret < 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) if (errno == EAGAIN)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) err("blocking read error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) err("short read");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) static void uffd_handle_page_fault(struct uffd_msg *msg,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) struct uffd_stats *stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) unsigned long offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649) if (msg->event != UFFD_EVENT_PAGEFAULT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) err("unexpected msg event %u", msg->event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) /* Write protect page faults */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) wp_range(uffd, msg->arg.pagefault.address, page_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655) stats->wp_faults++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) } else if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_MINOR) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) uint8_t *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) int b;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * Minor page faults
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) * To prove we can modify the original range for testing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) * purposes, we're going to bit flip this range before
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665) * continuing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * Note that this requires all minor page fault tests operate on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * area_dst (non-UFFD-registered) and area_dst_alias
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) * (UFFD-registered).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) area = (uint8_t *)(area_dst +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) ((char *)msg->arg.pagefault.address -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) area_dst_alias));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) for (b = 0; b < page_size; ++b)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676) area[b] = ~area[b];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) continue_range(uffd, msg->arg.pagefault.address, page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) stats->minor_faults++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) /* Missing page faults */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) if (msg->arg.pagefault.flags & UFFD_PAGEFAULT_FLAG_WRITE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) err("unexpected write fault");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684) offset = (char *)(unsigned long)msg->arg.pagefault.address - area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) offset &= ~(page_size-1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687) if (copy_page(uffd, offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) stats->missing_faults++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) static void *uffd_poll_thread(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) struct uffd_stats *stats = (struct uffd_stats *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) unsigned long cpu = stats->cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) struct pollfd pollfd[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) struct uffd_msg msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698) struct uffdio_register uffd_reg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) char tmp_chr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) pollfd[0].fd = uffd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703) pollfd[0].events = POLLIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) pollfd[1].fd = pipefd[cpu*2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) pollfd[1].events = POLLIN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) ret = poll(pollfd, 2, -1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) if (ret <= 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) err("poll error: %d", ret);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) if (pollfd[1].revents & POLLIN) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) if (read(pollfd[1].fd, &tmp_chr, 1) != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) err("read pipefd error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) if (!(pollfd[0].revents & POLLIN))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) err("pollfd[0].revents %d", pollfd[0].revents);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) if (uffd_read_msg(uffd, &msg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) switch (msg.event) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721) default:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) err("unexpected msg event %u\n", msg.event);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724) case UFFD_EVENT_PAGEFAULT:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) uffd_handle_page_fault(&msg, stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) case UFFD_EVENT_FORK:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) close(uffd);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) uffd = msg.arg.fork.ufd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) pollfd[0].fd = uffd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732) case UFFD_EVENT_REMOVE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) uffd_reg.range.start = msg.arg.remove.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) uffd_reg.range.len = msg.arg.remove.end -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) msg.arg.remove.start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) if (ioctl(uffd, UFFDIO_UNREGISTER, &uffd_reg.range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737) err("remove failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) case UFFD_EVENT_REMAP:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) area_dst = (char *)(unsigned long)msg.arg.remap.to;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) pthread_mutex_t uffd_read_mutex = PTHREAD_MUTEX_INITIALIZER;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) static void *uffd_read_thread(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) struct uffd_stats *stats = (struct uffd_stats *)arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) struct uffd_msg msg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) pthread_mutex_unlock(&uffd_read_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) /* from here cancellation is ok */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) for (;;) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (uffd_read_msg(uffd, &msg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) uffd_handle_page_fault(&msg, stats);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) static void *background_thread(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) unsigned long cpu = (unsigned long) arg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) unsigned long page_nr, start_nr, mid_nr, end_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) start_nr = cpu * nr_pages_per_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773) end_nr = (cpu+1) * nr_pages_per_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) mid_nr = (start_nr + end_nr) / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) /* Copy the first half of the pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) for (page_nr = start_nr; page_nr < mid_nr; page_nr++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) copy_page_retry(uffd, page_nr * page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) * If we need to test uffd-wp, set it up now. Then we'll have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) * at least the first half of the pages mapped already which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783) * can be write-protected for testing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) wp_range(uffd, (unsigned long)area_dst + start_nr * page_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) nr_pages_per_cpu * page_size, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 789) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 790) * Continue the 2nd half of the page copying, handling write
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 791) * protection faults if any
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 792) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 793) for (page_nr = mid_nr; page_nr < end_nr; page_nr++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 794) copy_page_retry(uffd, page_nr * page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 795)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 796) return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 797) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 798)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 799) static int stress(struct uffd_stats *uffd_stats)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 800) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 801) unsigned long cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 802) pthread_t locking_threads[nr_cpus];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 803) pthread_t uffd_threads[nr_cpus];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 804) pthread_t background_threads[nr_cpus];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 805)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 806) finished = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 807) for (cpu = 0; cpu < nr_cpus; cpu++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 808) if (pthread_create(&locking_threads[cpu], &attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 809) locking_thread, (void *)cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 810) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 811) if (bounces & BOUNCE_POLL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 812) if (pthread_create(&uffd_threads[cpu], &attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 813) uffd_poll_thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 814) (void *)&uffd_stats[cpu]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 815) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 816) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 817) if (pthread_create(&uffd_threads[cpu], &attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 818) uffd_read_thread,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 819) (void *)&uffd_stats[cpu]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 820) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 821) pthread_mutex_lock(&uffd_read_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 822) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 823) if (pthread_create(&background_threads[cpu], &attr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 824) background_thread, (void *)cpu))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 825) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 826) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 827) for (cpu = 0; cpu < nr_cpus; cpu++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 828) if (pthread_join(background_threads[cpu], NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 829) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 830)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 831) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 832) * Be strict and immediately zap area_src, the whole area has
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 833) * been transferred already by the background treads. The
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 834) * area_src could then be faulted in in a racy way by still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 835) * running uffdio_threads reading zeropages after we zapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 836) * area_src (but they're guaranteed to get -EEXIST from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 837) * UFFDIO_COPY without writing zero pages into area_dst
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 838) * because the background threads already completed).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 839) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 840) uffd_test_ops->release_pages(area_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 841)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 842) finished = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 843) for (cpu = 0; cpu < nr_cpus; cpu++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 844) if (pthread_join(locking_threads[cpu], NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 845) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 846)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 847) for (cpu = 0; cpu < nr_cpus; cpu++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 848) char c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 849) if (bounces & BOUNCE_POLL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 850) if (write(pipefd[cpu*2+1], &c, 1) != 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 851) err("pipefd write error");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 852) if (pthread_join(uffd_threads[cpu],
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 853) (void *)&uffd_stats[cpu]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 854) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 855) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 856) if (pthread_cancel(uffd_threads[cpu]))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 857) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 858) if (pthread_join(uffd_threads[cpu], NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 859) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 860) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 861) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 862)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 863) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 864) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 865)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 866) sigjmp_buf jbuf, *sigbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 867)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 868) static void sighndl(int sig, siginfo_t *siginfo, void *ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 869) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 870) if (sig == SIGBUS) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 871) if (sigbuf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 872) siglongjmp(*sigbuf, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 873) abort();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 874) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 875) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 876)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 877) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 878) * For non-cooperative userfaultfd test we fork() a process that will
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 879) * generate pagefaults, will mremap the area monitored by the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 880) * userfaultfd and at last this process will release the monitored
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 881) * area.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 882) * For the anonymous and shared memory the area is divided into two
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 883) * parts, the first part is accessed before mremap, and the second
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 884) * part is accessed after mremap. Since hugetlbfs does not support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 885) * mremap, the entire monitored area is accessed in a single pass for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 886) * HUGETLB_TEST.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 887) * The release of the pages currently generates event for shmem and
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 888) * anonymous memory (UFFD_EVENT_REMOVE), hence it is not checked
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 889) * for hugetlb.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 890) * For signal test(UFFD_FEATURE_SIGBUS), signal_test = 1, we register
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 891) * monitored area, generate pagefaults and test that signal is delivered.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 892) * Use UFFDIO_COPY to allocate missing page and retry. For signal_test = 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 893) * test robustness use case - we release monitored area, fork a process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 894) * that will generate pagefaults and verify signal is generated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 895) * This also tests UFFD_FEATURE_EVENT_FORK event along with the signal
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 896) * feature. Using monitor thread, verify no userfault events are generated.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 897) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 898) static int faulting_process(int signal_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 899) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 900) unsigned long nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 901) unsigned long long count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 902) unsigned long split_nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 903) unsigned long lastnr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 904) struct sigaction act;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 905) unsigned long signalled = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 906)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 907) if (test_type != TEST_HUGETLB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 908) split_nr_pages = (nr_pages + 1) / 2;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 909) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 910) split_nr_pages = nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 911)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 912) if (signal_test) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 913) sigbuf = &jbuf;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 914) memset(&act, 0, sizeof(act));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 915) act.sa_sigaction = sighndl;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 916) act.sa_flags = SA_SIGINFO;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 917) if (sigaction(SIGBUS, &act, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 918) err("sigaction");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 919) lastnr = (unsigned long)-1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 920) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 921)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 922) for (nr = 0; nr < split_nr_pages; nr++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 923) int steps = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 924) unsigned long offset = nr * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 925)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 926) if (signal_test) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 927) if (sigsetjmp(*sigbuf, 1) != 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 928) if (steps == 1 && nr == lastnr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 929) err("Signal repeated");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 930)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 931) lastnr = nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 932) if (signal_test == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 933) if (steps == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 934) /* This is a MISSING request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 935) steps++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 936) if (copy_page(uffd, offset))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 937) signalled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 938) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 939) /* This is a WP request */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 940) assert(steps == 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 941) wp_range(uffd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 942) (__u64)area_dst +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 943) offset,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 944) page_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 945) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 946) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 947) signalled++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 948) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 949) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 950) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 951) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 952)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 953) count = *area_count(area_dst, nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 954) if (count != count_verify[nr])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 955) err("nr %lu memory corruption %llu %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 956) nr, count, count_verify[nr]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 957) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 958) * Trigger write protection if there is by writing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 959) * the same value back.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 960) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 961) *area_count(area_dst, nr) = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 962) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 963)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 964) if (signal_test)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 965) return signalled != split_nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 966)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 967) if (test_type == TEST_HUGETLB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 968) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 969)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 970) area_dst = mremap(area_dst, nr_pages * page_size, nr_pages * page_size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 971) MREMAP_MAYMOVE | MREMAP_FIXED, area_src);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 972) if (area_dst == MAP_FAILED)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 973) err("mremap");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 974) /* Reset area_src since we just clobbered it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 975) area_src = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 976)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 977) for (; nr < nr_pages; nr++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 978) count = *area_count(area_dst, nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 979) if (count != count_verify[nr]) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 980) err("nr %lu memory corruption %llu %llu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 981) nr, count, count_verify[nr]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 982) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 983) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 984) * Trigger write protection if there is by writing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 985) * the same value back.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 986) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 987) *area_count(area_dst, nr) = count;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 988) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 989)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 990) uffd_test_ops->release_pages(area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 991)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 992) for (nr = 0; nr < nr_pages; nr++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 993) if (my_bcmp(area_dst + nr * page_size, zeropage, page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 994) err("nr %lu is not zero", nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 995)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 996) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 997) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 998)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 999) static void retry_uffdio_zeropage(int ufd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1000) struct uffdio_zeropage *uffdio_zeropage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1001) unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1002) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1003) uffd_test_ops->alias_mapping(&uffdio_zeropage->range.start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1004) uffdio_zeropage->range.len,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1005) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1006) if (ioctl(ufd, UFFDIO_ZEROPAGE, uffdio_zeropage)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1007) if (uffdio_zeropage->zeropage != -EEXIST)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1008) err("UFFDIO_ZEROPAGE error: %"PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1009) (int64_t)uffdio_zeropage->zeropage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1010) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1011) err("UFFDIO_ZEROPAGE error: %"PRId64,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1012) (int64_t)uffdio_zeropage->zeropage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1013) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1014) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1015)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1016) static int __uffdio_zeropage(int ufd, unsigned long offset, bool retry)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1017) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1018) struct uffdio_zeropage uffdio_zeropage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1019) int ret;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1020) unsigned long has_zeropage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1021) __s64 res;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1022)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1023) has_zeropage = uffd_test_ops->expected_ioctls & (1 << _UFFDIO_ZEROPAGE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1025) if (offset >= nr_pages * page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1026) err("unexpected offset %lu", offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1027) uffdio_zeropage.range.start = (unsigned long) area_dst + offset;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1028) uffdio_zeropage.range.len = page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1029) uffdio_zeropage.mode = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1030) ret = ioctl(ufd, UFFDIO_ZEROPAGE, &uffdio_zeropage);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1031) res = uffdio_zeropage.zeropage;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1032) if (ret) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1033) /* real retval in ufdio_zeropage.zeropage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1034) if (has_zeropage)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1035) err("UFFDIO_ZEROPAGE error: %"PRId64, (int64_t)res);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1036) else if (res != -EINVAL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1037) err("UFFDIO_ZEROPAGE not -EINVAL");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1038) } else if (has_zeropage) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1039) if (res != page_size) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1040) err("UFFDIO_ZEROPAGE unexpected size");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1041) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1042) if (test_uffdio_zeropage_eexist && retry) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1043) test_uffdio_zeropage_eexist = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1044) retry_uffdio_zeropage(ufd, &uffdio_zeropage,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1045) offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1046) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1047) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1048) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1049) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1050) err("UFFDIO_ZEROPAGE succeeded");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1051)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1052) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1053) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1054)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1055) static int uffdio_zeropage(int ufd, unsigned long offset)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1056) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1057) return __uffdio_zeropage(ufd, offset, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1058) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1059)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1060) /* exercise UFFDIO_ZEROPAGE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1061) static int userfaultfd_zeropage_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1062) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1063) struct uffdio_register uffdio_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1064) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1065)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1066) printf("testing UFFDIO_ZEROPAGE: ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1067) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1068)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1069) uffd_test_ctx_init(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1070)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1071) uffdio_register.range.start = (unsigned long) area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1072) uffdio_register.range.len = nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1073) uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1074) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1075) uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1076) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1077) err("register failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1078)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1079) expected_ioctls = uffd_test_ops->expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1080) if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1081) err("unexpected missing ioctl for anon memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1082)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1083) if (uffdio_zeropage(uffd, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1084) if (my_bcmp(area_dst, zeropage, page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1085) err("zeropage is not zero");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1086)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1087) printf("done.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1088) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1089) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1090)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1091) static int userfaultfd_events_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1092) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1093) struct uffdio_register uffdio_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1094) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1095) pthread_t uffd_mon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1096) int err, features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1097) pid_t pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1098) char c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1099) struct uffd_stats stats = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1100)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1101) printf("testing events (fork, remap, remove): ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1102) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1103)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1104) features = UFFD_FEATURE_EVENT_FORK | UFFD_FEATURE_EVENT_REMAP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1105) UFFD_FEATURE_EVENT_REMOVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1106) uffd_test_ctx_init(features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1108) fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1110) uffdio_register.range.start = (unsigned long) area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1111) uffdio_register.range.len = nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1112) uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1113) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1114) uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1115) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1116) err("register failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1117)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1118) expected_ioctls = uffd_test_ops->expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1119) if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1120) err("unexpected missing ioctl for anon memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1121)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1122) if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1123) err("uffd_poll_thread create");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1124)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1125) pid = fork();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1126) if (pid < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1127) err("fork");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1128)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1129) if (!pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1130) exit(faulting_process(0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1131)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1132) waitpid(pid, &err, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1133) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1134) err("faulting process failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1135) if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1136) err("pipe write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1137) if (pthread_join(uffd_mon, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1138) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1139)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1140) uffd_stats_report(&stats, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1141)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1142) return stats.missing_faults != nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1144)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1145) static int userfaultfd_sig_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1147) struct uffdio_register uffdio_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1148) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1149) unsigned long userfaults;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1150) pthread_t uffd_mon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1151) int err, features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1152) pid_t pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1153) char c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1154) struct uffd_stats stats = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1155)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1156) printf("testing signal delivery: ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1157) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1159) features = UFFD_FEATURE_EVENT_FORK|UFFD_FEATURE_SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1160) uffd_test_ctx_init(features);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1161)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1162) fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1164) uffdio_register.range.start = (unsigned long) area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1165) uffdio_register.range.len = nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1166) uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1167) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1168) uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1169) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1170) err("register failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1171)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1172) expected_ioctls = uffd_test_ops->expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1173) if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1174) err("unexpected missing ioctl for anon memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1176) if (faulting_process(1))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1177) err("faulting process failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1179) uffd_test_ops->release_pages(area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1180)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1181) if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1182) err("uffd_poll_thread create");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1183)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1184) pid = fork();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1185) if (pid < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1186) err("fork");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1188) if (!pid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1189) exit(faulting_process(2));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1190)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1191) waitpid(pid, &err, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1192) if (err)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1193) err("faulting process failed");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1194) if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1195) err("pipe write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1196) if (pthread_join(uffd_mon, (void **)&userfaults))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1197) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1199) printf("done.\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1200) if (userfaults)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1201) err("Signal test failed, userfaults: %ld", userfaults);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1203) return userfaults != 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1204) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1205)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1206) static int userfaultfd_minor_test(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1207) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1208) struct uffdio_register uffdio_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1209) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1210) unsigned long p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1211) pthread_t uffd_mon;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1212) uint8_t expected_byte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1213) void *expected_page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1214) char c;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1215) struct uffd_stats stats = { 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1216) uint64_t req_features, features_out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1217)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1218) if (!test_uffdio_minor)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1219) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1220)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1221) printf("testing minor faults: ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1222) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1223)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1224) if (test_type == TEST_HUGETLB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1225) req_features = UFFD_FEATURE_MINOR_HUGETLBFS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1226) else if (test_type == TEST_SHMEM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1227) req_features = UFFD_FEATURE_MINOR_SHMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1228) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1229) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1230)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1231) features_out = req_features;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1232) uffd_test_ctx_init_ext(&features_out);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1233) /* If kernel reports required features aren't supported, skip test. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1234) if ((features_out & req_features) != req_features) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1235) printf("skipping test due to lack of feature support\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1236) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1237) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1238) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1240) uffdio_register.range.start = (unsigned long)area_dst_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1241) uffdio_register.range.len = nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1242) uffdio_register.mode = UFFDIO_REGISTER_MODE_MINOR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1243) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1244) err("register failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1246) expected_ioctls = uffd_test_ops->expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1247) expected_ioctls |= 1 << _UFFDIO_CONTINUE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1248) if ((uffdio_register.ioctls & expected_ioctls) != expected_ioctls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1249) err("unexpected missing ioctl(s)");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1251) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1252) * After registering with UFFD, populate the non-UFFD-registered side of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1253) * the shared mapping. This should *not* trigger any UFFD minor faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1254) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1255) for (p = 0; p < nr_pages; ++p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1256) memset(area_dst + (p * page_size), p % ((uint8_t)-1),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1257) page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1258) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1259)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1260) if (pthread_create(&uffd_mon, &attr, uffd_poll_thread, &stats))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1261) err("uffd_poll_thread create");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1262)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1263) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1264) * Read each of the pages back using the UFFD-registered mapping. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1265) * expect that the first time we touch a page, it will result in a minor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1266) * fault. uffd_poll_thread will resolve the fault by bit-flipping the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1267) * page's contents, and then issuing a CONTINUE ioctl.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1268) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1269)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1270) if (posix_memalign(&expected_page, page_size, page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1271) err("out of memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1272)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1273) for (p = 0; p < nr_pages; ++p) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1274) expected_byte = ~((uint8_t)(p % ((uint8_t)-1)));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1275) memset(expected_page, expected_byte, page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1276) if (my_bcmp(expected_page, area_dst_alias + (p * page_size),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1277) page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1278) err("unexpected page contents after minor fault");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1279) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1280)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1281) if (write(pipefd[1], &c, sizeof(c)) != sizeof(c))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1282) err("pipe write");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1283) if (pthread_join(uffd_mon, NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1284) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1286) uffd_stats_report(&stats, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1287)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1288) return stats.missing_faults != 0 || stats.minor_faults != nr_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1289) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1291) static int userfaultfd_stress(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1292) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1293) void *area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1294) char *tmp_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1295) unsigned long nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1296) struct uffdio_register uffdio_register;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1297) struct uffd_stats uffd_stats[nr_cpus];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1299) uffd_test_ctx_init(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1300)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1301) if (posix_memalign(&area, page_size, page_size))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1302) err("out of memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1303) zeropage = area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1304) bzero(zeropage, page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1305)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1306) pthread_mutex_lock(&uffd_read_mutex);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1307)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1308) pthread_attr_init(&attr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1309) pthread_attr_setstacksize(&attr, 16*1024*1024);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1310)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1311) while (bounces--) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1312) unsigned long expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1313)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1314) printf("bounces: %d, mode:", bounces);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1315) if (bounces & BOUNCE_RANDOM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1316) printf(" rnd");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1317) if (bounces & BOUNCE_RACINGFAULTS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1318) printf(" racing");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1319) if (bounces & BOUNCE_VERIFY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1320) printf(" ver");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1321) if (bounces & BOUNCE_POLL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1322) printf(" poll");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1323) printf(", ");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1324) fflush(stdout);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1325)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1326) if (bounces & BOUNCE_POLL)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1327) fcntl(uffd, F_SETFL, uffd_flags | O_NONBLOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1328) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1329) fcntl(uffd, F_SETFL, uffd_flags & ~O_NONBLOCK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1331) /* register */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1332) uffdio_register.range.start = (unsigned long) area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1333) uffdio_register.range.len = nr_pages * page_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1334) uffdio_register.mode = UFFDIO_REGISTER_MODE_MISSING;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1335) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1336) uffdio_register.mode |= UFFDIO_REGISTER_MODE_WP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1337) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1338) err("register failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1339) expected_ioctls = uffd_test_ops->expected_ioctls;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1340) if ((uffdio_register.ioctls & expected_ioctls) !=
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1341) expected_ioctls)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1342) err("unexpected missing ioctl for anon memory");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1344) if (area_dst_alias) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1345) uffdio_register.range.start = (unsigned long)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1346) area_dst_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1347) if (ioctl(uffd, UFFDIO_REGISTER, &uffdio_register))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1348) err("register failure alias");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1349) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1350)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1351) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1352) * The madvise done previously isn't enough: some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1353) * uffd_thread could have read userfaults (one of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1354) * those already resolved by the background thread)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1355) * and it may be in the process of calling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1356) * UFFDIO_COPY. UFFDIO_COPY will read the zapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1357) * area_src and it would map a zero page in it (of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1358) * course such a UFFDIO_COPY is perfectly safe as it'd
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1359) * return -EEXIST). The problem comes at the next
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1360) * bounce though: that racing UFFDIO_COPY would
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1361) * generate zeropages in the area_src, so invalidating
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1362) * the previous MADV_DONTNEED. Without this additional
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1363) * MADV_DONTNEED those zeropages leftovers in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1364) * area_src would lead to -EEXIST failure during the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1365) * next bounce, effectively leaving a zeropage in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1366) * area_dst.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1367) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1368) * Try to comment this out madvise to see the memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1369) * corruption being caught pretty quick.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1370) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1371) * khugepaged is also inhibited to collapse THP after
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1372) * MADV_DONTNEED only after the UFFDIO_REGISTER, so it's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1373) * required to MADV_DONTNEED here.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1374) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1375) uffd_test_ops->release_pages(area_dst);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1376)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1377) uffd_stats_reset(uffd_stats, nr_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1378)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1379) /* bounce pass */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1380) if (stress(uffd_stats))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1381) return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1383) /* Clear all the write protections if there is any */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1384) if (test_uffdio_wp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1385) wp_range(uffd, (unsigned long)area_dst,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1386) nr_pages * page_size, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1388) /* unregister */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1389) if (ioctl(uffd, UFFDIO_UNREGISTER, &uffdio_register.range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1390) err("unregister failure");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1391) if (area_dst_alias) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1392) uffdio_register.range.start = (unsigned long) area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1393) if (ioctl(uffd, UFFDIO_UNREGISTER,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1394) &uffdio_register.range))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1395) err("unregister failure alias");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1398) /* verification */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1399) if (bounces & BOUNCE_VERIFY)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1400) for (nr = 0; nr < nr_pages; nr++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1401) if (*area_count(area_dst, nr) != count_verify[nr])
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1402) err("error area_count %llu %llu %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1403) *area_count(area_src, nr),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1404) count_verify[nr], nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1405)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1406) /* prepare next bounce */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1407) tmp_area = area_src;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1408) area_src = area_dst;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1409) area_dst = tmp_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1410)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1411) tmp_area = area_src_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1412) area_src_alias = area_dst_alias;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1413) area_dst_alias = tmp_area;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1415) uffd_stats_report(uffd_stats, nr_cpus);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1416) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1418) return userfaultfd_zeropage_test() || userfaultfd_sig_test()
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1419) || userfaultfd_events_test() || userfaultfd_minor_test();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1420) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1421)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1422) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1423) * Copied from mlock2-tests.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1424) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1425) unsigned long default_huge_page_size(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1426) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1427) unsigned long hps = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1428) char *line = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1429) size_t linelen = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1430) FILE *f = fopen("/proc/meminfo", "r");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1431)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1432) if (!f)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1433) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1434) while (getline(&line, &linelen, f) > 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1435) if (sscanf(line, "Hugepagesize: %lu kB", &hps) == 1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1436) hps <<= 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1437) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1438) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1439) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1441) free(line);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1442) fclose(f);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1443) return hps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1444) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1445)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1446) static void set_test_type(const char *type)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1447) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1448) if (!strcmp(type, "anon")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1449) test_type = TEST_ANON;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1450) uffd_test_ops = &anon_uffd_test_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1451) /* Only enable write-protect test for anonymous test */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1452) test_uffdio_wp = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1453) } else if (!strcmp(type, "hugetlb")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1454) test_type = TEST_HUGETLB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1455) uffd_test_ops = &hugetlb_uffd_test_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1456) } else if (!strcmp(type, "hugetlb_shared")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1457) map_shared = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1458) test_type = TEST_HUGETLB;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1459) uffd_test_ops = &hugetlb_uffd_test_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1460) /* Minor faults require shared hugetlb; only enable here. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1461) test_uffdio_minor = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1462) } else if (!strcmp(type, "shmem")) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1463) map_shared = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1464) test_type = TEST_SHMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1465) uffd_test_ops = &shmem_uffd_test_ops;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1466) test_uffdio_minor = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1467) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1468) err("Unknown test type: %s", type);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1469) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1470)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1471) if (test_type == TEST_HUGETLB)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1472) page_size = default_huge_page_size();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1473) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1474) page_size = sysconf(_SC_PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1476) if (!page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1477) err("Unable to determine page size");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1478) if ((unsigned long) area_count(NULL, 0) + sizeof(unsigned long long) * 2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1479) > page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1480) err("Impossible to run this test");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1481) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1483) static void sigalrm(int sig)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1484) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1485) if (sig != SIGALRM)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1486) abort();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1487) test_uffdio_copy_eexist = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1488) test_uffdio_zeropage_eexist = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1489) alarm(ALARM_INTERVAL_SECS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1490) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1491)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1492) int main(int argc, char **argv)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1493) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1494) if (argc < 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1495) usage();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1497) if (signal(SIGALRM, sigalrm) == SIG_ERR)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1498) err("failed to arm SIGALRM");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1499) alarm(ALARM_INTERVAL_SECS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1501) set_test_type(argv[1]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1502)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1503) nr_cpus = sysconf(_SC_NPROCESSORS_ONLN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1504) nr_pages_per_cpu = atol(argv[2]) * 1024*1024 / page_size /
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1505) nr_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1506) if (!nr_pages_per_cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1507) _err("invalid MiB");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1508) usage();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1509) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1510)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1511) bounces = atoi(argv[3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1512) if (bounces <= 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1513) _err("invalid bounces");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1514) usage();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1515) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1516) nr_pages = nr_pages_per_cpu * nr_cpus;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1517)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1518) if (test_type == TEST_HUGETLB) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1519) if (argc < 5)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1520) usage();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1521) huge_fd = open(argv[4], O_CREAT | O_RDWR, 0755);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1522) if (huge_fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1523) err("Open of %s failed", argv[4]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1524) if (ftruncate(huge_fd, 0))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1525) err("ftruncate %s to size 0 failed", argv[4]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1526) } else if (test_type == TEST_SHMEM) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1527) shm_fd = memfd_create(argv[0], 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1528) if (shm_fd < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1529) err("memfd_create");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1530) if (ftruncate(shm_fd, nr_pages * page_size * 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1531) err("ftruncate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1532) if (fallocate(shm_fd,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1533) FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, 0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1534) nr_pages * page_size * 2))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1535) err("fallocate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1537) printf("nr_pages: %lu, nr_pages_per_cpu: %lu\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1538) nr_pages, nr_pages_per_cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1539) return userfaultfd_stress();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1540) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1541)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1542) #else /* __NR_userfaultfd */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1544) #warning "missing __NR_userfaultfd definition"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1545)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1546) int main(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1547) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1548) printf("skip: Skipping userfaultfd test (missing __NR_userfaultfd)\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1549) return KSFT_SKIP;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1552) #endif /* __NR_userfaultfd */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags