^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 2) * hugetlbpage-backed filesystem. Based on ramfs.
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 3) *
6d49e352ae9ae (Nadia Yvette Chambers 2012-12-06 10:39:54 +0100 4) * Nadia Yvette Chambers, 2002
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 5) *
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 6) * Copyright (C) 2002 Linus Torvalds.
3e89e1c5ea842 (Paul Gortmaker 2016-01-14 15:21:52 -0800 7) * License: GPL
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 8) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 9)
9b857d26d08f0 (Andrew Morton 2014-06-04 16:07:21 -0700 10) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9b857d26d08f0 (Andrew Morton 2014-06-04 16:07:21 -0700 11)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 12) #include <linux/thread_info.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 13) #include <asm/current.h>
174cd4b1e5fbd (Ingo Molnar 2017-02-02 19:15:33 +0100 14) #include <linux/sched/signal.h> /* remove ASAP */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 15) #include <linux/falloc.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 16) #include <linux/fs.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 17) #include <linux/mount.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 18) #include <linux/file.h>
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 19) #include <linux/kernel.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 20) #include <linux/writeback.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 21) #include <linux/pagemap.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 22) #include <linux/highmem.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 23) #include <linux/init.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 24) #include <linux/string.h>
16f7e0fe2ecc3 (Randy Dunlap 2006-01-11 12:17:46 -0800 25) #include <linux/capability.h>
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 26) #include <linux/ctype.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 27) #include <linux/backing-dev.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 28) #include <linux/hugetlb.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 29) #include <linux/pagevec.h>
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 30) #include <linux/fs_parser.h>
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 31) #include <linux/mman.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 32) #include <linux/slab.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 33) #include <linux/dnotify.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 34) #include <linux/statfs.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 35) #include <linux/security.h>
1fd7317d02ec0 (Nick Black 2009-09-22 16:43:33 -0700 36) #include <linux/magic.h>
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 37) #include <linux/migrate.h>
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 38) #include <linux/uio.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 39)
7c0f6ba682b9c (Linus Torvalds 2016-12-24 11:46:01 -0800 40) #include <linux/uaccess.h>
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 41) #include <linux/sched/mm.h>
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 42)
ee9b6d61a2a43 (Josef 'Jeff' Sipek 2007-02-12 00:55:41 -0800 43) static const struct super_operations hugetlbfs_ops;
f5e54d6e53a20 (Christoph Hellwig 2006-06-28 04:26:44 -0700 44) static const struct address_space_operations hugetlbfs_aops;
4b6f5d20b04dc (Arjan van de Ven 2006-03-28 01:56:42 -0800 45) const struct file_operations hugetlbfs_file_operations;
92e1d5be91a0e (Arjan van de Ven 2007-02-12 00:55:39 -0800 46) static const struct inode_operations hugetlbfs_dir_inode_operations;
92e1d5be91a0e (Arjan van de Ven 2007-02-12 00:55:39 -0800 47) static const struct inode_operations hugetlbfs_inode_operations;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 48)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 49) enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 50)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 51) struct hugetlbfs_fs_context {
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 52) struct hstate *hstate;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 53) unsigned long long max_size_opt;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 54) unsigned long long min_size_opt;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 55) long max_hpages;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 56) long nr_inodes;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 57) long min_hpages;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 58) enum hugetlbfs_size_type max_val_type;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 59) enum hugetlbfs_size_type min_val_type;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 60) kuid_t uid;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 61) kgid_t gid;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 62) umode_t mode;
a1d776ee3147c (David Gibson 2012-03-21 16:34:12 -0700 63) };
a1d776ee3147c (David Gibson 2012-03-21 16:34:12 -0700 64)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 65) int sysctl_hugetlb_shm_group;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 66)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 67) enum hugetlb_param {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 68) Opt_gid,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 69) Opt_min_size,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 70) Opt_mode,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 71) Opt_nr_inodes,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 72) Opt_pagesize,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 73) Opt_size,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 74) Opt_uid,
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 75) };
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 76)
d7167b149943e (Al Viro 2019-09-07 07:23:15 -0400 77) static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 78) fsparam_u32 ("gid", Opt_gid),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 79) fsparam_string("min_size", Opt_min_size),
79da14fac0b5f (Mike Kravetz 2021-07-23 15:50:44 -0700 80) fsparam_u32oct("mode", Opt_mode),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 81) fsparam_string("nr_inodes", Opt_nr_inodes),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 82) fsparam_string("pagesize", Opt_pagesize),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 83) fsparam_string("size", Opt_size),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 84) fsparam_u32 ("uid", Opt_uid),
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 85) {}
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 86) };
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 87)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 88) #ifdef CONFIG_NUMA
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 89) static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 90) struct inode *inode, pgoff_t index)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 91) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 92) vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 93) index);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 94) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 95)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 96) static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 97) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 98) mpol_cond_put(vma->vm_policy);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 99) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 100) #else
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 101) static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 102) struct inode *inode, pgoff_t index)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 103) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 104) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 105)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 106) static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 107) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 108) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 109) #endif
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 110)
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 111) static void huge_pagevec_release(struct pagevec *pvec)
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 112) {
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 113) int i;
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 114)
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 115) for (i = 0; i < pagevec_count(pvec); ++i)
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 116) put_page(pvec->pages[i]);
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 117)
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 118) pagevec_reinit(pvec);
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 119) }
2e9b367c2273e (Adam Litke 2005-10-29 18:16:47 -0700 120)
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 121) /*
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 122) * Mask used when checking the page offset value passed in via system
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 123) * calls. This value will be converted to a loff_t which is signed.
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 124) * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 125) * value. The extra bit (- 1 in the shift value) is to take the sign
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 126) * bit into account.
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 127) */
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 128) #define PGOFF_LOFFT_MAX \
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 129) (((1UL << (PAGE_SHIFT + 1)) - 1) << (BITS_PER_LONG - (PAGE_SHIFT + 1)))
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 130)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 131) static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 132) {
496ad9aa8ef44 (Al Viro 2013-01-23 17:07:38 -0500 133) struct inode *inode = file_inode(file);
22247efd822e6 (Peter Xu 2021-05-14 17:27:04 -0700 134) struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 135) loff_t len, vma_len;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 136) int ret;
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 137) struct hstate *h = hstate_file(file);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 138)
68589bc353037 (Hugh Dickins 2006-11-14 02:03:32 -0800 139) /*
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 140) * vma address alignment (but not the pgoff alignment) has
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 141) * already been checked by prepare_hugepage_range. If you add
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 142) * any error returns here, do so after setting VM_HUGETLB, so
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 143) * is_vm_hugetlb_page tests below unmap_region go the right
45e55300f1149 (Peter Collingbourne 2020-08-06 23:23:37 -0700 144) * way when do_mmap unwinds (may be important on powerpc
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 145) * and ia64).
68589bc353037 (Hugh Dickins 2006-11-14 02:03:32 -0800 146) */
a2fce9143057f (Naoya Horiguchi 2013-04-17 15:58:27 -0700 147) vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
68589bc353037 (Hugh Dickins 2006-11-14 02:03:32 -0800 148) vma->vm_ops = &hugetlb_vm_ops;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 149)
22247efd822e6 (Peter Xu 2021-05-14 17:27:04 -0700 150) ret = seal_check_future_write(info->seals, vma);
22247efd822e6 (Peter Xu 2021-05-14 17:27:04 -0700 151) if (ret)
22247efd822e6 (Peter Xu 2021-05-14 17:27:04 -0700 152) return ret;
22247efd822e6 (Peter Xu 2021-05-14 17:27:04 -0700 153)
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 154) /*
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 155) * page based offset in vm_pgoff could be sufficiently large to
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 156) * overflow a loff_t when converted to byte offset. This can
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 157) * only happen on architectures where sizeof(loff_t) ==
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 158) * sizeof(unsigned long). So, only check in those instances.
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 159) */
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 160) if (sizeof(unsigned long) == sizeof(loff_t)) {
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 161) if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 162) return -EINVAL;
5df63c2a149ae (Mike Kravetz 2018-04-05 16:18:21 -0700 163) }
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 164)
63489f8e82114 (Mike Kravetz 2018-03-22 16:17:13 -0700 165) /* must be huge page aligned */
2b37c35e6552b (Becky Bruce 2011-07-25 17:11:49 -0700 166) if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 167) return -EINVAL;
dec4ad86c2fbe (David Gibson 2007-08-30 23:56:40 -0700 168)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 169) vma_len = (loff_t)(vma->vm_end - vma->vm_start);
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 170) len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 171) /* check for overflow */
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 172) if (len < vma_len)
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 173) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 174)
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 175) inode_lock(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 176) file_accessed(file);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 177)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 178) ret = -ENOMEM;
33b8f84a4ee78 (Mike Kravetz 2021-02-24 12:09:54 -0800 179) if (!hugetlb_reserve_pages(inode,
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 180) vma->vm_pgoff >> huge_page_order(h),
5a6fe12595067 (Mel Gorman 2009-02-10 14:02:27 +0000 181) len >> huge_page_shift(h), vma,
5a6fe12595067 (Mel Gorman 2009-02-10 14:02:27 +0000 182) vma->vm_flags))
a43a8c39bbb49 (Kenneth W Chen 2006-06-23 02:03:15 -0700 183) goto out;
b45b5bd65f668 (David Gibson 2006-03-22 00:08:55 -0800 184)
4c88726597721 (Adam Litke 2005-10-29 18:16:46 -0700 185) ret = 0;
b6174df5eec9c (Zhang, Yanmin 2006-07-10 04:44:49 -0700 186) if (vma->vm_flags & VM_WRITE && inode->i_size < len)
045c7a3f53d94 (Mike Kravetz 2017-04-13 14:56:32 -0700 187) i_size_write(inode, len);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 188) out:
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 189) inode_unlock(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 190)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 191) return ret;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 192) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 193)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 194) /*
3e4e28c5a8f01 (Michel Lespinasse 2020-06-08 21:33:51 -0700 195) * Called under mmap_write_lock(mm).
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 196) */
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 197)
d2ba27e8007b3 (Adrian Bunk 2007-05-06 14:49:00 -0700 198) #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 199) static unsigned long
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 200) hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr,
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 201) unsigned long len, unsigned long pgoff, unsigned long flags)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 202) {
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 203) struct hstate *h = hstate_file(file);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 204) struct vm_unmapped_area_info info;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 205)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 206) info.flags = 0;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 207) info.length = len;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 208) info.low_limit = current->mm->mmap_base;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 209) info.high_limit = TASK_SIZE;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 210) info.align_mask = PAGE_MASK & ~huge_page_mask(h);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 211) info.align_offset = 0;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 212) return vm_unmapped_area(&info);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 213) }
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 214)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 215) static unsigned long
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 216) hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr,
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 217) unsigned long len, unsigned long pgoff, unsigned long flags)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 218) {
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 219) struct hstate *h = hstate_file(file);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 220) struct vm_unmapped_area_info info;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 221)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 222) info.flags = VM_UNMAPPED_AREA_TOPDOWN;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 223) info.length = len;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 224) info.low_limit = max(PAGE_SIZE, mmap_min_addr);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 225) info.high_limit = current->mm->mmap_base;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 226) info.align_mask = PAGE_MASK & ~huge_page_mask(h);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 227) info.align_offset = 0;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 228) addr = vm_unmapped_area(&info);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 229)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 230) /*
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 231) * A failed mmap() very likely causes application failure,
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 232) * so fall back to the bottom-up function here. This scenario
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 233) * can happen with large stack limits and large mmap()
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 234) * allocations.
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 235) */
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 236) if (unlikely(offset_in_page(addr))) {
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 237) VM_BUG_ON(addr != -ENOMEM);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 238) info.flags = 0;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 239) info.low_limit = current->mm->mmap_base;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 240) info.high_limit = TASK_SIZE;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 241) addr = vm_unmapped_area(&info);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 242) }
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 243)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 244) return addr;
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 245) }
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 246)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 247) static unsigned long
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 248) hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 249) unsigned long len, unsigned long pgoff, unsigned long flags)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 250) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 251) struct mm_struct *mm = current->mm;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 252) struct vm_area_struct *vma;
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 253) struct hstate *h = hstate_file(file);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 254)
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 255) if (len & ~huge_page_mask(h))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 256) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 257) if (len > TASK_SIZE)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 258) return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 259)
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 260) if (flags & MAP_FIXED) {
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 261) if (prepare_hugepage_range(file, addr, len))
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 262) return -EINVAL;
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 263) return addr;
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 264) }
036e08568cbee (Benjamin Herrenschmidt 2007-05-06 14:50:12 -0700 265)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 266) if (addr) {
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 267) addr = ALIGN(addr, huge_page_size(h));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 268) vma = find_vma(mm, addr);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 269) if (TASK_SIZE - len >= addr &&
1be7107fbe18e (Hugh Dickins 2017-06-19 04:03:24 -0700 270) (!vma || addr + len <= vm_start_gap(vma)))
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 271) return addr;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 272) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 273)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 274) /*
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 275) * Use mm->get_unmapped_area value as a hint to use topdown routine.
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 276) * If architectures have special needs, they should define their own
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 277) * version of hugetlb_get_unmapped_area.
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 278) */
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 279) if (mm->get_unmapped_area == arch_get_unmapped_area_topdown)
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 280) return hugetlb_get_unmapped_area_topdown(file, addr, len,
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 281) pgoff, flags);
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 282) return hugetlb_get_unmapped_area_bottomup(file, addr, len,
885902531586d (Shijie Hu 2020-06-03 16:03:34 -0700 283) pgoff, flags);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 284) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 285) #endif
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 286)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 287) static size_t
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 288) hugetlbfs_read_actor(struct page *page, unsigned long offset,
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 289) struct iov_iter *to, unsigned long size)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 290) {
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 291) size_t copied = 0;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 292) int i, chunksize;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 293)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 294) /* Find which 4k chunk and offset with in that chunk */
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 295) i = offset >> PAGE_SHIFT;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 296) offset = offset & ~PAGE_MASK;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 297)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 298) while (size) {
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 299) size_t n;
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 300) chunksize = PAGE_SIZE;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 301) if (offset)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 302) chunksize -= offset;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 303) if (chunksize > size)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 304) chunksize = size;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 305) n = copy_page_to_iter(&page[i], offset, chunksize, to);
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 306) copied += n;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 307) if (n != chunksize)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 308) return copied;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 309) offset = 0;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 310) size -= chunksize;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 311) i++;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 312) }
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 313) return copied;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 314) }
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 315)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 316) /*
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 317) * Support for read() - Find the page attached to f_mapping and copy out the
c7e285e31f764 (Miaohe Lin 2021-02-24 12:10:08 -0800 318) * data. Its *very* similar to generic_file_buffered_read(), we can't use that
ea1754a084760 (Kirill A. Shutemov 2016-04-01 15:29:48 +0300 319) * since it has PAGE_SIZE assumptions.
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 320) */
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 321) static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 322) {
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 323) struct file *file = iocb->ki_filp;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 324) struct hstate *h = hstate_file(file);
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 325) struct address_space *mapping = file->f_mapping;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 326) struct inode *inode = mapping->host;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 327) unsigned long index = iocb->ki_pos >> huge_page_shift(h);
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 328) unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 329) unsigned long end_index;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 330) loff_t isize;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 331) ssize_t retval = 0;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 332)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 333) while (iov_iter_count(to)) {
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 334) struct page *page;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 335) size_t nr, copied;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 336)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 337) /* nr is the maximum number of bytes to copy from this page */
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 338) nr = huge_page_size(h);
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 339) isize = i_size_read(inode);
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 340) if (!isize)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 341) break;
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 342) end_index = (isize - 1) >> huge_page_shift(h);
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 343) if (index > end_index)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 344) break;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 345) if (index == end_index) {
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 346) nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 347) if (nr <= offset)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 348) break;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 349) }
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 350) nr = nr - offset;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 351)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 352) /* Find the page */
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 353) page = find_lock_page(mapping, index);
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 354) if (unlikely(page == NULL)) {
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 355) /*
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 356) * We have a HOLE, zero out the user-buffer for the
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 357) * length of the hole or request.
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 358) */
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 359) copied = iov_iter_zero(nr, to);
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 360) } else {
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 361) unlock_page(page);
a05b0855fd155 (Aneesh Kumar K.V 2012-03-21 16:34:08 -0700 362)
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 363) /*
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 364) * We have the page, copy it to user space buffer.
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 365) */
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 366) copied = hugetlbfs_read_actor(page, offset, to, nr);
09cbfeaf1a5a6 (Kirill A. Shutemov 2016-04-01 15:29:47 +0300 367) put_page(page);
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 368) }
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 369) offset += copied;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 370) retval += copied;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 371) if (copied != nr && iov_iter_count(to)) {
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 372) if (!retval)
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 373) retval = -EFAULT;
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 374) break;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 375) }
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 376) index += offset >> huge_page_shift(h);
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 377) offset &= ~huge_page_mask(h);
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 378) }
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 379) iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 380) return retval;
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 381) }
e63e1e5a6b754 (Badari Pulavarty 2007-10-16 01:26:22 -0700 382)
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 383) static int hugetlbfs_write_begin(struct file *file,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 384) struct address_space *mapping,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 385) loff_t pos, unsigned len, unsigned flags,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 386) struct page **pagep, void **fsdata)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 387) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 388) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 389) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 390)
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 391) static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 392) loff_t pos, unsigned len, unsigned copied,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 393) struct page *page, void *fsdata)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 394) {
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 395) BUG();
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 396) return -EINVAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 397) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 398)
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 399) static void remove_huge_page(struct page *page)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 400) {
b9ea25152e563 (Konstantin Khlebnikov 2015-04-14 15:45:27 -0700 401) ClearPageDirty(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 402) ClearPageUptodate(page);
bd65cb86c98a7 (Minchan Kim 2011-03-22 16:30:54 -0700 403) delete_from_page_cache(page);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 404) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 405)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 406) static void
f808c13fd3738 (Davidlohr Bueso 2017-09-08 16:15:08 -0700 407) hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 408) {
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 409) struct vm_area_struct *vma;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 410)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 411) /*
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 412) * end == 0 indicates that the entire range after
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 413) * start should be unmapped.
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 414) */
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 415) vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 416) unsigned long v_offset;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 417) unsigned long v_end;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 418)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 419) /*
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 420) * Can the expression below overflow on 32-bit arches?
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 421) * No, because the interval tree returns us only those vmas
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 422) * which overlap the truncated area starting at pgoff,
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 423) * and no vma on a 32-bit arch can span beyond the 4GB.
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 424) */
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 425) if (vma->vm_pgoff < start)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 426) v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 427) else
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 428) v_offset = 0;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 429)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 430) if (!end)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 431) v_end = vma->vm_end;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 432) else {
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 433) v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 434) + vma->vm_start;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 435) if (v_end > vma->vm_end)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 436) v_end = vma->vm_end;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 437) }
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 438)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 439) unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 440) NULL);
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 441) }
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 442) }
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 443)
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 444) /*
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 445) * remove_inode_hugepages handles two distinct cases: truncation and hole
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 446) * punch. There are subtle differences in operation for each case.
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 447) *
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 448) * truncation is indicated by end of range being LLONG_MAX
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 449) * In this case, we first scan the range and release found pages.
1935ebd3cf6c4 (Miaohe Lin 2021-02-24 12:10:21 -0800 450) * After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 451) * maps and global counts. Page faults can not race with truncation
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 452) * in this routine. hugetlb_no_page() holds i_mmap_rwsem and prevents
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 453) * page faults in the truncated range by checking i_size. i_size is
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 454) * modified while holding i_mmap_rwsem.
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 455) * hole punch is indicated if end is not LLONG_MAX
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 456) * In the hole punch case we scan the range and release found pages.
1935ebd3cf6c4 (Miaohe Lin 2021-02-24 12:10:21 -0800 457) * Only when releasing a page is the associated region/reserve map
1935ebd3cf6c4 (Miaohe Lin 2021-02-24 12:10:21 -0800 458) * deleted. The region/reserve map for ranges without associated
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 459) * pages are not modified. Page faults can race with hole punch.
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 460) * This is indicated if we find a mapped page.
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 461) * Note: If the passed end of range value is beyond the end of file, but
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 462) * not LLONG_MAX this routine still performs a hole punch operation.
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 463) */
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 464) static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 465) loff_t lend)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 466) {
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 467) struct hstate *h = hstate_inode(inode);
b45b5bd65f668 (David Gibson 2006-03-22 00:08:55 -0800 468) struct address_space *mapping = &inode->i_data;
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 469) const pgoff_t start = lstart >> huge_page_shift(h);
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 470) const pgoff_t end = lend >> huge_page_shift(h);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 471) struct pagevec pvec;
d72dc8a25afc7 (Jan Kara 2017-09-06 16:21:18 -0700 472) pgoff_t next, index;
a43a8c39bbb49 (Kenneth W Chen 2006-06-23 02:03:15 -0700 473) int i, freed = 0;
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 474) bool truncate_op = (lend == LLONG_MAX);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 475)
8667982014d60 (Mel Gorman 2017-11-15 17:37:52 -0800 476) pagevec_init(&pvec);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 477) next = start;
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 478) while (next < end) {
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 479) /*
1817889e3b2cc (Mike Kravetz 2015-11-20 15:57:13 -0800 480) * When no more pages are found, we are done.
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 481) */
397162ffa2ed1 (Jan Kara 2017-09-06 16:21:43 -0700 482) if (!pagevec_lookup_range(&pvec, mapping, &next, end - 1))
1817889e3b2cc (Mike Kravetz 2015-11-20 15:57:13 -0800 483) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 484)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 485) for (i = 0; i < pagevec_count(&pvec); ++i) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 486) struct page *page = pvec.pages[i];
d4241a049ac00 (Miaohe Lin 2021-05-04 18:33:34 -0700 487) u32 hash = 0;
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 488)
d72dc8a25afc7 (Jan Kara 2017-09-06 16:21:18 -0700 489) index = page->index;
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 490) if (!truncate_op) {
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 491) /*
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 492) * Only need to hold the fault mutex in the
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 493) * hole punch case. This prevents races with
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 494) * page faults. Races are not possible in the
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 495) * case of truncation.
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 496) */
d4241a049ac00 (Miaohe Lin 2021-05-04 18:33:34 -0700 497) hash = hugetlb_fault_mutex_hash(mapping, index);
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 498) mutex_lock(&hugetlb_fault_mutex_table[hash]);
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 499) }
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 500)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 501) /*
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 502) * If page is mapped, it was faulted in after being
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 503) * unmapped in caller. Unmap (again) now after taking
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 504) * the fault mutex. The mutex will prevent faults
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 505) * until we finish removing the page.
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 506) *
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 507) * This race can only happen in the hole punch case.
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 508) * Getting here in a truncate operation is a bug.
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 509) */
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 510) if (unlikely(page_mapped(page))) {
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 511) BUG_ON(truncate_op);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 512)
c0d0381ade798 (Mike Kravetz 2020-04-01 21:11:05 -0700 513) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 514) i_mmap_lock_write(mapping);
c0d0381ade798 (Mike Kravetz 2020-04-01 21:11:05 -0700 515) mutex_lock(&hugetlb_fault_mutex_table[hash]);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 516) hugetlb_vmdelete_list(&mapping->i_mmap,
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 517) index * pages_per_huge_page(h),
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 518) (index + 1) * pages_per_huge_page(h));
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 519) i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 520) }
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 521)
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 522) lock_page(page);
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 523) /*
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 524) * We must free the huge page and remove from page
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 525) * cache (remove_huge_page) BEFORE removing the
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 526) * region/reserve map (hugetlb_unreserve_pages). In
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 527) * rare out of memory conditions, removal of the
72e2936c04f7d (zhong jiang 2016-10-07 17:02:01 -0700 528) * region/reserve map could fail. Correspondingly,
72e2936c04f7d (zhong jiang 2016-10-07 17:02:01 -0700 529) * the subpool and global reserve usage count can need
72e2936c04f7d (zhong jiang 2016-10-07 17:02:01 -0700 530) * to be adjusted.
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 531) */
e32905e57358f (Mike Kravetz 2021-05-22 17:42:11 -0700 532) VM_BUG_ON(HPageRestoreReserve(page));
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 533) remove_huge_page(page);
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 534) freed++;
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 535) if (!truncate_op) {
4aae8d1c051ea (Mike Kravetz 2016-01-15 16:57:40 -0800 536) if (unlikely(hugetlb_unreserve_pages(inode,
d72dc8a25afc7 (Jan Kara 2017-09-06 16:21:18 -0700 537) index, index + 1, 1)))
72e2936c04f7d (zhong jiang 2016-10-07 17:02:01 -0700 538) hugetlb_fix_reserve_counts(inode);
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 539) }
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 540)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 541) unlock_page(page);
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 542) if (!truncate_op)
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 543) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 544) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 545) huge_pagevec_release(&pvec);
1817889e3b2cc (Mike Kravetz 2015-11-20 15:57:13 -0800 546) cond_resched();
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 547) }
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 548)
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 549) if (truncate_op)
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 550) (void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 551) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 552)
2bbbda308f5ca (Al Viro 2010-06-04 19:52:12 -0400 553) static void hugetlbfs_evict_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 554) {
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 555) struct resv_map *resv_map;
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 556)
b5cec28d36f5e (Mike Kravetz 2015-09-08 15:01:41 -0700 557) remove_inode_hugepages(inode, 0, LLONG_MAX);
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 558)
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 559) /*
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 560) * Get the resv_map from the address space embedded in the inode.
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 561) * This is the address space which points to any resv_map allocated
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 562) * at inode creation time. If this is a device special inode,
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 563) * i_mapping may not point to the original address space.
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 564) */
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 565) resv_map = (struct resv_map *)(&inode->i_data)->private_data;
f27a5136f70a8 (Mike Kravetz 2019-05-13 17:22:55 -0700 566) /* Only regular and link inodes have associated reserve maps */
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 567) if (resv_map)
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 568) resv_map_release(&resv_map->refs);
dbd5768f87ff6 (Jan Kara 2012-05-03 14:48:02 +0200 569) clear_inode(inode);
149f4211afda8 (Christoph Hellwig 2005-10-29 18:16:43 -0700 570) }
149f4211afda8 (Christoph Hellwig 2005-10-29 18:16:43 -0700 571)
e5d319dedafd2 (Miaohe Lin 2021-02-24 12:10:25 -0800 572) static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 573) {
856fc29505556 (Hugh Dickins 2006-10-28 10:38:43 -0700 574) pgoff_t pgoff;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 575) struct address_space *mapping = inode->i_mapping;
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 576) struct hstate *h = hstate_inode(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 577)
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 578) BUG_ON(offset & ~huge_page_mask(h));
856fc29505556 (Hugh Dickins 2006-10-28 10:38:43 -0700 579) pgoff = offset >> PAGE_SHIFT;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 580)
83cde9e8ba95d (Davidlohr Bueso 2014-12-12 16:54:21 -0800 581) i_mmap_lock_write(mapping);
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 582) i_size_write(inode, offset);
f808c13fd3738 (Davidlohr Bueso 2017-09-08 16:15:08 -0700 583) if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
1bfad99ab4256 (Mike Kravetz 2015-09-08 15:01:38 -0700 584) hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
c86aa7bbfd556 (Mike Kravetz 2018-12-28 00:39:42 -0800 585) i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 586) remove_inode_hugepages(inode, offset, LLONG_MAX);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 587) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 588)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 589) static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 590) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 591) struct hstate *h = hstate_inode(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 592) loff_t hpage_size = huge_page_size(h);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 593) loff_t hole_start, hole_end;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 594)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 595) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 596) * For hole punch round up the beginning offset of the hole and
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 597) * round down the end.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 598) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 599) hole_start = round_up(offset, hpage_size);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 600) hole_end = round_down(offset + len, hpage_size);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 601)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 602) if (hole_end > hole_start) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 603) struct address_space *mapping = inode->i_mapping;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 604) struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 605)
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 606) inode_lock(inode);
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 607)
398c0da7364c9 (Miaohe Lin 2021-02-24 12:10:18 -0800 608) /* protected by i_rwsem */
ab3948f58ff84 (Joel Fernandes (Google) 2019-03-05 15:47:54 -0800 609) if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 610) inode_unlock(inode);
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 611) return -EPERM;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 612) }
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 613)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 614) i_mmap_lock_write(mapping);
f808c13fd3738 (Davidlohr Bueso 2017-09-08 16:15:08 -0700 615) if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 616) hugetlb_vmdelete_list(&mapping->i_mmap,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 617) hole_start >> PAGE_SHIFT,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 618) hole_end >> PAGE_SHIFT);
c86aa7bbfd556 (Mike Kravetz 2018-12-28 00:39:42 -0800 619) i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz 2019-01-08 15:23:32 -0800 620) remove_inode_hugepages(inode, hole_start, hole_end);
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 621) inode_unlock(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 622) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 623)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 624) return 0;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 625) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 626)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 627) static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 628) loff_t len)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 629) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 630) struct inode *inode = file_inode(file);
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 631) struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 632) struct address_space *mapping = inode->i_mapping;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 633) struct hstate *h = hstate_inode(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 634) struct vm_area_struct pseudo_vma;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 635) struct mm_struct *mm = current->mm;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 636) loff_t hpage_size = huge_page_size(h);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 637) unsigned long hpage_shift = huge_page_shift(h);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 638) pgoff_t start, index, end;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 639) int error;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 640) u32 hash;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 641)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 642) if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 643) return -EOPNOTSUPP;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 644)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 645) if (mode & FALLOC_FL_PUNCH_HOLE)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 646) return hugetlbfs_punch_hole(inode, offset, len);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 647)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 648) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 649) * Default preallocate case.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 650) * For this range, start is rounded down and end is rounded up
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 651) * as well as being converted to page offsets.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 652) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 653) start = offset >> hpage_shift;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 654) end = (offset + len + hpage_size - 1) >> hpage_shift;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 655)
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 656) inode_lock(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 657)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 658) /* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 659) error = inode_newsize_ok(inode, offset + len);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 660) if (error)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 661) goto out;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 662)
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 663) if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 664) error = -EPERM;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 665) goto out;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 666) }
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 667)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 668) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 669) * Initialize a pseudo vma as this is required by the huge page
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 670) * allocation routines. If NUMA is configured, use page index
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 671) * as input to create an allocation policy.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 672) */
2c4541e24c55e (Kirill A. Shutemov 2018-07-26 16:37:30 -0700 673) vma_init(&pseudo_vma, mm);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 674) pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 675) pseudo_vma.vm_file = file;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 676)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 677) for (index = start; index < end; index++) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 678) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 679) * This is supposed to be the vaddr where the page is being
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 680) * faulted in, but we have no vaddr here.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 681) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 682) struct page *page;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 683) unsigned long addr;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 684)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 685) cond_resched();
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 686)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 687) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 688) * fallocate(2) manpage permits EINTR; we may have been
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 689) * interrupted because we are using up too much memory.
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 690) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 691) if (signal_pending(current)) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 692) error = -EINTR;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 693) break;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 694) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 695)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 696) /* Set numa allocation policy based on index */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 697) hugetlb_set_vma_policy(&pseudo_vma, inode, index);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 698)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 699) /* addr is the offset within the file (zero based) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 700) addr = index * hpage_size;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 701)
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 702) /*
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 703) * fault mutex taken here, protects against fault path
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 704) * and hole punch. inode_lock previously taken protects
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 705) * against truncation.
87bf91d39bb52 (Mike Kravetz 2020-04-01 21:11:08 -0700 706) */
188b04a7d9386 (Wei Yang 2019-11-30 17:57:02 -0800 707) hash = hugetlb_fault_mutex_hash(mapping, index);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 708) mutex_lock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 709)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 710) /* See if already present in mapping to avoid alloc/free */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 711) page = find_get_page(mapping, index);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 712) if (page) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 713) put_page(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 714) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 715) hugetlb_drop_vma_policy(&pseudo_vma);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 716) continue;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 717) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 718)
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 719) /*
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 720) * Allocate page without setting the avoid_reserve argument.
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 721) * There certainly are no reserves associated with the
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 722) * pseudo_vma. However, there could be shared mappings with
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 723) * reserves for the file at the inode level. If we fallocate
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 724) * pages in these areas, we need to consume the reserves
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 725) * to keep reservation accounting consistent.
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 726) */
88ce3fef47f3f (Miaohe Lin 2021-02-24 12:10:11 -0800 727) page = alloc_huge_page(&pseudo_vma, addr, 0);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 728) hugetlb_drop_vma_policy(&pseudo_vma);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 729) if (IS_ERR(page)) {
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 730) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 731) error = PTR_ERR(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 732) goto out;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 733) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 734) clear_huge_page(page, addr, pages_per_huge_page(h));
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 735) __SetPageUptodate(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 736) error = huge_add_to_page_cache(page, mapping, index);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 737) if (unlikely(error)) {
846be08578edb (Mike Kravetz 2021-06-15 18:23:29 -0700 738) restore_reserve_on_error(h, &pseudo_vma, addr, page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 739) put_page(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 740) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 741) goto out;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 742) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 743)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 744) mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 745)
8f251a3d5ce3b (Mike Kravetz 2021-02-24 12:08:56 -0800 746) SetHPageMigratable(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 747) /*
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 748) * unlock_page because locked by add_to_page_cache()
585fc0d2871c9 (Muchun Song 2021-02-04 18:32:03 -0800 749) * put_page() due to reference from alloc_huge_page()
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 750) */
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 751) unlock_page(page);
72639e6df4128 (Nadav Amit 2017-11-29 16:11:33 -0800 752) put_page(page);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 753) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 754)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 755) if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 756) i_size_write(inode, offset + len);
078cd8279e659 (Deepa Dinamani 2016-09-14 07:48:04 -0700 757) inode->i_ctime = current_time(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 758) out:
5955102c9984f (Al Viro 2016-01-22 15:40:57 -0500 759) inode_unlock(inode);
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 760) return error;
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 761) }
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 762)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 763) static int hugetlbfs_setattr(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 764) struct dentry *dentry, struct iattr *attr)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 765) {
2b0143b5c986b (David Howells 2015-03-17 22:25:59 +0000 766) struct inode *inode = d_inode(dentry);
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 767) struct hstate *h = hstate_inode(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 768) int error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 769) unsigned int ia_valid = attr->ia_valid;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 770) struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 771)
2f221d6f7b881 (Christian Brauner 2021-01-21 14:19:26 +0100 772) error = setattr_prepare(&init_user_ns, dentry, attr);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 773) if (error)
1025774ce411f (Christoph Hellwig 2010-06-04 11:30:02 +0200 774) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 775)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 776) if (ia_valid & ATTR_SIZE) {
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 777) loff_t oldsize = inode->i_size;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 778) loff_t newsize = attr->ia_size;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 779)
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 780) if (newsize & ~huge_page_mask(h))
1025774ce411f (Christoph Hellwig 2010-06-04 11:30:02 +0200 781) return -EINVAL;
398c0da7364c9 (Miaohe Lin 2021-02-24 12:10:18 -0800 782) /* protected by i_rwsem */
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 783) if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 784) (newsize > oldsize && (info->seals & F_SEAL_GROW)))
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 785) return -EPERM;
e5d319dedafd2 (Miaohe Lin 2021-02-24 12:10:25 -0800 786) hugetlb_vmtruncate(inode, newsize);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 787) }
1025774ce411f (Christoph Hellwig 2010-06-04 11:30:02 +0200 788)
2f221d6f7b881 (Christian Brauner 2021-01-21 14:19:26 +0100 789) setattr_copy(&init_user_ns, inode, attr);
1025774ce411f (Christoph Hellwig 2010-06-04 11:30:02 +0200 790) mark_inode_dirty(inode);
1025774ce411f (Christoph Hellwig 2010-06-04 11:30:02 +0200 791) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 792) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 793)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 794) static struct inode *hugetlbfs_get_root(struct super_block *sb,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 795) struct hugetlbfs_fs_context *ctx)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 796) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 797) struct inode *inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 798)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 799) inode = new_inode(sb);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 800) if (inode) {
85fe4025c616a (Christoph Hellwig 2010-10-23 11:19:54 -0400 801) inode->i_ino = get_next_ino();
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 802) inode->i_mode = S_IFDIR | ctx->mode;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 803) inode->i_uid = ctx->uid;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 804) inode->i_gid = ctx->gid;
078cd8279e659 (Deepa Dinamani 2016-09-14 07:48:04 -0700 805) inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 806) inode->i_op = &hugetlbfs_dir_inode_operations;
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 807) inode->i_fop = &simple_dir_operations;
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 808) /* directory inodes start off with i_nlink == 2 (for "." entry) */
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 809) inc_nlink(inode);
65ed76010dfed (Aneesh Kumar K.V 2012-04-25 16:01:50 -0700 810) lockdep_annotate_inode_mutex_key(inode);
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 811) }
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 812) return inode;
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 813) }
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 814)
b610ded719186 (Michal Hocko 2013-08-13 16:00:55 -0700 815) /*
c8c06efa8b552 (Davidlohr Bueso 2014-12-12 16:54:24 -0800 816) * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
b610ded719186 (Michal Hocko 2013-08-13 16:00:55 -0700 817) * be taken from reclaim -- unlike regular filesystems. This needs an
88f306b68cbb3 (Kirill A. Shutemov 2016-01-15 16:57:31 -0800 818) * annotation because huge_pmd_share() does an allocation under hugetlb's
c8c06efa8b552 (Davidlohr Bueso 2014-12-12 16:54:24 -0800 819) * i_mmap_rwsem.
b610ded719186 (Michal Hocko 2013-08-13 16:00:55 -0700 820) */
c8c06efa8b552 (Davidlohr Bueso 2014-12-12 16:54:24 -0800 821) static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
b610ded719186 (Michal Hocko 2013-08-13 16:00:55 -0700 822)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 823) static struct inode *hugetlbfs_get_inode(struct super_block *sb,
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 824) struct inode *dir,
18df22524202e (Al Viro 2011-07-24 23:17:40 -0400 825) umode_t mode, dev_t dev)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 826) {
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 827) struct inode *inode;
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 828) struct resv_map *resv_map = NULL;
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 829)
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 830) /*
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 831) * Reserve maps are only needed for inodes that can have associated
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 832) * page allocations.
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 833) */
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 834) if (S_ISREG(mode) || S_ISLNK(mode)) {
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 835) resv_map = resv_map_alloc();
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 836) if (!resv_map)
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 837) return NULL;
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 838) }
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 839)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 840) inode = new_inode(sb);
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 841) if (inode) {
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 842) struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 843)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 844) inode->i_ino = get_next_ino();
21cb47be6fb9e (Christian Brauner 2021-01-21 14:19:25 +0100 845) inode_init_owner(&init_user_ns, inode, dir, mode);
c8c06efa8b552 (Davidlohr Bueso 2014-12-12 16:54:24 -0800 846) lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
c8c06efa8b552 (Davidlohr Bueso 2014-12-12 16:54:24 -0800 847) &hugetlbfs_i_mmap_rwsem_key);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 848) inode->i_mapping->a_ops = &hugetlbfs_aops;
078cd8279e659 (Deepa Dinamani 2016-09-14 07:48:04 -0700 849) inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 850) inode->i_mapping->private_data = resv_map;
ff62a34210441 (Marc-André Lureau 2018-01-31 16:19:25 -0800 851) info->seals = F_SEAL_SEAL;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 852) switch (mode & S_IFMT) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 853) default:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 854) init_special_inode(inode, mode, dev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 855) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 856) case S_IFREG:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 857) inode->i_op = &hugetlbfs_inode_operations;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 858) inode->i_fop = &hugetlbfs_file_operations;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 859) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 860) case S_IFDIR:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 861) inode->i_op = &hugetlbfs_dir_inode_operations;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 862) inode->i_fop = &simple_dir_operations;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 863)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 864) /* directory inodes start off with i_nlink == 2 (for "." entry) */
d8c76e6f45c11 (Dave Hansen 2006-09-30 23:29:04 -0700 865) inc_nlink(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 866) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 867) case S_IFLNK:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 868) inode->i_op = &page_symlink_inode_operations;
21fc61c73c390 (Al Viro 2015-11-17 01:07:57 -0500 869) inode_nohighmem(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 870) break;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 871) }
e096d0c7e2e4e (Josh Boyer 2011-08-25 07:48:12 -0400 872) lockdep_annotate_inode_mutex_key(inode);
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 873) } else {
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 874) if (resv_map)
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 875) kref_put(&resv_map->refs, resv_map_release);
58b6e5e8f1add (Mike Kravetz 2019-04-05 18:39:06 -0700 876) }
9119a41e9091f (Joonsoo Kim 2014-04-03 14:47:25 -0700 877)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 878) return inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 879) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 880)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 881) /*
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 882) * File creation. Allocate an inode, and we're done..
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 883) */
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 884) static int do_hugetlbfs_mknod(struct inode *dir,
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 885) struct dentry *dentry,
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 886) umode_t mode,
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 887) dev_t dev,
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 888) bool tmpfile)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 889) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 890) struct inode *inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 891) int error = -ENOSPC;
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 892)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 893) inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 894) if (inode) {
078cd8279e659 (Deepa Dinamani 2016-09-14 07:48:04 -0700 895) dir->i_ctime = dir->i_mtime = current_time(dir);
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 896) if (tmpfile) {
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 897) d_tmpfile(dentry, inode);
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 898) } else {
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 899) d_instantiate(dentry, inode);
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 900) dget(dentry);/* Extra count - pin the dentry in core */
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 901) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 902) error = 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 903) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 904) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 905) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 906)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 907) static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 908) struct dentry *dentry, umode_t mode, dev_t dev)
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 909) {
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 910) return do_hugetlbfs_mknod(dir, dentry, mode, dev, false);
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 911) }
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 912)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 913) static int hugetlbfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 914) struct dentry *dentry, umode_t mode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 915) {
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 916) int retval = hugetlbfs_mknod(&init_user_ns, dir, dentry,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 917) mode | S_IFDIR, 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 918) if (!retval)
d8c76e6f45c11 (Dave Hansen 2006-09-30 23:29:04 -0700 919) inc_nlink(dir);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 920) return retval;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 921) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 922)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 923) static int hugetlbfs_create(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 924) struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 925) umode_t mode, bool excl)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 926) {
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 927) return hugetlbfs_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 928) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 929)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 930) static int hugetlbfs_tmpfile(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 931) struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 932) umode_t mode)
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 933) {
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 934) return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true);
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 935) }
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 936)
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 937) static int hugetlbfs_symlink(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 938) struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner 2021-01-21 14:19:43 +0100 939) const char *symname)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 940) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 941) struct inode *inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 942) int error = -ENOSPC;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 943)
7d54fa6472609 (Al Viro 2011-07-24 20:20:48 -0400 944) inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 945) if (inode) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 946) int l = strlen(symname)+1;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 947) error = page_symlink(inode, symname, l);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 948) if (!error) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 949) d_instantiate(dentry, inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 950) dget(dentry);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 951) } else
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 952) iput(inode);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 953) }
078cd8279e659 (Deepa Dinamani 2016-09-14 07:48:04 -0700 954) dir->i_ctime = dir->i_mtime = current_time(dir);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 955)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 956) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 957) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 958)
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 959) static int hugetlbfs_migrate_page(struct address_space *mapping,
b969c4ab9f182 (Mel Gorman 2012-01-12 17:19:34 -0800 960) struct page *newpage, struct page *page,
a6bc32b899223 (Mel Gorman 2012-01-12 17:19:43 -0800 961) enum migrate_mode mode)
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 962) {
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 963) int rc;
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 964)
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 965) rc = migrate_huge_page_move_mapping(mapping, newpage, page);
78bd52097d042 (Rafael Aquini 2012-12-11 16:02:31 -0800 966) if (rc != MIGRATEPAGE_SUCCESS)
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 967) return rc;
cb6acd01e2e43 (Mike Kravetz 2019-02-28 16:22:02 -0800 968)
d6995da311221 (Mike Kravetz 2021-02-24 12:08:51 -0800 969) if (hugetlb_page_subpool(page)) {
d6995da311221 (Mike Kravetz 2021-02-24 12:08:51 -0800 970) hugetlb_set_page_subpool(newpage, hugetlb_page_subpool(page));
d6995da311221 (Mike Kravetz 2021-02-24 12:08:51 -0800 971) hugetlb_set_page_subpool(page, NULL);
cb6acd01e2e43 (Mike Kravetz 2019-02-28 16:22:02 -0800 972) }
cb6acd01e2e43 (Mike Kravetz 2019-02-28 16:22:02 -0800 973)
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 974) if (mode != MIGRATE_SYNC_NO_COPY)
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 975) migrate_page_copy(newpage, page);
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 976) else
2916ecc0f9d43 (Jérôme Glisse 2017-09-08 16:12:06 -0700 977) migrate_page_states(newpage, page);
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 978)
78bd52097d042 (Rafael Aquini 2012-12-11 16:02:31 -0800 979) return MIGRATEPAGE_SUCCESS;
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 980) }
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 981)
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 982) static int hugetlbfs_error_remove_page(struct address_space *mapping,
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 983) struct page *page)
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 984) {
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 985) struct inode *inode = mapping->host;
ab615a5b87929 (Mike Kravetz 2017-11-02 15:59:41 -0700 986) pgoff_t index = page->index;
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 987)
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 988) remove_huge_page(page);
ab615a5b87929 (Mike Kravetz 2017-11-02 15:59:41 -0700 989) if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
ab615a5b87929 (Mike Kravetz 2017-11-02 15:59:41 -0700 990) hugetlb_fix_reserve_counts(inode);
ab615a5b87929 (Mike Kravetz 2017-11-02 15:59:41 -0700 991)
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 992) return 0;
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 993) }
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 994)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 995) /*
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 996) * Display the mount options in /proc/mounts.
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 997) */
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 998) static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 999) {
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1000) struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1001) struct hugepage_subpool *spool = sbinfo->spool;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1002) unsigned long hpage_size = huge_page_size(sbinfo->hstate);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1003) unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1004) char mod;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1005)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1006) if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1007) seq_printf(m, ",uid=%u",
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1008) from_kuid_munged(&init_user_ns, sbinfo->uid));
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1009) if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1010) seq_printf(m, ",gid=%u",
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1011) from_kgid_munged(&init_user_ns, sbinfo->gid));
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1012) if (sbinfo->mode != 0755)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1013) seq_printf(m, ",mode=%o", sbinfo->mode);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1014) if (sbinfo->max_inodes != -1)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1015) seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1016)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1017) hpage_size /= 1024;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1018) mod = 'K';
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1019) if (hpage_size >= 1024) {
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1020) hpage_size /= 1024;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1021) mod = 'M';
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1022) }
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1023) seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1024) if (spool) {
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1025) if (spool->max_hpages != -1)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1026) seq_printf(m, ",size=%llu",
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1027) (unsigned long long)spool->max_hpages << hpage_shift);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1028) if (spool->min_hpages != -1)
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1029) seq_printf(m, ",min_size=%llu",
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1030) (unsigned long long)spool->min_hpages << hpage_shift);
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1031) }
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1032) return 0;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1033) }
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1034)
726c334223180 (David Howells 2006-06-23 02:02:58 -0700 1035) static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1036) {
726c334223180 (David Howells 2006-06-23 02:02:58 -0700 1037) struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
2b0143b5c986b (David Howells 2015-03-17 22:25:59 +0000 1038) struct hstate *h = hstate_inode(d_inode(dentry));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1039)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1040) buf->f_type = HUGETLBFS_MAGIC;
a5516438959d9 (Andi Kleen 2008-07-23 21:27:41 -0700 1041) buf->f_bsize = huge_page_size(h);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1042) if (sbinfo) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1043) spin_lock(&sbinfo->stat_lock);
74a8a65c52618 (David Gibson 2005-11-21 21:32:24 -0800 1044) /* If no limits set, just report 0 for max/free/used
74a8a65c52618 (David Gibson 2005-11-21 21:32:24 -0800 1045) * blocks, like simple_statfs() */
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1046) if (sbinfo->spool) {
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1047) long free_pages;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1048)
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1049) spin_lock(&sbinfo->spool->lock);
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1050) buf->f_blocks = sbinfo->spool->max_hpages;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1051) free_pages = sbinfo->spool->max_hpages
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1052) - sbinfo->spool->used_hpages;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1053) buf->f_bavail = buf->f_bfree = free_pages;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1054) spin_unlock(&sbinfo->spool->lock);
74a8a65c52618 (David Gibson 2005-11-21 21:32:24 -0800 1055) buf->f_files = sbinfo->max_inodes;
74a8a65c52618 (David Gibson 2005-11-21 21:32:24 -0800 1056) buf->f_ffree = sbinfo->free_inodes;
74a8a65c52618 (David Gibson 2005-11-21 21:32:24 -0800 1057) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1058) spin_unlock(&sbinfo->stat_lock);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1059) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1060) buf->f_namelen = NAME_MAX;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1061) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1062) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1063)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1064) static void hugetlbfs_put_super(struct super_block *sb)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1065) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1066) struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1067)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1068) if (sbi) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1069) sb->s_fs_info = NULL;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1070)
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1071) if (sbi->spool)
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1072) hugepage_put_subpool(sbi->spool);
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1073)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1074) kfree(sbi);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1075) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1076) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1077)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1078) static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1079) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1080) if (sbinfo->free_inodes >= 0) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1081) spin_lock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1082) if (unlikely(!sbinfo->free_inodes)) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1083) spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1084) return 0;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1085) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1086) sbinfo->free_inodes--;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1087) spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1088) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1089)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1090) return 1;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1091) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1092)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1093) static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1094) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1095) if (sbinfo->free_inodes >= 0) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1096) spin_lock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1097) sbinfo->free_inodes++;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1098) spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1099) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1100) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1101)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1102)
e18b890bb0881 (Christoph Lameter 2006-12-06 20:33:20 -0800 1103) static struct kmem_cache *hugetlbfs_inode_cachep;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1104)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1105) static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1106) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1107) struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1108) struct hugetlbfs_inode_info *p;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1109)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1110) if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1111) return NULL;
e94b1766097d5 (Christoph Lameter 2006-12-06 20:33:17 -0800 1112) p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1113) if (unlikely(!p)) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1114) hugetlbfs_inc_free_inodes(sbinfo);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1115) return NULL;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1116) }
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1117)
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1118) /*
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1119) * Any time after allocation, hugetlbfs_destroy_inode can be called
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1120) * for the inode. mpol_free_shared_policy is unconditionally called
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1121) * as part of hugetlbfs_destroy_inode. So, initialize policy here
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1122) * in case of a quick call to destroy.
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1123) *
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1124) * Note that the policy is initialized even if we are creating a
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1125) * private inode. This simplifies hugetlbfs_destroy_inode.
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1126) */
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1127) mpol_shared_policy_init(&p->policy, NULL);
4742a35d9de74 (Mike Kravetz 2017-03-31 15:12:01 -0700 1128)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1129) return &p->vfs_inode;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1130) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1131)
b62de32257970 (Al Viro 2019-04-15 23:16:38 -0400 1132) static void hugetlbfs_free_inode(struct inode *inode)
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 1133) {
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 1134) kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 1135) }
fa0d7e3de6d6f (Nicholas Piggin 2011-01-07 17:49:49 +1100 1136)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1137) static void hugetlbfs_destroy_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1138) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1139) hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1140) mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1141) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1142)
f5e54d6e53a20 (Christoph Hellwig 2006-06-28 04:26:44 -0700 1143) static const struct address_space_operations hugetlbfs_aops = {
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 1144) .write_begin = hugetlbfs_write_begin,
800d15a53e7d1 (Nicholas Piggin 2007-10-16 01:25:03 -0700 1145) .write_end = hugetlbfs_write_end,
a4fa34cdcd182 (Mike Kravetz 2021-02-24 12:09:58 -0800 1146) .set_page_dirty = __set_page_dirty_no_writeback,
290408d4a2500 (Naoya Horiguchi 2010-09-08 10:19:35 +0900 1147) .migratepage = hugetlbfs_migrate_page,
78bb920344b8a (Naoya Horiguchi 2017-07-10 15:47:50 -0700 1148) .error_remove_page = hugetlbfs_error_remove_page,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1149) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1150)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1151)
51cc50685a427 (Alexey Dobriyan 2008-07-25 19:45:34 -0700 1152) static void init_once(void *foo)
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1153) {
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1154) struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1155)
a35afb830f8d7 (Christoph Lameter 2007-05-16 22:10:57 -0700 1156) inode_init_once(&ei->vfs_inode);
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1157) }
96527980d4cb8 (Christoph Hellwig 2005-10-29 18:16:42 -0700 1158)
4b6f5d20b04dc (Arjan van de Ven 2006-03-28 01:56:42 -0800 1159) const struct file_operations hugetlbfs_file_operations = {
34d0640e26892 (Al Viro 2015-04-03 11:31:35 -0400 1160) .read_iter = hugetlbfs_read_iter,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1161) .mmap = hugetlbfs_file_mmap,
1b061d9247f71 (Christoph Hellwig 2010-05-26 17:53:41 +0200 1162) .fsync = noop_fsync,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1163) .get_unmapped_area = hugetlb_get_unmapped_area,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 1164) .llseek = default_llseek,
70c3547e36f5c (Mike Kravetz 2015-09-08 15:01:54 -0700 1165) .fallocate = hugetlbfs_fallocate,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1166) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1167)
92e1d5be91a0e (Arjan van de Ven 2007-02-12 00:55:39 -0800 1168) static const struct inode_operations hugetlbfs_dir_inode_operations = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1169) .create = hugetlbfs_create,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1170) .lookup = simple_lookup,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1171) .link = simple_link,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1172) .unlink = simple_unlink,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1173) .symlink = hugetlbfs_symlink,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1174) .mkdir = hugetlbfs_mkdir,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1175) .rmdir = simple_rmdir,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1176) .mknod = hugetlbfs_mknod,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1177) .rename = simple_rename,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1178) .setattr = hugetlbfs_setattr,
1ab5b82f540b3 (Piotr Sarna 2019-11-30 17:56:43 -0800 1179) .tmpfile = hugetlbfs_tmpfile,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1180) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1181)
92e1d5be91a0e (Arjan van de Ven 2007-02-12 00:55:39 -0800 1182) static const struct inode_operations hugetlbfs_inode_operations = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1183) .setattr = hugetlbfs_setattr,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1184) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1185)
ee9b6d61a2a43 (Josef 'Jeff' Sipek 2007-02-12 00:55:41 -0800 1186) static const struct super_operations hugetlbfs_ops = {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1187) .alloc_inode = hugetlbfs_alloc_inode,
b62de32257970 (Al Viro 2019-04-15 23:16:38 -0400 1188) .free_inode = hugetlbfs_free_inode,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1189) .destroy_inode = hugetlbfs_destroy_inode,
2bbbda308f5ca (Al Viro 2010-06-04 19:52:12 -0400 1190) .evict_inode = hugetlbfs_evict_inode,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1191) .statfs = hugetlbfs_statfs,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1192) .put_super = hugetlbfs_put_super,
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1193) .show_options = hugetlbfs_show_options,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1194) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1195)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1196) /*
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1197) * Convert size option passed from command line to number of huge pages
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1198) * in the pool specified by hstate. Size option could be in bytes
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1199) * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1200) */
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1201) static long
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1202) hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1203) enum hugetlbfs_size_type val_type)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1204) {
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1205) if (val_type == NO_SIZE)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1206) return -1;
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1207)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1208) if (val_type == SIZE_PERCENT) {
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1209) size_opt <<= huge_page_shift(h);
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1210) size_opt *= h->max_huge_pages;
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1211) do_div(size_opt, 100);
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1212) }
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1213)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1214) size_opt >>= huge_page_shift(h);
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1215) return size_opt;
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1216) }
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1217)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1218) /*
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1219) * Parse one mount parameter.
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1220) */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1221) static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1222) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1223) struct hugetlbfs_fs_context *ctx = fc->fs_private;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1224) struct fs_parse_result result;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1225) char *rest;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1226) unsigned long ps;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1227) int opt;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1228)
d7167b149943e (Al Viro 2019-09-07 07:23:15 -0400 1229) opt = fs_parse(fc, hugetlb_fs_parameters, param, &result);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1230) if (opt < 0)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1231) return opt;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1232)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1233) switch (opt) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1234) case Opt_uid:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1235) ctx->uid = make_kuid(current_user_ns(), result.uint_32);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1236) if (!uid_valid(ctx->uid))
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1237) goto bad_val;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1238) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1239)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1240) case Opt_gid:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1241) ctx->gid = make_kgid(current_user_ns(), result.uint_32);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1242) if (!gid_valid(ctx->gid))
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1243) goto bad_val;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1244) return 0;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1245)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1246) case Opt_mode:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1247) ctx->mode = result.uint_32 & 01777U;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1248) return 0;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1249)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1250) case Opt_size:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1251) /* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1252) if (!isdigit(param->string[0]))
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1253) goto bad_val;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1254) ctx->max_size_opt = memparse(param->string, &rest);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1255) ctx->max_val_type = SIZE_STD;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1256) if (*rest == '%')
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1257) ctx->max_val_type = SIZE_PERCENT;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1258) return 0;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1259)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1260) case Opt_nr_inodes:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1261) /* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1262) if (!isdigit(param->string[0]))
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1263) goto bad_val;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1264) ctx->nr_inodes = memparse(param->string, &rest);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1265) return 0;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1266)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1267) case Opt_pagesize:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1268) ps = memparse(param->string, &rest);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1269) ctx->hstate = size_to_hstate(ps);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1270) if (!ctx->hstate) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1271) pr_err("Unsupported page size %lu MB\n", ps >> 20);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1272) return -EINVAL;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1273) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1274) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1275)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1276) case Opt_min_size:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1277) /* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1278) if (!isdigit(param->string[0]))
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1279) goto bad_val;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1280) ctx->min_size_opt = memparse(param->string, &rest);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1281) ctx->min_val_type = SIZE_STD;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1282) if (*rest == '%')
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1283) ctx->min_val_type = SIZE_PERCENT;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1284) return 0;
e73a75fa7f062 (Randy Dunlap 2007-07-15 23:40:52 -0700 1285)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1286) default:
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1287) return -EINVAL;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1288) }
a137e1cc6d6e7 (Andi Kleen 2008-07-23 21:27:43 -0700 1289)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1290) bad_val:
b5db30cfb9793 (Al Viro 2019-12-21 21:34:06 -0500 1291) return invalfc(fc, "Bad value '%s' for mount option '%s'\n",
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1292) param->string, param->key);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1293) }
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1294)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1295) /*
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1296) * Validate the parsed options.
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1297) */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1298) static int hugetlbfs_validate(struct fs_context *fc)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1299) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1300) struct hugetlbfs_fs_context *ctx = fc->fs_private;
a137e1cc6d6e7 (Andi Kleen 2008-07-23 21:27:43 -0700 1301)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1302) /*
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1303) * Use huge page pool size (in hstate) to convert the size
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1304) * options to number of huge pages. If NO_SIZE, -1 is returned.
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1305) */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1306) ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1307) ctx->max_size_opt,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1308) ctx->max_val_type);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1309) ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1310) ctx->min_size_opt,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1311) ctx->min_val_type);
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1312)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1313) /*
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1314) * If max_size was specified, then min_size must be smaller
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1315) */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1316) if (ctx->max_val_type > NO_SIZE &&
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1317) ctx->min_hpages > ctx->max_hpages) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1318) pr_err("Minimum size can not be greater than maximum size\n");
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1319) return -EINVAL;
a137e1cc6d6e7 (Andi Kleen 2008-07-23 21:27:43 -0700 1320) }
a137e1cc6d6e7 (Andi Kleen 2008-07-23 21:27:43 -0700 1321)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1322) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1323) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1324)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1325) static int
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1326) hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1327) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1328) struct hugetlbfs_fs_context *ctx = fc->fs_private;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1329) struct hugetlbfs_sb_info *sbinfo;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1330)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1331) sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1332) if (!sbinfo)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1333) return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1334) sb->s_fs_info = sbinfo;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1335) spin_lock_init(&sbinfo->stat_lock);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1336) sbinfo->hstate = ctx->hstate;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1337) sbinfo->max_inodes = ctx->nr_inodes;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1338) sbinfo->free_inodes = ctx->nr_inodes;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1339) sbinfo->spool = NULL;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1340) sbinfo->uid = ctx->uid;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1341) sbinfo->gid = ctx->gid;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1342) sbinfo->mode = ctx->mode;
4a25220d4e43b (David Howells 2017-07-05 16:24:18 +0100 1343)
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1344) /*
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1345) * Allocate and initialize subpool if maximum or minimum size is
1935ebd3cf6c4 (Miaohe Lin 2021-02-24 12:10:21 -0800 1346) * specified. Any needed reservations (for minimum size) are taken
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1347) * taken when the subpool is created.
7ca02d0ae586f (Mike Kravetz 2015-04-15 16:13:42 -0700 1348) */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1349) if (ctx->max_hpages != -1 || ctx->min_hpages != -1) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1350) sbinfo->spool = hugepage_new_subpool(ctx->hstate,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1351) ctx->max_hpages,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1352) ctx->min_hpages);
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1353) if (!sbinfo->spool)
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1354) goto out_free;
90481622d7571 (David Gibson 2012-03-21 16:34:12 -0700 1355) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1356) sb->s_maxbytes = MAX_LFS_FILESIZE;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1357) sb->s_blocksize = huge_page_size(ctx->hstate);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1358) sb->s_blocksize_bits = huge_page_shift(ctx->hstate);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1359) sb->s_magic = HUGETLBFS_MAGIC;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1360) sb->s_op = &hugetlbfs_ops;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1361) sb->s_time_gran = 1;
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1362)
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1363) /*
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1364) * Due to the special and limited functionality of hugetlbfs, it does
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1365) * not work well as a stacking filesystem.
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1366) */
15568299b7d99 (Mike Kravetz 2020-08-11 18:31:35 -0700 1367) sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1368) sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx));
48fde701aff66 (Al Viro 2012-01-08 22:15:13 -0500 1369) if (!sb->s_root)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1370) goto out_free;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1371) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1372) out_free:
6e6870d4fd19e (Fabian Frederick 2014-06-04 16:10:40 -0700 1373) kfree(sbinfo->spool);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1374) kfree(sbinfo);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1375) return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1376) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1377)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1378) static int hugetlbfs_get_tree(struct fs_context *fc)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1379) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1380) int err = hugetlbfs_validate(fc);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1381) if (err)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1382) return err;
2ac295d4f0c09 (Al Viro 2019-06-01 20:48:55 -0400 1383) return get_tree_nodev(fc, hugetlbfs_fill_super);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1384) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1385)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1386) static void hugetlbfs_fs_context_free(struct fs_context *fc)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1387) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1388) kfree(fc->fs_private);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1389) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1390)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1391) static const struct fs_context_operations hugetlbfs_fs_context_ops = {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1392) .free = hugetlbfs_fs_context_free,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1393) .parse_param = hugetlbfs_parse_param,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1394) .get_tree = hugetlbfs_get_tree,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1395) };
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1396)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1397) static int hugetlbfs_init_fs_context(struct fs_context *fc)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1398) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1399) struct hugetlbfs_fs_context *ctx;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1400)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1401) ctx = kzalloc(sizeof(struct hugetlbfs_fs_context), GFP_KERNEL);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1402) if (!ctx)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1403) return -ENOMEM;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1404)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1405) ctx->max_hpages = -1; /* No limit on size by default */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1406) ctx->nr_inodes = -1; /* No limit on number of inodes by default */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1407) ctx->uid = current_fsuid();
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1408) ctx->gid = current_fsgid();
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1409) ctx->mode = 0755;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1410) ctx->hstate = &default_hstate;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1411) ctx->min_hpages = -1; /* No default minimum size */
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1412) ctx->max_val_type = NO_SIZE;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1413) ctx->min_val_type = NO_SIZE;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1414) fc->fs_private = ctx;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1415) fc->ops = &hugetlbfs_fs_context_ops;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1416) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1417) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1418)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1419) static struct file_system_type hugetlbfs_fs_type = {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1420) .name = "hugetlbfs",
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1421) .init_fs_context = hugetlbfs_init_fs_context,
d7167b149943e (Al Viro 2019-09-07 07:23:15 -0400 1422) .parameters = hugetlb_fs_parameters,
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1423) .kill_sb = kill_litter_super,
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1424) };
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1425)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1426) static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1427)
ef1ff6b8c0895 (From: Mel Gorman 2009-09-23 15:56:05 -0700 1428) static int can_do_hugetlb_shm(void)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1429) {
a0eb3a05a8cbe (Eric W. Biederman 2012-02-07 16:19:25 -0800 1430) kgid_t shm_group;
a0eb3a05a8cbe (Eric W. Biederman 2012-02-07 16:19:25 -0800 1431) shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
a0eb3a05a8cbe (Eric W. Biederman 2012-02-07 16:19:25 -0800 1432) return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1433) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1434)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1435) static int get_hstate_idx(int page_size_log)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1436) {
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1437) struct hstate *h = hstate_sizelog(page_size_log);
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1438)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1439) if (!h)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1440) return -1;
04adbc3f7bff4 (Miaohe Lin 2021-05-04 18:33:22 -0700 1441) return hstate_index(h);
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1442) }
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1443)
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1444) /*
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1445) * Note that size should be aligned to proper hugepage size in caller side,
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1446) * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1447) */
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1448) struct file *hugetlb_file_setup(const char *name, size_t size,
af73e4d9506d3 (Naoya Horiguchi 2013-05-07 16:18:13 -0700 1449) vm_flags_t acctflag, struct user_struct **user,
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1450) int creat_flags, int page_size_log)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1451) {
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1452) struct inode *inode;
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1453) struct vfsmount *mnt;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1454) int hstate_idx;
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1455) struct file *file;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1456)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1457) hstate_idx = get_hstate_idx(page_size_log);
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1458) if (hstate_idx < 0)
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1459) return ERR_PTR(-ENODEV);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1460)
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1461) *user = NULL;
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1462) mnt = hugetlbfs_vfsmount[hstate_idx];
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1463) if (!mnt)
5bc98594d5967 (Akinobu Mita 2007-05-06 14:50:18 -0700 1464) return ERR_PTR(-ENOENT);
5bc98594d5967 (Akinobu Mita 2007-05-06 14:50:18 -0700 1465)
ef1ff6b8c0895 (From: Mel Gorman 2009-09-23 15:56:05 -0700 1466) if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1467) *user = current_user();
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1468) if (user_shm_lock(size, *user)) {
21a3c273f88c9 (David Rientjes 2012-03-21 16:34:13 -0700 1469) task_lock(current);
9b857d26d08f0 (Andrew Morton 2014-06-04 16:07:21 -0700 1470) pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
21a3c273f88c9 (David Rientjes 2012-03-21 16:34:13 -0700 1471) current->comm, current->pid);
21a3c273f88c9 (David Rientjes 2012-03-21 16:34:13 -0700 1472) task_unlock(current);
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1473) } else {
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1474) *user = NULL;
2584e517320bd (Ravikiran G Thirumalai 2009-03-31 15:21:26 -0700 1475) return ERR_PTR(-EPERM);
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1476) }
2584e517320bd (Ravikiran G Thirumalai 2009-03-31 15:21:26 -0700 1477) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1478)
39b6525274574 (Anatol Pomozov 2012-09-12 20:11:55 -0700 1479) file = ERR_PTR(-ENOSPC);
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1480) inode = hugetlbfs_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1481) if (!inode)
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1482) goto out;
e1832f2923ec9 (Stephen Smalley 2015-08-06 15:46:55 -0700 1483) if (creat_flags == HUGETLB_SHMFS_INODE)
e1832f2923ec9 (Stephen Smalley 2015-08-06 15:46:55 -0700 1484) inode->i_flags |= S_PRIVATE;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1485)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1486) inode->i_size = size;
6d6b77f163c7e (Miklos Szeredi 2011-10-28 14:13:28 +0200 1487) clear_nlink(inode);
ce8d2cdf3d2b7 (Dave Hansen 2007-10-16 23:31:13 -0700 1488)
33b8f84a4ee78 (Mike Kravetz 2021-02-24 12:09:54 -0800 1489) if (!hugetlb_reserve_pages(inode, 0,
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1490) size >> huge_page_shift(hstate_inode(inode)), NULL,
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1491) acctflag))
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1492) file = ERR_PTR(-ENOMEM);
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1493) else
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1494) file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1495) &hugetlbfs_file_operations);
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1496) if (!IS_ERR(file))
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1497) return file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1498)
b45b5bd65f668 (David Gibson 2006-03-22 00:08:55 -0800 1499) iput(inode);
e68375c850b0d (Al Viro 2018-06-09 09:50:46 -0400 1500) out:
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1501) if (*user) {
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1502) user_shm_unlock(size, *user);
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1503) *user = NULL;
353d5c30c6665 (Hugh Dickins 2009-08-24 16:30:28 +0100 1504) }
39b6525274574 (Anatol Pomozov 2012-09-12 20:11:55 -0700 1505) return file;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1506) }
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1507)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1508) static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1509) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1510) struct fs_context *fc;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1511) struct vfsmount *mnt;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1512)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1513) fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1514) if (IS_ERR(fc)) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1515) mnt = ERR_CAST(fc);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1516) } else {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1517) struct hugetlbfs_fs_context *ctx = fc->fs_private;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1518) ctx->hstate = h;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1519) mnt = fc_mount(fc);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1520) put_fs_context(fc);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1521) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1522) if (IS_ERR(mnt))
a25fddced835a (Miaohe Lin 2021-02-24 12:10:14 -0800 1523) pr_err("Cannot mount internal hugetlbfs for page size %luK",
a25fddced835a (Miaohe Lin 2021-02-24 12:10:14 -0800 1524) huge_page_size(h) >> 10);
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1525) return mnt;
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1526) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1527)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1528) static int __init init_hugetlbfs_fs(void)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1529) {
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1530) struct vfsmount *mnt;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1531) struct hstate *h;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1532) int error;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1533) int i;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1534)
457c1b27ed56e (Nishanth Aravamudan 2014-05-06 12:50:00 -0700 1535) if (!hugepages_supported()) {
9b857d26d08f0 (Andrew Morton 2014-06-04 16:07:21 -0700 1536) pr_info("disabling because there are no supported hugepage sizes\n");
457c1b27ed56e (Nishanth Aravamudan 2014-05-06 12:50:00 -0700 1537) return -ENOTSUPP;
457c1b27ed56e (Nishanth Aravamudan 2014-05-06 12:50:00 -0700 1538) }
457c1b27ed56e (Nishanth Aravamudan 2014-05-06 12:50:00 -0700 1539)
d1d5e05ffdc11 (Hillf Danton 2012-03-21 16:34:15 -0700 1540) error = -ENOMEM;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1541) hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1542) sizeof(struct hugetlbfs_inode_info),
5d097056c9a01 (Vladimir Davydov 2016-01-14 15:18:21 -0800 1543) 0, SLAB_ACCOUNT, init_once);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1544) if (hugetlbfs_inode_cachep == NULL)
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1545) goto out;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1546)
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1547) error = register_filesystem(&hugetlbfs_fs_type);
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1548) if (error)
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1549) goto out_free;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1550)
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1551) /* default hstate mount is required */
3b2275a8d83a2 (Miaohe Lin 2021-02-24 12:10:04 -0800 1552) mnt = mount_one_hugetlbfs(&default_hstate);
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1553) if (IS_ERR(mnt)) {
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1554) error = PTR_ERR(mnt);
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1555) goto out_unreg;
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1556) }
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1557) hugetlbfs_vfsmount[default_hstate_idx] = mnt;
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1558)
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1559) /* other hstates are optional */
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1560) i = 0;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1561) for_each_hstate(h) {
15f0ec941f4f9 (Jan Stancek 2020-01-03 18:37:18 +0100 1562) if (i == default_hstate_idx) {
15f0ec941f4f9 (Jan Stancek 2020-01-03 18:37:18 +0100 1563) i++;
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1564) continue;
15f0ec941f4f9 (Jan Stancek 2020-01-03 18:37:18 +0100 1565) }
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1566)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1567) mnt = mount_one_hugetlbfs(h);
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1568) if (IS_ERR(mnt))
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1569) hugetlbfs_vfsmount[i] = NULL;
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1570) else
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1571) hugetlbfs_vfsmount[i] = mnt;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1572) i++;
42d7395feb56f (Andi Kleen 2012-12-11 16:01:34 -0800 1573) }
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1574)
32021982a324d (David Howells 2018-11-01 23:07:26 +0000 1575) return 0;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1576)
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1577) out_unreg:
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1578) (void)unregister_filesystem(&hugetlbfs_fs_type);
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1579) out_free:
d1d5e05ffdc11 (Hillf Danton 2012-03-21 16:34:15 -0700 1580) kmem_cache_destroy(hugetlbfs_inode_cachep);
8fc312b32b25c (Mike Kravetz 2019-11-30 17:56:34 -0800 1581) out:
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1582) return error;
^1da177e4c3f4 (Linus Torvalds 2005-04-16 15:20:36 -0700 1583) }
3e89e1c5ea842 (Paul Gortmaker 2016-01-14 15:21:52 -0800 1584) fs_initcall(init_hugetlbfs_fs)