VisionFive2 Linux kernel

StarFive Tech Linux Kernel for VisionFive (JH7110) boards (mirror)

More than 9999 Commits   34 Branches   58 Tags
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    1) /*
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    2)  * hugetlbpage-backed filesystem.  Based on ramfs.
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    3)  *
6d49e352ae9ae (Nadia Yvette Chambers   2012-12-06 10:39:54 +0100    4)  * Nadia Yvette Chambers, 2002
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    5)  *
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    6)  * Copyright (C) 2002 Linus Torvalds.
3e89e1c5ea842 (Paul Gortmaker          2016-01-14 15:21:52 -0800    7)  * License: GPL
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    8)  */
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700    9) 
9b857d26d08f0 (Andrew Morton           2014-06-04 16:07:21 -0700   10) #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
9b857d26d08f0 (Andrew Morton           2014-06-04 16:07:21 -0700   11) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   12) #include <linux/thread_info.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   13) #include <asm/current.h>
174cd4b1e5fbd (Ingo Molnar             2017-02-02 19:15:33 +0100   14) #include <linux/sched/signal.h>		/* remove ASAP */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   15) #include <linux/falloc.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   16) #include <linux/fs.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   17) #include <linux/mount.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   18) #include <linux/file.h>
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700   19) #include <linux/kernel.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   20) #include <linux/writeback.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   21) #include <linux/pagemap.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   22) #include <linux/highmem.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   23) #include <linux/init.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   24) #include <linux/string.h>
16f7e0fe2ecc3 (Randy Dunlap            2006-01-11 12:17:46 -0800   25) #include <linux/capability.h>
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700   26) #include <linux/ctype.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   27) #include <linux/backing-dev.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   28) #include <linux/hugetlb.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   29) #include <linux/pagevec.h>
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   30) #include <linux/fs_parser.h>
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700   31) #include <linux/mman.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   32) #include <linux/slab.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   33) #include <linux/dnotify.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   34) #include <linux/statfs.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   35) #include <linux/security.h>
1fd7317d02ec0 (Nick Black              2009-09-22 16:43:33 -0700   36) #include <linux/magic.h>
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900   37) #include <linux/migrate.h>
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400   38) #include <linux/uio.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   39) 
7c0f6ba682b9c (Linus Torvalds          2016-12-24 11:46:01 -0800   40) #include <linux/uaccess.h>
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700   41) #include <linux/sched/mm.h>
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   42) 
ee9b6d61a2a43 (Josef 'Jeff' Sipek      2007-02-12 00:55:41 -0800   43) static const struct super_operations hugetlbfs_ops;
f5e54d6e53a20 (Christoph Hellwig       2006-06-28 04:26:44 -0700   44) static const struct address_space_operations hugetlbfs_aops;
4b6f5d20b04dc (Arjan van de Ven        2006-03-28 01:56:42 -0800   45) const struct file_operations hugetlbfs_file_operations;
92e1d5be91a0e (Arjan van de Ven        2007-02-12 00:55:39 -0800   46) static const struct inode_operations hugetlbfs_dir_inode_operations;
92e1d5be91a0e (Arjan van de Ven        2007-02-12 00:55:39 -0800   47) static const struct inode_operations hugetlbfs_inode_operations;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   48) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   49) enum hugetlbfs_size_type { NO_SIZE, SIZE_STD, SIZE_PERCENT };
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   50) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   51) struct hugetlbfs_fs_context {
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   52) 	struct hstate		*hstate;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   53) 	unsigned long long	max_size_opt;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   54) 	unsigned long long	min_size_opt;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   55) 	long			max_hpages;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   56) 	long			nr_inodes;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   57) 	long			min_hpages;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   58) 	enum hugetlbfs_size_type max_val_type;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   59) 	enum hugetlbfs_size_type min_val_type;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   60) 	kuid_t			uid;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   61) 	kgid_t			gid;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100   62) 	umode_t			mode;
a1d776ee3147c (David Gibson            2012-03-21 16:34:12 -0700   63) };
a1d776ee3147c (David Gibson            2012-03-21 16:34:12 -0700   64) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   65) int sysctl_hugetlb_shm_group;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700   66) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   67) enum hugetlb_param {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   68) 	Opt_gid,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   69) 	Opt_min_size,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   70) 	Opt_mode,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   71) 	Opt_nr_inodes,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   72) 	Opt_pagesize,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   73) 	Opt_size,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   74) 	Opt_uid,
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700   75) };
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700   76) 
d7167b149943e (Al Viro                 2019-09-07 07:23:15 -0400   77) static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   78) 	fsparam_u32   ("gid",		Opt_gid),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   79) 	fsparam_string("min_size",	Opt_min_size),
79da14fac0b5f (Mike Kravetz            2021-07-23 15:50:44 -0700   80) 	fsparam_u32oct("mode",		Opt_mode),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   81) 	fsparam_string("nr_inodes",	Opt_nr_inodes),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   82) 	fsparam_string("pagesize",	Opt_pagesize),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   83) 	fsparam_string("size",		Opt_size),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   84) 	fsparam_u32   ("uid",		Opt_uid),
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   85) 	{}
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   86) };
32021982a324d (David Howells           2018-11-01 23:07:26 +0000   87) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   88) #ifdef CONFIG_NUMA
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   89) static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   90) 					struct inode *inode, pgoff_t index)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   91) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   92) 	vma->vm_policy = mpol_shared_policy_lookup(&HUGETLBFS_I(inode)->policy,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   93) 							index);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   94) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   95) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   96) static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   97) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   98) 	mpol_cond_put(vma->vm_policy);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700   99) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  100) #else
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  101) static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  102) 					struct inode *inode, pgoff_t index)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  103) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  104) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  105) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  106) static inline void hugetlb_drop_vma_policy(struct vm_area_struct *vma)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  107) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  108) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  109) #endif
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  110) 
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  111) static void huge_pagevec_release(struct pagevec *pvec)
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  112) {
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  113) 	int i;
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  114) 
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  115) 	for (i = 0; i < pagevec_count(pvec); ++i)
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  116) 		put_page(pvec->pages[i]);
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  117) 
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  118) 	pagevec_reinit(pvec);
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  119) }
2e9b367c2273e (Adam Litke              2005-10-29 18:16:47 -0700  120) 
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  121) /*
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  122)  * Mask used when checking the page offset value passed in via system
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  123)  * calls.  This value will be converted to a loff_t which is signed.
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  124)  * Therefore, we want to check the upper PAGE_SHIFT + 1 bits of the
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  125)  * value.  The extra bit (- 1 in the shift value) is to take the sign
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  126)  * bit into account.
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  127)  */
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  128) #define PGOFF_LOFFT_MAX \
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  129) 	(((1UL << (PAGE_SHIFT + 1)) - 1) <<  (BITS_PER_LONG - (PAGE_SHIFT + 1)))
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  130) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  131) static int hugetlbfs_file_mmap(struct file *file, struct vm_area_struct *vma)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  132) {
496ad9aa8ef44 (Al Viro                 2013-01-23 17:07:38 -0500  133) 	struct inode *inode = file_inode(file);
22247efd822e6 (Peter Xu                2021-05-14 17:27:04 -0700  134) 	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  135) 	loff_t len, vma_len;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  136) 	int ret;
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  137) 	struct hstate *h = hstate_file(file);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  138) 
68589bc353037 (Hugh Dickins            2006-11-14 02:03:32 -0800  139) 	/*
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  140) 	 * vma address alignment (but not the pgoff alignment) has
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  141) 	 * already been checked by prepare_hugepage_range.  If you add
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  142) 	 * any error returns here, do so after setting VM_HUGETLB, so
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  143) 	 * is_vm_hugetlb_page tests below unmap_region go the right
45e55300f1149 (Peter Collingbourne     2020-08-06 23:23:37 -0700  144) 	 * way when do_mmap unwinds (may be important on powerpc
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  145) 	 * and ia64).
68589bc353037 (Hugh Dickins            2006-11-14 02:03:32 -0800  146) 	 */
a2fce9143057f (Naoya Horiguchi         2013-04-17 15:58:27 -0700  147) 	vma->vm_flags |= VM_HUGETLB | VM_DONTEXPAND;
68589bc353037 (Hugh Dickins            2006-11-14 02:03:32 -0800  148) 	vma->vm_ops = &hugetlb_vm_ops;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  149) 
22247efd822e6 (Peter Xu                2021-05-14 17:27:04 -0700  150) 	ret = seal_check_future_write(info->seals, vma);
22247efd822e6 (Peter Xu                2021-05-14 17:27:04 -0700  151) 	if (ret)
22247efd822e6 (Peter Xu                2021-05-14 17:27:04 -0700  152) 		return ret;
22247efd822e6 (Peter Xu                2021-05-14 17:27:04 -0700  153) 
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  154) 	/*
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  155) 	 * page based offset in vm_pgoff could be sufficiently large to
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  156) 	 * overflow a loff_t when converted to byte offset.  This can
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  157) 	 * only happen on architectures where sizeof(loff_t) ==
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  158) 	 * sizeof(unsigned long).  So, only check in those instances.
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  159) 	 */
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  160) 	if (sizeof(unsigned long) == sizeof(loff_t)) {
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  161) 		if (vma->vm_pgoff & PGOFF_LOFFT_MAX)
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  162) 			return -EINVAL;
5df63c2a149ae (Mike Kravetz            2018-04-05 16:18:21 -0700  163) 	}
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  164) 
63489f8e82114 (Mike Kravetz            2018-03-22 16:17:13 -0700  165) 	/* must be huge page aligned */
2b37c35e6552b (Becky Bruce             2011-07-25 17:11:49 -0700  166) 	if (vma->vm_pgoff & (~huge_page_mask(h) >> PAGE_SHIFT))
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  167) 		return -EINVAL;
dec4ad86c2fbe (David Gibson            2007-08-30 23:56:40 -0700  168) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  169) 	vma_len = (loff_t)(vma->vm_end - vma->vm_start);
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  170) 	len = vma_len + ((loff_t)vma->vm_pgoff << PAGE_SHIFT);
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  171) 	/* check for overflow */
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  172) 	if (len < vma_len)
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  173) 		return -EINVAL;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  174) 
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  175) 	inode_lock(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  176) 	file_accessed(file);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  177) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  178) 	ret = -ENOMEM;
33b8f84a4ee78 (Mike Kravetz            2021-02-24 12:09:54 -0800  179) 	if (!hugetlb_reserve_pages(inode,
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  180) 				vma->vm_pgoff >> huge_page_order(h),
5a6fe12595067 (Mel Gorman              2009-02-10 14:02:27 +0000  181) 				len >> huge_page_shift(h), vma,
5a6fe12595067 (Mel Gorman              2009-02-10 14:02:27 +0000  182) 				vma->vm_flags))
a43a8c39bbb49 (Kenneth W Chen          2006-06-23 02:03:15 -0700  183) 		goto out;
b45b5bd65f668 (David Gibson            2006-03-22 00:08:55 -0800  184) 
4c88726597721 (Adam Litke              2005-10-29 18:16:46 -0700  185) 	ret = 0;
b6174df5eec9c (Zhang, Yanmin           2006-07-10 04:44:49 -0700  186) 	if (vma->vm_flags & VM_WRITE && inode->i_size < len)
045c7a3f53d94 (Mike Kravetz            2017-04-13 14:56:32 -0700  187) 		i_size_write(inode, len);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  188) out:
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  189) 	inode_unlock(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  190) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  191) 	return ret;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  192) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  193) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  194) /*
3e4e28c5a8f01 (Michel Lespinasse       2020-06-08 21:33:51 -0700  195)  * Called under mmap_write_lock(mm).
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  196)  */
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  197) 
d2ba27e8007b3 (Adrian Bunk             2007-05-06 14:49:00 -0700  198) #ifndef HAVE_ARCH_HUGETLB_UNMAPPED_AREA
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  199) static unsigned long
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  200) hugetlb_get_unmapped_area_bottomup(struct file *file, unsigned long addr,
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  201) 		unsigned long len, unsigned long pgoff, unsigned long flags)
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  202) {
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  203) 	struct hstate *h = hstate_file(file);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  204) 	struct vm_unmapped_area_info info;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  205) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  206) 	info.flags = 0;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  207) 	info.length = len;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  208) 	info.low_limit = current->mm->mmap_base;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  209) 	info.high_limit = TASK_SIZE;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  210) 	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  211) 	info.align_offset = 0;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  212) 	return vm_unmapped_area(&info);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  213) }
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  214) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  215) static unsigned long
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  216) hugetlb_get_unmapped_area_topdown(struct file *file, unsigned long addr,
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  217) 		unsigned long len, unsigned long pgoff, unsigned long flags)
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  218) {
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  219) 	struct hstate *h = hstate_file(file);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  220) 	struct vm_unmapped_area_info info;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  221) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  222) 	info.flags = VM_UNMAPPED_AREA_TOPDOWN;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  223) 	info.length = len;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  224) 	info.low_limit = max(PAGE_SIZE, mmap_min_addr);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  225) 	info.high_limit = current->mm->mmap_base;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  226) 	info.align_mask = PAGE_MASK & ~huge_page_mask(h);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  227) 	info.align_offset = 0;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  228) 	addr = vm_unmapped_area(&info);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  229) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  230) 	/*
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  231) 	 * A failed mmap() very likely causes application failure,
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  232) 	 * so fall back to the bottom-up function here. This scenario
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  233) 	 * can happen with large stack limits and large mmap()
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  234) 	 * allocations.
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  235) 	 */
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  236) 	if (unlikely(offset_in_page(addr))) {
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  237) 		VM_BUG_ON(addr != -ENOMEM);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  238) 		info.flags = 0;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  239) 		info.low_limit = current->mm->mmap_base;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  240) 		info.high_limit = TASK_SIZE;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  241) 		addr = vm_unmapped_area(&info);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  242) 	}
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  243) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  244) 	return addr;
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  245) }
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  246) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  247) static unsigned long
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  248) hugetlb_get_unmapped_area(struct file *file, unsigned long addr,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  249) 		unsigned long len, unsigned long pgoff, unsigned long flags)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  250) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  251) 	struct mm_struct *mm = current->mm;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  252) 	struct vm_area_struct *vma;
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  253) 	struct hstate *h = hstate_file(file);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  254) 
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  255) 	if (len & ~huge_page_mask(h))
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  256) 		return -EINVAL;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  257) 	if (len > TASK_SIZE)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  258) 		return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  259) 
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700  260) 	if (flags & MAP_FIXED) {
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  261) 		if (prepare_hugepage_range(file, addr, len))
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700  262) 			return -EINVAL;
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700  263) 		return addr;
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700  264) 	}
036e08568cbee (Benjamin Herrenschmidt  2007-05-06 14:50:12 -0700  265) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  266) 	if (addr) {
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  267) 		addr = ALIGN(addr, huge_page_size(h));
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  268) 		vma = find_vma(mm, addr);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  269) 		if (TASK_SIZE - len >= addr &&
1be7107fbe18e (Hugh Dickins            2017-06-19 04:03:24 -0700  270) 		    (!vma || addr + len <= vm_start_gap(vma)))
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  271) 			return addr;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  272) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  273) 
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  274) 	/*
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  275) 	 * Use mm->get_unmapped_area value as a hint to use topdown routine.
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  276) 	 * If architectures have special needs, they should define their own
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  277) 	 * version of hugetlb_get_unmapped_area.
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  278) 	 */
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  279) 	if (mm->get_unmapped_area == arch_get_unmapped_area_topdown)
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  280) 		return hugetlb_get_unmapped_area_topdown(file, addr, len,
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  281) 				pgoff, flags);
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  282) 	return hugetlb_get_unmapped_area_bottomup(file, addr, len,
885902531586d (Shijie Hu               2020-06-03 16:03:34 -0700  283) 			pgoff, flags);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  284) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  285) #endif
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  286) 
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  287) static size_t
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  288) hugetlbfs_read_actor(struct page *page, unsigned long offset,
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  289) 			struct iov_iter *to, unsigned long size)
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  290) {
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  291) 	size_t copied = 0;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  292) 	int i, chunksize;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  293) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  294) 	/* Find which 4k chunk and offset with in that chunk */
09cbfeaf1a5a6 (Kirill A. Shutemov      2016-04-01 15:29:47 +0300  295) 	i = offset >> PAGE_SHIFT;
09cbfeaf1a5a6 (Kirill A. Shutemov      2016-04-01 15:29:47 +0300  296) 	offset = offset & ~PAGE_MASK;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  297) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  298) 	while (size) {
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  299) 		size_t n;
09cbfeaf1a5a6 (Kirill A. Shutemov      2016-04-01 15:29:47 +0300  300) 		chunksize = PAGE_SIZE;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  301) 		if (offset)
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  302) 			chunksize -= offset;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  303) 		if (chunksize > size)
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  304) 			chunksize = size;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  305) 		n = copy_page_to_iter(&page[i], offset, chunksize, to);
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  306) 		copied += n;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  307) 		if (n != chunksize)
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  308) 			return copied;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  309) 		offset = 0;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  310) 		size -= chunksize;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  311) 		i++;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  312) 	}
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  313) 	return copied;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  314) }
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  315) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  316) /*
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  317)  * Support for read() - Find the page attached to f_mapping and copy out the
c7e285e31f764 (Miaohe Lin              2021-02-24 12:10:08 -0800  318)  * data. Its *very* similar to generic_file_buffered_read(), we can't use that
ea1754a084760 (Kirill A. Shutemov      2016-04-01 15:29:48 +0300  319)  * since it has PAGE_SIZE assumptions.
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  320)  */
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  321) static ssize_t hugetlbfs_read_iter(struct kiocb *iocb, struct iov_iter *to)
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  322) {
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  323) 	struct file *file = iocb->ki_filp;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  324) 	struct hstate *h = hstate_file(file);
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  325) 	struct address_space *mapping = file->f_mapping;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  326) 	struct inode *inode = mapping->host;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  327) 	unsigned long index = iocb->ki_pos >> huge_page_shift(h);
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  328) 	unsigned long offset = iocb->ki_pos & ~huge_page_mask(h);
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  329) 	unsigned long end_index;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  330) 	loff_t isize;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  331) 	ssize_t retval = 0;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  332) 
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  333) 	while (iov_iter_count(to)) {
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  334) 		struct page *page;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  335) 		size_t nr, copied;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  336) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  337) 		/* nr is the maximum number of bytes to copy from this page */
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  338) 		nr = huge_page_size(h);
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  339) 		isize = i_size_read(inode);
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  340) 		if (!isize)
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  341) 			break;
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  342) 		end_index = (isize - 1) >> huge_page_shift(h);
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  343) 		if (index > end_index)
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  344) 			break;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  345) 		if (index == end_index) {
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  346) 			nr = ((isize - 1) & ~huge_page_mask(h)) + 1;
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  347) 			if (nr <= offset)
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  348) 				break;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  349) 		}
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  350) 		nr = nr - offset;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  351) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  352) 		/* Find the page */
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  353) 		page = find_lock_page(mapping, index);
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  354) 		if (unlikely(page == NULL)) {
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  355) 			/*
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  356) 			 * We have a HOLE, zero out the user-buffer for the
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  357) 			 * length of the hole or request.
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  358) 			 */
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  359) 			copied = iov_iter_zero(nr, to);
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  360) 		} else {
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  361) 			unlock_page(page);
a05b0855fd155 (Aneesh Kumar K.V        2012-03-21 16:34:08 -0700  362) 
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  363) 			/*
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  364) 			 * We have the page, copy it to user space buffer.
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  365) 			 */
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  366) 			copied = hugetlbfs_read_actor(page, offset, to, nr);
09cbfeaf1a5a6 (Kirill A. Shutemov      2016-04-01 15:29:47 +0300  367) 			put_page(page);
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  368) 		}
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  369) 		offset += copied;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  370) 		retval += copied;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  371) 		if (copied != nr && iov_iter_count(to)) {
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  372) 			if (!retval)
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  373) 				retval = -EFAULT;
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  374) 			break;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  375) 		}
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  376) 		index += offset >> huge_page_shift(h);
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  377) 		offset &= ~huge_page_mask(h);
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  378) 	}
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400  379) 	iocb->ki_pos = ((loff_t)index << huge_page_shift(h)) + offset;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  380) 	return retval;
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  381) }
e63e1e5a6b754 (Badari Pulavarty        2007-10-16 01:26:22 -0700  382) 
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  383) static int hugetlbfs_write_begin(struct file *file,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  384) 			struct address_space *mapping,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  385) 			loff_t pos, unsigned len, unsigned flags,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  386) 			struct page **pagep, void **fsdata)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  387) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  388) 	return -EINVAL;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  389) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  390) 
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  391) static int hugetlbfs_write_end(struct file *file, struct address_space *mapping,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  392) 			loff_t pos, unsigned len, unsigned copied,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  393) 			struct page *page, void *fsdata)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  394) {
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700  395) 	BUG();
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  396) 	return -EINVAL;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  397) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  398) 
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  399) static void remove_huge_page(struct page *page)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  400) {
b9ea25152e563 (Konstantin Khlebnikov   2015-04-14 15:45:27 -0700  401) 	ClearPageDirty(page);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  402) 	ClearPageUptodate(page);
bd65cb86c98a7 (Minchan Kim             2011-03-22 16:30:54 -0700  403) 	delete_from_page_cache(page);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  404) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  405) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  406) static void
f808c13fd3738 (Davidlohr Bueso         2017-09-08 16:15:08 -0700  407) hugetlb_vmdelete_list(struct rb_root_cached *root, pgoff_t start, pgoff_t end)
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  408) {
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  409) 	struct vm_area_struct *vma;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  410) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  411) 	/*
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  412) 	 * end == 0 indicates that the entire range after
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  413) 	 * start should be unmapped.
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  414) 	 */
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  415) 	vma_interval_tree_foreach(vma, root, start, end ? end : ULONG_MAX) {
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  416) 		unsigned long v_offset;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  417) 		unsigned long v_end;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  418) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  419) 		/*
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  420) 		 * Can the expression below overflow on 32-bit arches?
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  421) 		 * No, because the interval tree returns us only those vmas
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  422) 		 * which overlap the truncated area starting at pgoff,
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  423) 		 * and no vma on a 32-bit arch can span beyond the 4GB.
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  424) 		 */
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  425) 		if (vma->vm_pgoff < start)
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  426) 			v_offset = (start - vma->vm_pgoff) << PAGE_SHIFT;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  427) 		else
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  428) 			v_offset = 0;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  429) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  430) 		if (!end)
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  431) 			v_end = vma->vm_end;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  432) 		else {
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  433) 			v_end = ((end - vma->vm_pgoff) << PAGE_SHIFT)
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  434) 							+ vma->vm_start;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  435) 			if (v_end > vma->vm_end)
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  436) 				v_end = vma->vm_end;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  437) 		}
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  438) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  439) 		unmap_hugepage_range(vma, vma->vm_start + v_offset, v_end,
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  440) 									NULL);
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  441) 	}
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  442) }
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  443) 
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  444) /*
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  445)  * remove_inode_hugepages handles two distinct cases: truncation and hole
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  446)  * punch.  There are subtle differences in operation for each case.
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  447)  *
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  448)  * truncation is indicated by end of range being LLONG_MAX
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  449)  *	In this case, we first scan the range and release found pages.
1935ebd3cf6c4 (Miaohe Lin              2021-02-24 12:10:21 -0800  450)  *	After releasing pages, hugetlb_unreserve_pages cleans up region/reserve
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  451)  *	maps and global counts.  Page faults can not race with truncation
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  452)  *	in this routine.  hugetlb_no_page() holds i_mmap_rwsem and prevents
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  453)  *	page faults in the truncated range by checking i_size.  i_size is
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  454)  *	modified while holding i_mmap_rwsem.
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  455)  * hole punch is indicated if end is not LLONG_MAX
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  456)  *	In the hole punch case we scan the range and release found pages.
1935ebd3cf6c4 (Miaohe Lin              2021-02-24 12:10:21 -0800  457)  *	Only when releasing a page is the associated region/reserve map
1935ebd3cf6c4 (Miaohe Lin              2021-02-24 12:10:21 -0800  458)  *	deleted.  The region/reserve map for ranges without associated
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  459)  *	pages are not modified.  Page faults can race with hole punch.
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  460)  *	This is indicated if we find a mapped page.
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  461)  * Note: If the passed end of range value is beyond the end of file, but
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  462)  * not LLONG_MAX this routine still performs a hole punch operation.
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  463)  */
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  464) static void remove_inode_hugepages(struct inode *inode, loff_t lstart,
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  465) 				   loff_t lend)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  466) {
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  467) 	struct hstate *h = hstate_inode(inode);
b45b5bd65f668 (David Gibson            2006-03-22 00:08:55 -0800  468) 	struct address_space *mapping = &inode->i_data;
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  469) 	const pgoff_t start = lstart >> huge_page_shift(h);
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  470) 	const pgoff_t end = lend >> huge_page_shift(h);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  471) 	struct pagevec pvec;
d72dc8a25afc7 (Jan Kara                2017-09-06 16:21:18 -0700  472) 	pgoff_t next, index;
a43a8c39bbb49 (Kenneth W Chen          2006-06-23 02:03:15 -0700  473) 	int i, freed = 0;
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  474) 	bool truncate_op = (lend == LLONG_MAX);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  475) 
8667982014d60 (Mel Gorman              2017-11-15 17:37:52 -0800  476) 	pagevec_init(&pvec);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  477) 	next = start;
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  478) 	while (next < end) {
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  479) 		/*
1817889e3b2cc (Mike Kravetz            2015-11-20 15:57:13 -0800  480) 		 * When no more pages are found, we are done.
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  481) 		 */
397162ffa2ed1 (Jan Kara                2017-09-06 16:21:43 -0700  482) 		if (!pagevec_lookup_range(&pvec, mapping, &next, end - 1))
1817889e3b2cc (Mike Kravetz            2015-11-20 15:57:13 -0800  483) 			break;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  484) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  485) 		for (i = 0; i < pagevec_count(&pvec); ++i) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  486) 			struct page *page = pvec.pages[i];
d4241a049ac00 (Miaohe Lin              2021-05-04 18:33:34 -0700  487) 			u32 hash = 0;
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  488) 
d72dc8a25afc7 (Jan Kara                2017-09-06 16:21:18 -0700  489) 			index = page->index;
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  490) 			if (!truncate_op) {
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  491) 				/*
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  492) 				 * Only need to hold the fault mutex in the
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  493) 				 * hole punch case.  This prevents races with
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  494) 				 * page faults.  Races are not possible in the
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  495) 				 * case of truncation.
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  496) 				 */
d4241a049ac00 (Miaohe Lin              2021-05-04 18:33:34 -0700  497) 				hash = hugetlb_fault_mutex_hash(mapping, index);
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  498) 				mutex_lock(&hugetlb_fault_mutex_table[hash]);
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  499) 			}
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  500) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  501) 			/*
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  502) 			 * If page is mapped, it was faulted in after being
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  503) 			 * unmapped in caller.  Unmap (again) now after taking
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  504) 			 * the fault mutex.  The mutex will prevent faults
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  505) 			 * until we finish removing the page.
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  506) 			 *
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  507) 			 * This race can only happen in the hole punch case.
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  508) 			 * Getting here in a truncate operation is a bug.
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  509) 			 */
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  510) 			if (unlikely(page_mapped(page))) {
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  511) 				BUG_ON(truncate_op);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  512) 
c0d0381ade798 (Mike Kravetz            2020-04-01 21:11:05 -0700  513) 				mutex_unlock(&hugetlb_fault_mutex_table[hash]);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  514) 				i_mmap_lock_write(mapping);
c0d0381ade798 (Mike Kravetz            2020-04-01 21:11:05 -0700  515) 				mutex_lock(&hugetlb_fault_mutex_table[hash]);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  516) 				hugetlb_vmdelete_list(&mapping->i_mmap,
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  517) 					index * pages_per_huge_page(h),
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  518) 					(index + 1) * pages_per_huge_page(h));
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  519) 				i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  520) 			}
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  521) 
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  522) 			lock_page(page);
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  523) 			/*
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  524) 			 * We must free the huge page and remove from page
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  525) 			 * cache (remove_huge_page) BEFORE removing the
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  526) 			 * region/reserve map (hugetlb_unreserve_pages).  In
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  527) 			 * rare out of memory conditions, removal of the
72e2936c04f7d (zhong jiang             2016-10-07 17:02:01 -0700  528) 			 * region/reserve map could fail. Correspondingly,
72e2936c04f7d (zhong jiang             2016-10-07 17:02:01 -0700  529) 			 * the subpool and global reserve usage count can need
72e2936c04f7d (zhong jiang             2016-10-07 17:02:01 -0700  530) 			 * to be adjusted.
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  531) 			 */
e32905e57358f (Mike Kravetz            2021-05-22 17:42:11 -0700  532) 			VM_BUG_ON(HPageRestoreReserve(page));
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  533) 			remove_huge_page(page);
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  534) 			freed++;
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  535) 			if (!truncate_op) {
4aae8d1c051ea (Mike Kravetz            2016-01-15 16:57:40 -0800  536) 				if (unlikely(hugetlb_unreserve_pages(inode,
d72dc8a25afc7 (Jan Kara                2017-09-06 16:21:18 -0700  537) 							index, index + 1, 1)))
72e2936c04f7d (zhong jiang             2016-10-07 17:02:01 -0700  538) 					hugetlb_fix_reserve_counts(inode);
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  539) 			}
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  540) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  541) 			unlock_page(page);
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  542) 			if (!truncate_op)
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  543) 				mutex_unlock(&hugetlb_fault_mutex_table[hash]);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  544) 		}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  545) 		huge_pagevec_release(&pvec);
1817889e3b2cc (Mike Kravetz            2015-11-20 15:57:13 -0800  546) 		cond_resched();
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  547) 	}
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  548) 
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  549) 	if (truncate_op)
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  550) 		(void)hugetlb_unreserve_pages(inode, start, LONG_MAX, freed);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  551) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  552) 
2bbbda308f5ca (Al Viro                 2010-06-04 19:52:12 -0400  553) static void hugetlbfs_evict_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  554) {
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  555) 	struct resv_map *resv_map;
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  556) 
b5cec28d36f5e (Mike Kravetz            2015-09-08 15:01:41 -0700  557) 	remove_inode_hugepages(inode, 0, LLONG_MAX);
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  558) 
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  559) 	/*
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  560) 	 * Get the resv_map from the address space embedded in the inode.
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  561) 	 * This is the address space which points to any resv_map allocated
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  562) 	 * at inode creation time.  If this is a device special inode,
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  563) 	 * i_mapping may not point to the original address space.
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  564) 	 */
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  565) 	resv_map = (struct resv_map *)(&inode->i_data)->private_data;
f27a5136f70a8 (Mike Kravetz            2019-05-13 17:22:55 -0700  566) 	/* Only regular and link inodes have associated reserve maps */
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  567) 	if (resv_map)
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  568) 		resv_map_release(&resv_map->refs);
dbd5768f87ff6 (Jan Kara                2012-05-03 14:48:02 +0200  569) 	clear_inode(inode);
149f4211afda8 (Christoph Hellwig       2005-10-29 18:16:43 -0700  570) }
149f4211afda8 (Christoph Hellwig       2005-10-29 18:16:43 -0700  571) 
e5d319dedafd2 (Miaohe Lin              2021-02-24 12:10:25 -0800  572) static void hugetlb_vmtruncate(struct inode *inode, loff_t offset)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  573) {
856fc29505556 (Hugh Dickins            2006-10-28 10:38:43 -0700  574) 	pgoff_t pgoff;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  575) 	struct address_space *mapping = inode->i_mapping;
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  576) 	struct hstate *h = hstate_inode(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  577) 
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  578) 	BUG_ON(offset & ~huge_page_mask(h));
856fc29505556 (Hugh Dickins            2006-10-28 10:38:43 -0700  579) 	pgoff = offset >> PAGE_SHIFT;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  580) 
83cde9e8ba95d (Davidlohr Bueso         2014-12-12 16:54:21 -0800  581) 	i_mmap_lock_write(mapping);
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  582) 	i_size_write(inode, offset);
f808c13fd3738 (Davidlohr Bueso         2017-09-08 16:15:08 -0700  583) 	if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
1bfad99ab4256 (Mike Kravetz            2015-09-08 15:01:38 -0700  584) 		hugetlb_vmdelete_list(&mapping->i_mmap, pgoff, 0);
c86aa7bbfd556 (Mike Kravetz            2018-12-28 00:39:42 -0800  585) 	i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  586) 	remove_inode_hugepages(inode, offset, LLONG_MAX);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  587) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  588) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  589) static long hugetlbfs_punch_hole(struct inode *inode, loff_t offset, loff_t len)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  590) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  591) 	struct hstate *h = hstate_inode(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  592) 	loff_t hpage_size = huge_page_size(h);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  593) 	loff_t hole_start, hole_end;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  594) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  595) 	/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  596) 	 * For hole punch round up the beginning offset of the hole and
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  597) 	 * round down the end.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  598) 	 */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  599) 	hole_start = round_up(offset, hpage_size);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  600) 	hole_end = round_down(offset + len, hpage_size);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  601) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  602) 	if (hole_end > hole_start) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  603) 		struct address_space *mapping = inode->i_mapping;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  604) 		struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  605) 
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  606) 		inode_lock(inode);
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  607) 
398c0da7364c9 (Miaohe Lin              2021-02-24 12:10:18 -0800  608) 		/* protected by i_rwsem */
ab3948f58ff84 (Joel Fernandes (Google) 2019-03-05 15:47:54 -0800  609) 		if (info->seals & (F_SEAL_WRITE | F_SEAL_FUTURE_WRITE)) {
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  610) 			inode_unlock(inode);
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  611) 			return -EPERM;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  612) 		}
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  613) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  614) 		i_mmap_lock_write(mapping);
f808c13fd3738 (Davidlohr Bueso         2017-09-08 16:15:08 -0700  615) 		if (!RB_EMPTY_ROOT(&mapping->i_mmap.rb_root))
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  616) 			hugetlb_vmdelete_list(&mapping->i_mmap,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  617) 						hole_start >> PAGE_SHIFT,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  618) 						hole_end  >> PAGE_SHIFT);
c86aa7bbfd556 (Mike Kravetz            2018-12-28 00:39:42 -0800  619) 		i_mmap_unlock_write(mapping);
e7c58097793ef (Mike Kravetz            2019-01-08 15:23:32 -0800  620) 		remove_inode_hugepages(inode, hole_start, hole_end);
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  621) 		inode_unlock(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  622) 	}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  623) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  624) 	return 0;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  625) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  626) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  627) static long hugetlbfs_fallocate(struct file *file, int mode, loff_t offset,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  628) 				loff_t len)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  629) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  630) 	struct inode *inode = file_inode(file);
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  631) 	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  632) 	struct address_space *mapping = inode->i_mapping;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  633) 	struct hstate *h = hstate_inode(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  634) 	struct vm_area_struct pseudo_vma;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  635) 	struct mm_struct *mm = current->mm;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  636) 	loff_t hpage_size = huge_page_size(h);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  637) 	unsigned long hpage_shift = huge_page_shift(h);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  638) 	pgoff_t start, index, end;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  639) 	int error;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  640) 	u32 hash;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  641) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  642) 	if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  643) 		return -EOPNOTSUPP;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  644) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  645) 	if (mode & FALLOC_FL_PUNCH_HOLE)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  646) 		return hugetlbfs_punch_hole(inode, offset, len);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  647) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  648) 	/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  649) 	 * Default preallocate case.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  650) 	 * For this range, start is rounded down and end is rounded up
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  651) 	 * as well as being converted to page offsets.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  652) 	 */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  653) 	start = offset >> hpage_shift;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  654) 	end = (offset + len + hpage_size - 1) >> hpage_shift;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  655) 
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  656) 	inode_lock(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  657) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  658) 	/* We need to check rlimit even when FALLOC_FL_KEEP_SIZE */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  659) 	error = inode_newsize_ok(inode, offset + len);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  660) 	if (error)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  661) 		goto out;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  662) 
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  663) 	if ((info->seals & F_SEAL_GROW) && offset + len > inode->i_size) {
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  664) 		error = -EPERM;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  665) 		goto out;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  666) 	}
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  667) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  668) 	/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  669) 	 * Initialize a pseudo vma as this is required by the huge page
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  670) 	 * allocation routines.  If NUMA is configured, use page index
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  671) 	 * as input to create an allocation policy.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  672) 	 */
2c4541e24c55e (Kirill A. Shutemov      2018-07-26 16:37:30 -0700  673) 	vma_init(&pseudo_vma, mm);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  674) 	pseudo_vma.vm_flags = (VM_HUGETLB | VM_MAYSHARE | VM_SHARED);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  675) 	pseudo_vma.vm_file = file;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  676) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  677) 	for (index = start; index < end; index++) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  678) 		/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  679) 		 * This is supposed to be the vaddr where the page is being
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  680) 		 * faulted in, but we have no vaddr here.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  681) 		 */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  682) 		struct page *page;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  683) 		unsigned long addr;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  684) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  685) 		cond_resched();
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  686) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  687) 		/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  688) 		 * fallocate(2) manpage permits EINTR; we may have been
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  689) 		 * interrupted because we are using up too much memory.
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  690) 		 */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  691) 		if (signal_pending(current)) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  692) 			error = -EINTR;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  693) 			break;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  694) 		}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  695) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  696) 		/* Set numa allocation policy based on index */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  697) 		hugetlb_set_vma_policy(&pseudo_vma, inode, index);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  698) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  699) 		/* addr is the offset within the file (zero based) */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  700) 		addr = index * hpage_size;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  701) 
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  702) 		/*
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  703) 		 * fault mutex taken here, protects against fault path
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  704) 		 * and hole punch.  inode_lock previously taken protects
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  705) 		 * against truncation.
87bf91d39bb52 (Mike Kravetz            2020-04-01 21:11:08 -0700  706) 		 */
188b04a7d9386 (Wei Yang                2019-11-30 17:57:02 -0800  707) 		hash = hugetlb_fault_mutex_hash(mapping, index);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  708) 		mutex_lock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  709) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  710) 		/* See if already present in mapping to avoid alloc/free */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  711) 		page = find_get_page(mapping, index);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  712) 		if (page) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  713) 			put_page(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  714) 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  715) 			hugetlb_drop_vma_policy(&pseudo_vma);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  716) 			continue;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  717) 		}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  718) 
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  719) 		/*
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  720) 		 * Allocate page without setting the avoid_reserve argument.
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  721) 		 * There certainly are no reserves associated with the
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  722) 		 * pseudo_vma.  However, there could be shared mappings with
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  723) 		 * reserves for the file at the inode level.  If we fallocate
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  724) 		 * pages in these areas, we need to consume the reserves
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  725) 		 * to keep reservation accounting consistent.
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  726) 		 */
88ce3fef47f3f (Miaohe Lin              2021-02-24 12:10:11 -0800  727) 		page = alloc_huge_page(&pseudo_vma, addr, 0);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  728) 		hugetlb_drop_vma_policy(&pseudo_vma);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  729) 		if (IS_ERR(page)) {
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  730) 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  731) 			error = PTR_ERR(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  732) 			goto out;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  733) 		}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  734) 		clear_huge_page(page, addr, pages_per_huge_page(h));
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  735) 		__SetPageUptodate(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  736) 		error = huge_add_to_page_cache(page, mapping, index);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  737) 		if (unlikely(error)) {
846be08578edb (Mike Kravetz            2021-06-15 18:23:29 -0700  738) 			restore_reserve_on_error(h, &pseudo_vma, addr, page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  739) 			put_page(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  740) 			mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  741) 			goto out;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  742) 		}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  743) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  744) 		mutex_unlock(&hugetlb_fault_mutex_table[hash]);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  745) 
8f251a3d5ce3b (Mike Kravetz            2021-02-24 12:08:56 -0800  746) 		SetHPageMigratable(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  747) 		/*
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  748) 		 * unlock_page because locked by add_to_page_cache()
585fc0d2871c9 (Muchun Song             2021-02-04 18:32:03 -0800  749) 		 * put_page() due to reference from alloc_huge_page()
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  750) 		 */
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  751) 		unlock_page(page);
72639e6df4128 (Nadav Amit              2017-11-29 16:11:33 -0800  752) 		put_page(page);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  753) 	}
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  754) 
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  755) 	if (!(mode & FALLOC_FL_KEEP_SIZE) && offset + len > inode->i_size)
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  756) 		i_size_write(inode, offset + len);
078cd8279e659 (Deepa Dinamani          2016-09-14 07:48:04 -0700  757) 	inode->i_ctime = current_time(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  758) out:
5955102c9984f (Al Viro                 2016-01-22 15:40:57 -0500  759) 	inode_unlock(inode);
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  760) 	return error;
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  761) }
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700  762) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  763) static int hugetlbfs_setattr(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  764) 			     struct dentry *dentry, struct iattr *attr)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  765) {
2b0143b5c986b (David Howells           2015-03-17 22:25:59 +0000  766) 	struct inode *inode = d_inode(dentry);
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700  767) 	struct hstate *h = hstate_inode(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  768) 	int error;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  769) 	unsigned int ia_valid = attr->ia_valid;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  770) 	struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  771) 
2f221d6f7b881 (Christian Brauner       2021-01-21 14:19:26 +0100  772) 	error = setattr_prepare(&init_user_ns, dentry, attr);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  773) 	if (error)
1025774ce411f (Christoph Hellwig       2010-06-04 11:30:02 +0200  774) 		return error;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  775) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  776) 	if (ia_valid & ATTR_SIZE) {
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  777) 		loff_t oldsize = inode->i_size;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  778) 		loff_t newsize = attr->ia_size;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  779) 
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  780) 		if (newsize & ~huge_page_mask(h))
1025774ce411f (Christoph Hellwig       2010-06-04 11:30:02 +0200  781) 			return -EINVAL;
398c0da7364c9 (Miaohe Lin              2021-02-24 12:10:18 -0800  782) 		/* protected by i_rwsem */
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  783) 		if ((newsize < oldsize && (info->seals & F_SEAL_SHRINK)) ||
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  784) 		    (newsize > oldsize && (info->seals & F_SEAL_GROW)))
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  785) 			return -EPERM;
e5d319dedafd2 (Miaohe Lin              2021-02-24 12:10:25 -0800  786) 		hugetlb_vmtruncate(inode, newsize);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  787) 	}
1025774ce411f (Christoph Hellwig       2010-06-04 11:30:02 +0200  788) 
2f221d6f7b881 (Christian Brauner       2021-01-21 14:19:26 +0100  789) 	setattr_copy(&init_user_ns, inode, attr);
1025774ce411f (Christoph Hellwig       2010-06-04 11:30:02 +0200  790) 	mark_inode_dirty(inode);
1025774ce411f (Christoph Hellwig       2010-06-04 11:30:02 +0200  791) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  792) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  793) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  794) static struct inode *hugetlbfs_get_root(struct super_block *sb,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000  795) 					struct hugetlbfs_fs_context *ctx)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  796) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  797) 	struct inode *inode;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  798) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  799) 	inode = new_inode(sb);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  800) 	if (inode) {
85fe4025c616a (Christoph Hellwig       2010-10-23 11:19:54 -0400  801) 		inode->i_ino = get_next_ino();
32021982a324d (David Howells           2018-11-01 23:07:26 +0000  802) 		inode->i_mode = S_IFDIR | ctx->mode;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000  803) 		inode->i_uid = ctx->uid;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000  804) 		inode->i_gid = ctx->gid;
078cd8279e659 (Deepa Dinamani          2016-09-14 07:48:04 -0700  805) 		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  806) 		inode->i_op = &hugetlbfs_dir_inode_operations;
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  807) 		inode->i_fop = &simple_dir_operations;
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  808) 		/* directory inodes start off with i_nlink == 2 (for "." entry) */
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  809) 		inc_nlink(inode);
65ed76010dfed (Aneesh Kumar K.V        2012-04-25 16:01:50 -0700  810) 		lockdep_annotate_inode_mutex_key(inode);
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  811) 	}
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  812) 	return inode;
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  813) }
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  814) 
b610ded719186 (Michal Hocko            2013-08-13 16:00:55 -0700  815) /*
c8c06efa8b552 (Davidlohr Bueso         2014-12-12 16:54:24 -0800  816)  * Hugetlbfs is not reclaimable; therefore its i_mmap_rwsem will never
b610ded719186 (Michal Hocko            2013-08-13 16:00:55 -0700  817)  * be taken from reclaim -- unlike regular filesystems. This needs an
88f306b68cbb3 (Kirill A. Shutemov      2016-01-15 16:57:31 -0800  818)  * annotation because huge_pmd_share() does an allocation under hugetlb's
c8c06efa8b552 (Davidlohr Bueso         2014-12-12 16:54:24 -0800  819)  * i_mmap_rwsem.
b610ded719186 (Michal Hocko            2013-08-13 16:00:55 -0700  820)  */
c8c06efa8b552 (Davidlohr Bueso         2014-12-12 16:54:24 -0800  821) static struct lock_class_key hugetlbfs_i_mmap_rwsem_key;
b610ded719186 (Michal Hocko            2013-08-13 16:00:55 -0700  822) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  823) static struct inode *hugetlbfs_get_inode(struct super_block *sb,
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  824) 					struct inode *dir,
18df22524202e (Al Viro                 2011-07-24 23:17:40 -0400  825) 					umode_t mode, dev_t dev)
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  826) {
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  827) 	struct inode *inode;
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  828) 	struct resv_map *resv_map = NULL;
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  829) 
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  830) 	/*
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  831) 	 * Reserve maps are only needed for inodes that can have associated
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  832) 	 * page allocations.
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  833) 	 */
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  834) 	if (S_ISREG(mode) || S_ISLNK(mode)) {
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  835) 		resv_map = resv_map_alloc();
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  836) 		if (!resv_map)
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  837) 			return NULL;
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  838) 	}
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  839) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  840) 	inode = new_inode(sb);
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  841) 	if (inode) {
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  842) 		struct hugetlbfs_inode_info *info = HUGETLBFS_I(inode);
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  843) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  844) 		inode->i_ino = get_next_ino();
21cb47be6fb9e (Christian Brauner       2021-01-21 14:19:25 +0100  845) 		inode_init_owner(&init_user_ns, inode, dir, mode);
c8c06efa8b552 (Davidlohr Bueso         2014-12-12 16:54:24 -0800  846) 		lockdep_set_class(&inode->i_mapping->i_mmap_rwsem,
c8c06efa8b552 (Davidlohr Bueso         2014-12-12 16:54:24 -0800  847) 				&hugetlbfs_i_mmap_rwsem_key);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  848) 		inode->i_mapping->a_ops = &hugetlbfs_aops;
078cd8279e659 (Deepa Dinamani          2016-09-14 07:48:04 -0700  849) 		inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  850) 		inode->i_mapping->private_data = resv_map;
ff62a34210441 (Marc-André Lureau       2018-01-31 16:19:25 -0800  851) 		info->seals = F_SEAL_SEAL;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  852) 		switch (mode & S_IFMT) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  853) 		default:
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  854) 			init_special_inode(inode, mode, dev);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  855) 			break;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  856) 		case S_IFREG:
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  857) 			inode->i_op = &hugetlbfs_inode_operations;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  858) 			inode->i_fop = &hugetlbfs_file_operations;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  859) 			break;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  860) 		case S_IFDIR:
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  861) 			inode->i_op = &hugetlbfs_dir_inode_operations;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  862) 			inode->i_fop = &simple_dir_operations;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  863) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  864) 			/* directory inodes start off with i_nlink == 2 (for "." entry) */
d8c76e6f45c11 (Dave Hansen             2006-09-30 23:29:04 -0700  865) 			inc_nlink(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  866) 			break;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  867) 		case S_IFLNK:
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  868) 			inode->i_op = &page_symlink_inode_operations;
21fc61c73c390 (Al Viro                 2015-11-17 01:07:57 -0500  869) 			inode_nohighmem(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  870) 			break;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  871) 		}
e096d0c7e2e4e (Josh Boyer              2011-08-25 07:48:12 -0400  872) 		lockdep_annotate_inode_mutex_key(inode);
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  873) 	} else {
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  874) 		if (resv_map)
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  875) 			kref_put(&resv_map->refs, resv_map_release);
58b6e5e8f1add (Mike Kravetz            2019-04-05 18:39:06 -0700  876) 	}
9119a41e9091f (Joonsoo Kim             2014-04-03 14:47:25 -0700  877) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  878) 	return inode;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  879) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  880) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  881) /*
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  882)  * File creation. Allocate an inode, and we're done..
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  883)  */
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  884) static int do_hugetlbfs_mknod(struct inode *dir,
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  885) 			struct dentry *dentry,
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  886) 			umode_t mode,
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  887) 			dev_t dev,
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  888) 			bool tmpfile)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  889) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  890) 	struct inode *inode;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  891) 	int error = -ENOSPC;
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  892) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  893) 	inode = hugetlbfs_get_inode(dir->i_sb, dir, mode, dev);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  894) 	if (inode) {
078cd8279e659 (Deepa Dinamani          2016-09-14 07:48:04 -0700  895) 		dir->i_ctime = dir->i_mtime = current_time(dir);
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  896) 		if (tmpfile) {
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  897) 			d_tmpfile(dentry, inode);
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  898) 		} else {
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  899) 			d_instantiate(dentry, inode);
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  900) 			dget(dentry);/* Extra count - pin the dentry in core */
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  901) 		}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  902) 		error = 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  903) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  904) 	return error;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  905) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  906) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  907) static int hugetlbfs_mknod(struct user_namespace *mnt_userns, struct inode *dir,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  908) 			   struct dentry *dentry, umode_t mode, dev_t dev)
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  909) {
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  910) 	return do_hugetlbfs_mknod(dir, dentry, mode, dev, false);
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  911) }
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  912) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  913) static int hugetlbfs_mkdir(struct user_namespace *mnt_userns, struct inode *dir,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  914) 			   struct dentry *dentry, umode_t mode)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  915) {
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  916) 	int retval = hugetlbfs_mknod(&init_user_ns, dir, dentry,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  917) 				     mode | S_IFDIR, 0);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  918) 	if (!retval)
d8c76e6f45c11 (Dave Hansen             2006-09-30 23:29:04 -0700  919) 		inc_nlink(dir);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  920) 	return retval;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  921) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  922) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  923) static int hugetlbfs_create(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  924) 			    struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  925) 			    umode_t mode, bool excl)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  926) {
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  927) 	return hugetlbfs_mknod(&init_user_ns, dir, dentry, mode | S_IFREG, 0);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  928) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  929) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  930) static int hugetlbfs_tmpfile(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  931) 			     struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  932) 			     umode_t mode)
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  933) {
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  934) 	return do_hugetlbfs_mknod(dir, dentry, mode | S_IFREG, 0, true);
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  935) }
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800  936) 
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  937) static int hugetlbfs_symlink(struct user_namespace *mnt_userns,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  938) 			     struct inode *dir, struct dentry *dentry,
549c7297717c3 (Christian Brauner       2021-01-21 14:19:43 +0100  939) 			     const char *symname)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  940) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  941) 	struct inode *inode;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  942) 	int error = -ENOSPC;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  943) 
7d54fa6472609 (Al Viro                 2011-07-24 20:20:48 -0400  944) 	inode = hugetlbfs_get_inode(dir->i_sb, dir, S_IFLNK|S_IRWXUGO, 0);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  945) 	if (inode) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  946) 		int l = strlen(symname)+1;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  947) 		error = page_symlink(inode, symname, l);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  948) 		if (!error) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  949) 			d_instantiate(dentry, inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  950) 			dget(dentry);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  951) 		} else
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  952) 			iput(inode);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  953) 	}
078cd8279e659 (Deepa Dinamani          2016-09-14 07:48:04 -0700  954) 	dir->i_ctime = dir->i_mtime = current_time(dir);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  955) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  956) 	return error;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  957) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700  958) 
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  959) static int hugetlbfs_migrate_page(struct address_space *mapping,
b969c4ab9f182 (Mel Gorman              2012-01-12 17:19:34 -0800  960) 				struct page *newpage, struct page *page,
a6bc32b899223 (Mel Gorman              2012-01-12 17:19:43 -0800  961) 				enum migrate_mode mode)
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  962) {
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  963) 	int rc;
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  964) 
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  965) 	rc = migrate_huge_page_move_mapping(mapping, newpage, page);
78bd52097d042 (Rafael Aquini           2012-12-11 16:02:31 -0800  966) 	if (rc != MIGRATEPAGE_SUCCESS)
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  967) 		return rc;
cb6acd01e2e43 (Mike Kravetz            2019-02-28 16:22:02 -0800  968) 
d6995da311221 (Mike Kravetz            2021-02-24 12:08:51 -0800  969) 	if (hugetlb_page_subpool(page)) {
d6995da311221 (Mike Kravetz            2021-02-24 12:08:51 -0800  970) 		hugetlb_set_page_subpool(newpage, hugetlb_page_subpool(page));
d6995da311221 (Mike Kravetz            2021-02-24 12:08:51 -0800  971) 		hugetlb_set_page_subpool(page, NULL);
cb6acd01e2e43 (Mike Kravetz            2019-02-28 16:22:02 -0800  972) 	}
cb6acd01e2e43 (Mike Kravetz            2019-02-28 16:22:02 -0800  973) 
2916ecc0f9d43 (Jérôme Glisse           2017-09-08 16:12:06 -0700  974) 	if (mode != MIGRATE_SYNC_NO_COPY)
2916ecc0f9d43 (Jérôme Glisse           2017-09-08 16:12:06 -0700  975) 		migrate_page_copy(newpage, page);
2916ecc0f9d43 (Jérôme Glisse           2017-09-08 16:12:06 -0700  976) 	else
2916ecc0f9d43 (Jérôme Glisse           2017-09-08 16:12:06 -0700  977) 		migrate_page_states(newpage, page);
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  978) 
78bd52097d042 (Rafael Aquini           2012-12-11 16:02:31 -0800  979) 	return MIGRATEPAGE_SUCCESS;
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  980) }
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900  981) 
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  982) static int hugetlbfs_error_remove_page(struct address_space *mapping,
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  983) 				struct page *page)
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  984) {
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  985) 	struct inode *inode = mapping->host;
ab615a5b87929 (Mike Kravetz            2017-11-02 15:59:41 -0700  986) 	pgoff_t index = page->index;
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  987) 
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  988) 	remove_huge_page(page);
ab615a5b87929 (Mike Kravetz            2017-11-02 15:59:41 -0700  989) 	if (unlikely(hugetlb_unreserve_pages(inode, index, index + 1, 1)))
ab615a5b87929 (Mike Kravetz            2017-11-02 15:59:41 -0700  990) 		hugetlb_fix_reserve_counts(inode);
ab615a5b87929 (Mike Kravetz            2017-11-02 15:59:41 -0700  991) 
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  992) 	return 0;
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  993) }
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700  994) 
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100  995) /*
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100  996)  * Display the mount options in /proc/mounts.
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100  997)  */
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100  998) static int hugetlbfs_show_options(struct seq_file *m, struct dentry *root)
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100  999) {
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1000) 	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(root->d_sb);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1001) 	struct hugepage_subpool *spool = sbinfo->spool;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1002) 	unsigned long hpage_size = huge_page_size(sbinfo->hstate);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1003) 	unsigned hpage_shift = huge_page_shift(sbinfo->hstate);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1004) 	char mod;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1005) 
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1006) 	if (!uid_eq(sbinfo->uid, GLOBAL_ROOT_UID))
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1007) 		seq_printf(m, ",uid=%u",
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1008) 			   from_kuid_munged(&init_user_ns, sbinfo->uid));
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1009) 	if (!gid_eq(sbinfo->gid, GLOBAL_ROOT_GID))
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1010) 		seq_printf(m, ",gid=%u",
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1011) 			   from_kgid_munged(&init_user_ns, sbinfo->gid));
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1012) 	if (sbinfo->mode != 0755)
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1013) 		seq_printf(m, ",mode=%o", sbinfo->mode);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1014) 	if (sbinfo->max_inodes != -1)
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1015) 		seq_printf(m, ",nr_inodes=%lu", sbinfo->max_inodes);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1016) 
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1017) 	hpage_size /= 1024;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1018) 	mod = 'K';
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1019) 	if (hpage_size >= 1024) {
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1020) 		hpage_size /= 1024;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1021) 		mod = 'M';
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1022) 	}
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1023) 	seq_printf(m, ",pagesize=%lu%c", hpage_size, mod);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1024) 	if (spool) {
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1025) 		if (spool->max_hpages != -1)
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1026) 			seq_printf(m, ",size=%llu",
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1027) 				   (unsigned long long)spool->max_hpages << hpage_shift);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1028) 		if (spool->min_hpages != -1)
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1029) 			seq_printf(m, ",min_size=%llu",
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1030) 				   (unsigned long long)spool->min_hpages << hpage_shift);
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1031) 	}
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1032) 	return 0;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1033) }
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1034) 
726c334223180 (David Howells           2006-06-23 02:02:58 -0700 1035) static int hugetlbfs_statfs(struct dentry *dentry, struct kstatfs *buf)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1036) {
726c334223180 (David Howells           2006-06-23 02:02:58 -0700 1037) 	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(dentry->d_sb);
2b0143b5c986b (David Howells           2015-03-17 22:25:59 +0000 1038) 	struct hstate *h = hstate_inode(d_inode(dentry));
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1039) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1040) 	buf->f_type = HUGETLBFS_MAGIC;
a5516438959d9 (Andi Kleen              2008-07-23 21:27:41 -0700 1041) 	buf->f_bsize = huge_page_size(h);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1042) 	if (sbinfo) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1043) 		spin_lock(&sbinfo->stat_lock);
74a8a65c52618 (David Gibson            2005-11-21 21:32:24 -0800 1044) 		/* If no limits set, just report 0 for max/free/used
74a8a65c52618 (David Gibson            2005-11-21 21:32:24 -0800 1045) 		 * blocks, like simple_statfs() */
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1046) 		if (sbinfo->spool) {
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1047) 			long free_pages;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1048) 
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1049) 			spin_lock(&sbinfo->spool->lock);
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1050) 			buf->f_blocks = sbinfo->spool->max_hpages;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1051) 			free_pages = sbinfo->spool->max_hpages
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1052) 				- sbinfo->spool->used_hpages;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1053) 			buf->f_bavail = buf->f_bfree = free_pages;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1054) 			spin_unlock(&sbinfo->spool->lock);
74a8a65c52618 (David Gibson            2005-11-21 21:32:24 -0800 1055) 			buf->f_files = sbinfo->max_inodes;
74a8a65c52618 (David Gibson            2005-11-21 21:32:24 -0800 1056) 			buf->f_ffree = sbinfo->free_inodes;
74a8a65c52618 (David Gibson            2005-11-21 21:32:24 -0800 1057) 		}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1058) 		spin_unlock(&sbinfo->stat_lock);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1059) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1060) 	buf->f_namelen = NAME_MAX;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1061) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1062) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1063) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1064) static void hugetlbfs_put_super(struct super_block *sb)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1065) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1066) 	struct hugetlbfs_sb_info *sbi = HUGETLBFS_SB(sb);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1067) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1068) 	if (sbi) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1069) 		sb->s_fs_info = NULL;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1070) 
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1071) 		if (sbi->spool)
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1072) 			hugepage_put_subpool(sbi->spool);
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1073) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1074) 		kfree(sbi);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1075) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1076) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1077) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1078) static inline int hugetlbfs_dec_free_inodes(struct hugetlbfs_sb_info *sbinfo)
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1079) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1080) 	if (sbinfo->free_inodes >= 0) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1081) 		spin_lock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1082) 		if (unlikely(!sbinfo->free_inodes)) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1083) 			spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1084) 			return 0;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1085) 		}
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1086) 		sbinfo->free_inodes--;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1087) 		spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1088) 	}
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1089) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1090) 	return 1;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1091) }
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1092) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1093) static void hugetlbfs_inc_free_inodes(struct hugetlbfs_sb_info *sbinfo)
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1094) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1095) 	if (sbinfo->free_inodes >= 0) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1096) 		spin_lock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1097) 		sbinfo->free_inodes++;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1098) 		spin_unlock(&sbinfo->stat_lock);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1099) 	}
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1100) }
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1101) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1102) 
e18b890bb0881 (Christoph Lameter       2006-12-06 20:33:20 -0800 1103) static struct kmem_cache *hugetlbfs_inode_cachep;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1104) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1105) static struct inode *hugetlbfs_alloc_inode(struct super_block *sb)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1106) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1107) 	struct hugetlbfs_sb_info *sbinfo = HUGETLBFS_SB(sb);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1108) 	struct hugetlbfs_inode_info *p;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1109) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1110) 	if (unlikely(!hugetlbfs_dec_free_inodes(sbinfo)))
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1111) 		return NULL;
e94b1766097d5 (Christoph Lameter       2006-12-06 20:33:17 -0800 1112) 	p = kmem_cache_alloc(hugetlbfs_inode_cachep, GFP_KERNEL);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1113) 	if (unlikely(!p)) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1114) 		hugetlbfs_inc_free_inodes(sbinfo);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1115) 		return NULL;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1116) 	}
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1117) 
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1118) 	/*
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1119) 	 * Any time after allocation, hugetlbfs_destroy_inode can be called
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1120) 	 * for the inode.  mpol_free_shared_policy is unconditionally called
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1121) 	 * as part of hugetlbfs_destroy_inode.  So, initialize policy here
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1122) 	 * in case of a quick call to destroy.
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1123) 	 *
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1124) 	 * Note that the policy is initialized even if we are creating a
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1125) 	 * private inode.  This simplifies hugetlbfs_destroy_inode.
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1126) 	 */
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1127) 	mpol_shared_policy_init(&p->policy, NULL);
4742a35d9de74 (Mike Kravetz            2017-03-31 15:12:01 -0700 1128) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1129) 	return &p->vfs_inode;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1130) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1131) 
b62de32257970 (Al Viro                 2019-04-15 23:16:38 -0400 1132) static void hugetlbfs_free_inode(struct inode *inode)
fa0d7e3de6d6f (Nicholas Piggin         2011-01-07 17:49:49 +1100 1133) {
fa0d7e3de6d6f (Nicholas Piggin         2011-01-07 17:49:49 +1100 1134) 	kmem_cache_free(hugetlbfs_inode_cachep, HUGETLBFS_I(inode));
fa0d7e3de6d6f (Nicholas Piggin         2011-01-07 17:49:49 +1100 1135) }
fa0d7e3de6d6f (Nicholas Piggin         2011-01-07 17:49:49 +1100 1136) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1137) static void hugetlbfs_destroy_inode(struct inode *inode)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1138) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1139) 	hugetlbfs_inc_free_inodes(HUGETLBFS_SB(inode->i_sb));
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1140) 	mpol_free_shared_policy(&HUGETLBFS_I(inode)->policy);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1141) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1142) 
f5e54d6e53a20 (Christoph Hellwig       2006-06-28 04:26:44 -0700 1143) static const struct address_space_operations hugetlbfs_aops = {
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700 1144) 	.write_begin	= hugetlbfs_write_begin,
800d15a53e7d1 (Nicholas Piggin         2007-10-16 01:25:03 -0700 1145) 	.write_end	= hugetlbfs_write_end,
a4fa34cdcd182 (Mike Kravetz            2021-02-24 12:09:58 -0800 1146) 	.set_page_dirty	=  __set_page_dirty_no_writeback,
290408d4a2500 (Naoya Horiguchi         2010-09-08 10:19:35 +0900 1147) 	.migratepage    = hugetlbfs_migrate_page,
78bb920344b8a (Naoya Horiguchi         2017-07-10 15:47:50 -0700 1148) 	.error_remove_page	= hugetlbfs_error_remove_page,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1149) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1150) 
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1151) 
51cc50685a427 (Alexey Dobriyan         2008-07-25 19:45:34 -0700 1152) static void init_once(void *foo)
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1153) {
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1154) 	struct hugetlbfs_inode_info *ei = (struct hugetlbfs_inode_info *)foo;
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1155) 
a35afb830f8d7 (Christoph Lameter       2007-05-16 22:10:57 -0700 1156) 	inode_init_once(&ei->vfs_inode);
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1157) }
96527980d4cb8 (Christoph Hellwig       2005-10-29 18:16:42 -0700 1158) 
4b6f5d20b04dc (Arjan van de Ven        2006-03-28 01:56:42 -0800 1159) const struct file_operations hugetlbfs_file_operations = {
34d0640e26892 (Al Viro                 2015-04-03 11:31:35 -0400 1160) 	.read_iter		= hugetlbfs_read_iter,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1161) 	.mmap			= hugetlbfs_file_mmap,
1b061d9247f71 (Christoph Hellwig       2010-05-26 17:53:41 +0200 1162) 	.fsync			= noop_fsync,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1163) 	.get_unmapped_area	= hugetlb_get_unmapped_area,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700 1164) 	.llseek			= default_llseek,
70c3547e36f5c (Mike Kravetz            2015-09-08 15:01:54 -0700 1165) 	.fallocate		= hugetlbfs_fallocate,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1166) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1167) 
92e1d5be91a0e (Arjan van de Ven        2007-02-12 00:55:39 -0800 1168) static const struct inode_operations hugetlbfs_dir_inode_operations = {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1169) 	.create		= hugetlbfs_create,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1170) 	.lookup		= simple_lookup,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1171) 	.link		= simple_link,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1172) 	.unlink		= simple_unlink,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1173) 	.symlink	= hugetlbfs_symlink,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1174) 	.mkdir		= hugetlbfs_mkdir,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1175) 	.rmdir		= simple_rmdir,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1176) 	.mknod		= hugetlbfs_mknod,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1177) 	.rename		= simple_rename,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1178) 	.setattr	= hugetlbfs_setattr,
1ab5b82f540b3 (Piotr Sarna             2019-11-30 17:56:43 -0800 1179) 	.tmpfile	= hugetlbfs_tmpfile,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1180) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1181) 
92e1d5be91a0e (Arjan van de Ven        2007-02-12 00:55:39 -0800 1182) static const struct inode_operations hugetlbfs_inode_operations = {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1183) 	.setattr	= hugetlbfs_setattr,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1184) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1185) 
ee9b6d61a2a43 (Josef 'Jeff' Sipek      2007-02-12 00:55:41 -0800 1186) static const struct super_operations hugetlbfs_ops = {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1187) 	.alloc_inode    = hugetlbfs_alloc_inode,
b62de32257970 (Al Viro                 2019-04-15 23:16:38 -0400 1188) 	.free_inode     = hugetlbfs_free_inode,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1189) 	.destroy_inode  = hugetlbfs_destroy_inode,
2bbbda308f5ca (Al Viro                 2010-06-04 19:52:12 -0400 1190) 	.evict_inode	= hugetlbfs_evict_inode,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1191) 	.statfs		= hugetlbfs_statfs,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1192) 	.put_super	= hugetlbfs_put_super,
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1193) 	.show_options	= hugetlbfs_show_options,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1194) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1195) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1196) /*
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1197)  * Convert size option passed from command line to number of huge pages
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1198)  * in the pool specified by hstate.  Size option could be in bytes
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1199)  * (val_type == SIZE_STD) or percentage of the pool (val_type == SIZE_PERCENT).
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1200)  */
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1201) static long
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1202) hugetlbfs_size_to_hpages(struct hstate *h, unsigned long long size_opt,
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1203) 			 enum hugetlbfs_size_type val_type)
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1204) {
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1205) 	if (val_type == NO_SIZE)
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1206) 		return -1;
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1207) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1208) 	if (val_type == SIZE_PERCENT) {
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1209) 		size_opt <<= huge_page_shift(h);
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1210) 		size_opt *= h->max_huge_pages;
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1211) 		do_div(size_opt, 100);
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1212) 	}
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1213) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1214) 	size_opt >>= huge_page_shift(h);
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1215) 	return size_opt;
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1216) }
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1217) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1218) /*
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1219)  * Parse one mount parameter.
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1220)  */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1221) static int hugetlbfs_parse_param(struct fs_context *fc, struct fs_parameter *param)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1222) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1223) 	struct hugetlbfs_fs_context *ctx = fc->fs_private;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1224) 	struct fs_parse_result result;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1225) 	char *rest;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1226) 	unsigned long ps;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1227) 	int opt;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1228) 
d7167b149943e (Al Viro                 2019-09-07 07:23:15 -0400 1229) 	opt = fs_parse(fc, hugetlb_fs_parameters, param, &result);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1230) 	if (opt < 0)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1231) 		return opt;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1232) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1233) 	switch (opt) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1234) 	case Opt_uid:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1235) 		ctx->uid = make_kuid(current_user_ns(), result.uint_32);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1236) 		if (!uid_valid(ctx->uid))
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1237) 			goto bad_val;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1238) 		return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1239) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1240) 	case Opt_gid:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1241) 		ctx->gid = make_kgid(current_user_ns(), result.uint_32);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1242) 		if (!gid_valid(ctx->gid))
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1243) 			goto bad_val;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1244) 		return 0;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1245) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1246) 	case Opt_mode:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1247) 		ctx->mode = result.uint_32 & 01777U;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1248) 		return 0;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1249) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1250) 	case Opt_size:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1251) 		/* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1252) 		if (!isdigit(param->string[0]))
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1253) 			goto bad_val;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1254) 		ctx->max_size_opt = memparse(param->string, &rest);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1255) 		ctx->max_val_type = SIZE_STD;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1256) 		if (*rest == '%')
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1257) 			ctx->max_val_type = SIZE_PERCENT;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1258) 		return 0;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1259) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1260) 	case Opt_nr_inodes:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1261) 		/* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1262) 		if (!isdigit(param->string[0]))
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1263) 			goto bad_val;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1264) 		ctx->nr_inodes = memparse(param->string, &rest);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1265) 		return 0;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1266) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1267) 	case Opt_pagesize:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1268) 		ps = memparse(param->string, &rest);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1269) 		ctx->hstate = size_to_hstate(ps);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1270) 		if (!ctx->hstate) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1271) 			pr_err("Unsupported page size %lu MB\n", ps >> 20);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1272) 			return -EINVAL;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1273) 		}
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1274) 		return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1275) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1276) 	case Opt_min_size:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1277) 		/* memparse() will accept a K/M/G without a digit */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1278) 		if (!isdigit(param->string[0]))
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1279) 			goto bad_val;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1280) 		ctx->min_size_opt = memparse(param->string, &rest);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1281) 		ctx->min_val_type = SIZE_STD;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1282) 		if (*rest == '%')
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1283) 			ctx->min_val_type = SIZE_PERCENT;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1284) 		return 0;
e73a75fa7f062 (Randy Dunlap            2007-07-15 23:40:52 -0700 1285) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1286) 	default:
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1287) 		return -EINVAL;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1288) 	}
a137e1cc6d6e7 (Andi Kleen              2008-07-23 21:27:43 -0700 1289) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1290) bad_val:
b5db30cfb9793 (Al Viro                 2019-12-21 21:34:06 -0500 1291) 	return invalfc(fc, "Bad value '%s' for mount option '%s'\n",
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1292) 		      param->string, param->key);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1293) }
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1294) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1295) /*
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1296)  * Validate the parsed options.
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1297)  */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1298) static int hugetlbfs_validate(struct fs_context *fc)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1299) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1300) 	struct hugetlbfs_fs_context *ctx = fc->fs_private;
a137e1cc6d6e7 (Andi Kleen              2008-07-23 21:27:43 -0700 1301) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1302) 	/*
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1303) 	 * Use huge page pool size (in hstate) to convert the size
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1304) 	 * options to number of huge pages.  If NO_SIZE, -1 is returned.
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1305) 	 */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1306) 	ctx->max_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1307) 						   ctx->max_size_opt,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1308) 						   ctx->max_val_type);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1309) 	ctx->min_hpages = hugetlbfs_size_to_hpages(ctx->hstate,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1310) 						   ctx->min_size_opt,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1311) 						   ctx->min_val_type);
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1312) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1313) 	/*
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1314) 	 * If max_size was specified, then min_size must be smaller
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1315) 	 */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1316) 	if (ctx->max_val_type > NO_SIZE &&
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1317) 	    ctx->min_hpages > ctx->max_hpages) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1318) 		pr_err("Minimum size can not be greater than maximum size\n");
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1319) 		return -EINVAL;
a137e1cc6d6e7 (Andi Kleen              2008-07-23 21:27:43 -0700 1320) 	}
a137e1cc6d6e7 (Andi Kleen              2008-07-23 21:27:43 -0700 1321) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1322) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1323) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1324) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1325) static int
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1326) hugetlbfs_fill_super(struct super_block *sb, struct fs_context *fc)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1327) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1328) 	struct hugetlbfs_fs_context *ctx = fc->fs_private;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1329) 	struct hugetlbfs_sb_info *sbinfo;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1330) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1331) 	sbinfo = kmalloc(sizeof(struct hugetlbfs_sb_info), GFP_KERNEL);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1332) 	if (!sbinfo)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1333) 		return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1334) 	sb->s_fs_info = sbinfo;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1335) 	spin_lock_init(&sbinfo->stat_lock);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1336) 	sbinfo->hstate		= ctx->hstate;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1337) 	sbinfo->max_inodes	= ctx->nr_inodes;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1338) 	sbinfo->free_inodes	= ctx->nr_inodes;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1339) 	sbinfo->spool		= NULL;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1340) 	sbinfo->uid		= ctx->uid;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1341) 	sbinfo->gid		= ctx->gid;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1342) 	sbinfo->mode		= ctx->mode;
4a25220d4e43b (David Howells           2017-07-05 16:24:18 +0100 1343) 
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1344) 	/*
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1345) 	 * Allocate and initialize subpool if maximum or minimum size is
1935ebd3cf6c4 (Miaohe Lin              2021-02-24 12:10:21 -0800 1346) 	 * specified.  Any needed reservations (for minimum size) are taken
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1347) 	 * taken when the subpool is created.
7ca02d0ae586f (Mike Kravetz            2015-04-15 16:13:42 -0700 1348) 	 */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1349) 	if (ctx->max_hpages != -1 || ctx->min_hpages != -1) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1350) 		sbinfo->spool = hugepage_new_subpool(ctx->hstate,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1351) 						     ctx->max_hpages,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1352) 						     ctx->min_hpages);
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1353) 		if (!sbinfo->spool)
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1354) 			goto out_free;
90481622d7571 (David Gibson            2012-03-21 16:34:12 -0700 1355) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1356) 	sb->s_maxbytes = MAX_LFS_FILESIZE;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1357) 	sb->s_blocksize = huge_page_size(ctx->hstate);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1358) 	sb->s_blocksize_bits = huge_page_shift(ctx->hstate);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1359) 	sb->s_magic = HUGETLBFS_MAGIC;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1360) 	sb->s_op = &hugetlbfs_ops;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1361) 	sb->s_time_gran = 1;
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1362) 
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1363) 	/*
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1364) 	 * Due to the special and limited functionality of hugetlbfs, it does
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1365) 	 * not work well as a stacking filesystem.
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1366) 	 */
15568299b7d99 (Mike Kravetz            2020-08-11 18:31:35 -0700 1367) 	sb->s_stack_depth = FILESYSTEM_MAX_STACK_DEPTH;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1368) 	sb->s_root = d_make_root(hugetlbfs_get_root(sb, ctx));
48fde701aff66 (Al Viro                 2012-01-08 22:15:13 -0500 1369) 	if (!sb->s_root)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1370) 		goto out_free;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1371) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1372) out_free:
6e6870d4fd19e (Fabian Frederick        2014-06-04 16:10:40 -0700 1373) 	kfree(sbinfo->spool);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1374) 	kfree(sbinfo);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1375) 	return -ENOMEM;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1376) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1377) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1378) static int hugetlbfs_get_tree(struct fs_context *fc)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1379) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1380) 	int err = hugetlbfs_validate(fc);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1381) 	if (err)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1382) 		return err;
2ac295d4f0c09 (Al Viro                 2019-06-01 20:48:55 -0400 1383) 	return get_tree_nodev(fc, hugetlbfs_fill_super);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1384) }
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1385) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1386) static void hugetlbfs_fs_context_free(struct fs_context *fc)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1387) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1388) 	kfree(fc->fs_private);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1389) }
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1390) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1391) static const struct fs_context_operations hugetlbfs_fs_context_ops = {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1392) 	.free		= hugetlbfs_fs_context_free,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1393) 	.parse_param	= hugetlbfs_parse_param,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1394) 	.get_tree	= hugetlbfs_get_tree,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1395) };
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1396) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1397) static int hugetlbfs_init_fs_context(struct fs_context *fc)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1398) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1399) 	struct hugetlbfs_fs_context *ctx;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1400) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1401) 	ctx = kzalloc(sizeof(struct hugetlbfs_fs_context), GFP_KERNEL);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1402) 	if (!ctx)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1403) 		return -ENOMEM;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1404) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1405) 	ctx->max_hpages	= -1; /* No limit on size by default */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1406) 	ctx->nr_inodes	= -1; /* No limit on number of inodes by default */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1407) 	ctx->uid	= current_fsuid();
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1408) 	ctx->gid	= current_fsgid();
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1409) 	ctx->mode	= 0755;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1410) 	ctx->hstate	= &default_hstate;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1411) 	ctx->min_hpages	= -1; /* No default minimum size */
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1412) 	ctx->max_val_type = NO_SIZE;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1413) 	ctx->min_val_type = NO_SIZE;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1414) 	fc->fs_private = ctx;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1415) 	fc->ops	= &hugetlbfs_fs_context_ops;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1416) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1417) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1418) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1419) static struct file_system_type hugetlbfs_fs_type = {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1420) 	.name			= "hugetlbfs",
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1421) 	.init_fs_context	= hugetlbfs_init_fs_context,
d7167b149943e (Al Viro                 2019-09-07 07:23:15 -0400 1422) 	.parameters		= hugetlb_fs_parameters,
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1423) 	.kill_sb		= kill_litter_super,
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1424) };
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1425) 
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1426) static struct vfsmount *hugetlbfs_vfsmount[HUGE_MAX_HSTATE];
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1427) 
ef1ff6b8c0895 (From: Mel Gorman        2009-09-23 15:56:05 -0700 1428) static int can_do_hugetlb_shm(void)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1429) {
a0eb3a05a8cbe (Eric W. Biederman       2012-02-07 16:19:25 -0800 1430) 	kgid_t shm_group;
a0eb3a05a8cbe (Eric W. Biederman       2012-02-07 16:19:25 -0800 1431) 	shm_group = make_kgid(&init_user_ns, sysctl_hugetlb_shm_group);
a0eb3a05a8cbe (Eric W. Biederman       2012-02-07 16:19:25 -0800 1432) 	return capable(CAP_IPC_LOCK) || in_group_p(shm_group);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1433) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1434) 
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1435) static int get_hstate_idx(int page_size_log)
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1436) {
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1437) 	struct hstate *h = hstate_sizelog(page_size_log);
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1438) 
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1439) 	if (!h)
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1440) 		return -1;
04adbc3f7bff4 (Miaohe Lin              2021-05-04 18:33:22 -0700 1441) 	return hstate_index(h);
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1442) }
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1443) 
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1444) /*
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1445)  * Note that size should be aligned to proper hugepage size in caller side,
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1446)  * otherwise hugetlb_reserve_pages reserves one less hugepages than intended.
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1447)  */
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1448) struct file *hugetlb_file_setup(const char *name, size_t size,
af73e4d9506d3 (Naoya Horiguchi         2013-05-07 16:18:13 -0700 1449) 				vm_flags_t acctflag, struct user_struct **user,
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1450) 				int creat_flags, int page_size_log)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1451) {
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1452) 	struct inode *inode;
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1453) 	struct vfsmount *mnt;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1454) 	int hstate_idx;
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1455) 	struct file *file;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1456) 
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1457) 	hstate_idx = get_hstate_idx(page_size_log);
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1458) 	if (hstate_idx < 0)
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1459) 		return ERR_PTR(-ENODEV);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1460) 
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1461) 	*user = NULL;
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1462) 	mnt = hugetlbfs_vfsmount[hstate_idx];
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1463) 	if (!mnt)
5bc98594d5967 (Akinobu Mita            2007-05-06 14:50:18 -0700 1464) 		return ERR_PTR(-ENOENT);
5bc98594d5967 (Akinobu Mita            2007-05-06 14:50:18 -0700 1465) 
ef1ff6b8c0895 (From: Mel Gorman        2009-09-23 15:56:05 -0700 1466) 	if (creat_flags == HUGETLB_SHMFS_INODE && !can_do_hugetlb_shm()) {
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1467) 		*user = current_user();
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1468) 		if (user_shm_lock(size, *user)) {
21a3c273f88c9 (David Rientjes          2012-03-21 16:34:13 -0700 1469) 			task_lock(current);
9b857d26d08f0 (Andrew Morton           2014-06-04 16:07:21 -0700 1470) 			pr_warn_once("%s (%d): Using mlock ulimits for SHM_HUGETLB is deprecated\n",
21a3c273f88c9 (David Rientjes          2012-03-21 16:34:13 -0700 1471) 				current->comm, current->pid);
21a3c273f88c9 (David Rientjes          2012-03-21 16:34:13 -0700 1472) 			task_unlock(current);
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1473) 		} else {
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1474) 			*user = NULL;
2584e517320bd (Ravikiran G Thirumalai  2009-03-31 15:21:26 -0700 1475) 			return ERR_PTR(-EPERM);
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1476) 		}
2584e517320bd (Ravikiran G Thirumalai  2009-03-31 15:21:26 -0700 1477) 	}
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1478) 
39b6525274574 (Anatol Pomozov          2012-09-12 20:11:55 -0700 1479) 	file = ERR_PTR(-ENOSPC);
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1480) 	inode = hugetlbfs_get_inode(mnt->mnt_sb, NULL, S_IFREG | S_IRWXUGO, 0);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1481) 	if (!inode)
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1482) 		goto out;
e1832f2923ec9 (Stephen Smalley         2015-08-06 15:46:55 -0700 1483) 	if (creat_flags == HUGETLB_SHMFS_INODE)
e1832f2923ec9 (Stephen Smalley         2015-08-06 15:46:55 -0700 1484) 		inode->i_flags |= S_PRIVATE;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1485) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1486) 	inode->i_size = size;
6d6b77f163c7e (Miklos Szeredi          2011-10-28 14:13:28 +0200 1487) 	clear_nlink(inode);
ce8d2cdf3d2b7 (Dave Hansen             2007-10-16 23:31:13 -0700 1488) 
33b8f84a4ee78 (Mike Kravetz            2021-02-24 12:09:54 -0800 1489) 	if (!hugetlb_reserve_pages(inode, 0,
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1490) 			size >> huge_page_shift(hstate_inode(inode)), NULL,
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1491) 			acctflag))
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1492) 		file = ERR_PTR(-ENOMEM);
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1493) 	else
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1494) 		file = alloc_file_pseudo(inode, mnt, name, O_RDWR,
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1495) 					&hugetlbfs_file_operations);
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1496) 	if (!IS_ERR(file))
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1497) 		return file;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1498) 
b45b5bd65f668 (David Gibson            2006-03-22 00:08:55 -0800 1499) 	iput(inode);
e68375c850b0d (Al Viro                 2018-06-09 09:50:46 -0400 1500) out:
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1501) 	if (*user) {
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1502) 		user_shm_unlock(size, *user);
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1503) 		*user = NULL;
353d5c30c6665 (Hugh Dickins            2009-08-24 16:30:28 +0100 1504) 	}
39b6525274574 (Anatol Pomozov          2012-09-12 20:11:55 -0700 1505) 	return file;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1506) }
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1507) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1508) static struct vfsmount *__init mount_one_hugetlbfs(struct hstate *h)
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1509) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1510) 	struct fs_context *fc;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1511) 	struct vfsmount *mnt;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1512) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1513) 	fc = fs_context_for_mount(&hugetlbfs_fs_type, SB_KERNMOUNT);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1514) 	if (IS_ERR(fc)) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1515) 		mnt = ERR_CAST(fc);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1516) 	} else {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1517) 		struct hugetlbfs_fs_context *ctx = fc->fs_private;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1518) 		ctx->hstate = h;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1519) 		mnt = fc_mount(fc);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1520) 		put_fs_context(fc);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1521) 	}
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1522) 	if (IS_ERR(mnt))
a25fddced835a (Miaohe Lin              2021-02-24 12:10:14 -0800 1523) 		pr_err("Cannot mount internal hugetlbfs for page size %luK",
a25fddced835a (Miaohe Lin              2021-02-24 12:10:14 -0800 1524) 		       huge_page_size(h) >> 10);
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1525) 	return mnt;
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1526) }
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1527) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1528) static int __init init_hugetlbfs_fs(void)
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1529) {
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1530) 	struct vfsmount *mnt;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1531) 	struct hstate *h;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1532) 	int error;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1533) 	int i;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1534) 
457c1b27ed56e (Nishanth Aravamudan     2014-05-06 12:50:00 -0700 1535) 	if (!hugepages_supported()) {
9b857d26d08f0 (Andrew Morton           2014-06-04 16:07:21 -0700 1536) 		pr_info("disabling because there are no supported hugepage sizes\n");
457c1b27ed56e (Nishanth Aravamudan     2014-05-06 12:50:00 -0700 1537) 		return -ENOTSUPP;
457c1b27ed56e (Nishanth Aravamudan     2014-05-06 12:50:00 -0700 1538) 	}
457c1b27ed56e (Nishanth Aravamudan     2014-05-06 12:50:00 -0700 1539) 
d1d5e05ffdc11 (Hillf Danton            2012-03-21 16:34:15 -0700 1540) 	error = -ENOMEM;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1541) 	hugetlbfs_inode_cachep = kmem_cache_create("hugetlbfs_inode_cache",
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1542) 					sizeof(struct hugetlbfs_inode_info),
5d097056c9a01 (Vladimir Davydov        2016-01-14 15:18:21 -0800 1543) 					0, SLAB_ACCOUNT, init_once);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1544) 	if (hugetlbfs_inode_cachep == NULL)
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1545) 		goto out;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1546) 
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1547) 	error = register_filesystem(&hugetlbfs_fs_type);
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1548) 	if (error)
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1549) 		goto out_free;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1550) 
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1551) 	/* default hstate mount is required */
3b2275a8d83a2 (Miaohe Lin              2021-02-24 12:10:04 -0800 1552) 	mnt = mount_one_hugetlbfs(&default_hstate);
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1553) 	if (IS_ERR(mnt)) {
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1554) 		error = PTR_ERR(mnt);
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1555) 		goto out_unreg;
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1556) 	}
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1557) 	hugetlbfs_vfsmount[default_hstate_idx] = mnt;
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1558) 
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1559) 	/* other hstates are optional */
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1560) 	i = 0;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1561) 	for_each_hstate(h) {
15f0ec941f4f9 (Jan Stancek             2020-01-03 18:37:18 +0100 1562) 		if (i == default_hstate_idx) {
15f0ec941f4f9 (Jan Stancek             2020-01-03 18:37:18 +0100 1563) 			i++;
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1564) 			continue;
15f0ec941f4f9 (Jan Stancek             2020-01-03 18:37:18 +0100 1565) 		}
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1566) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1567) 		mnt = mount_one_hugetlbfs(h);
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1568) 		if (IS_ERR(mnt))
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1569) 			hugetlbfs_vfsmount[i] = NULL;
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1570) 		else
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1571) 			hugetlbfs_vfsmount[i] = mnt;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1572) 		i++;
42d7395feb56f (Andi Kleen              2012-12-11 16:01:34 -0800 1573) 	}
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1574) 
32021982a324d (David Howells           2018-11-01 23:07:26 +0000 1575) 	return 0;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1576) 
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1577)  out_unreg:
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1578) 	(void)unregister_filesystem(&hugetlbfs_fs_type);
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1579)  out_free:
d1d5e05ffdc11 (Hillf Danton            2012-03-21 16:34:15 -0700 1580) 	kmem_cache_destroy(hugetlbfs_inode_cachep);
8fc312b32b25c (Mike Kravetz            2019-11-30 17:56:34 -0800 1581)  out:
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1582) 	return error;
^1da177e4c3f4 (Linus Torvalds          2005-04-16 15:20:36 -0700 1583) }
3e89e1c5ea842 (Paul Gortmaker          2016-01-14 15:21:52 -0800 1584) fs_initcall(init_hugetlbfs_fs)