Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * vdso setup for s390
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *  Copyright IBM Corp. 2008
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *  Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) #include <linux/stddef.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) #include <linux/unistd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <linux/user.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <linux/elf.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <linux/security.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <linux/compat.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <linux/binfmts.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <vdso/datapage.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <asm/asm-offsets.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <asm/mmu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <asm/vdso.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <asm/facility.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) extern char vdso64_start, vdso64_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) static void *vdso64_kbase = &vdso64_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) static unsigned int vdso64_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) static struct page **vdso64_pagelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39)  * Should the kernel map a VDSO page into processes and pass its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40)  * address down to glibc upon exec()?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) unsigned int __read_mostly vdso_enabled = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) static vm_fault_t vdso_fault(const struct vm_special_mapping *sm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 		      struct vm_area_struct *vma, struct vm_fault *vmf)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	struct page **vdso_pagelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	unsigned long vdso_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	vdso_pagelist = vdso64_pagelist;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	vdso_pages = vdso64_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	if (vmf->pgoff >= vdso_pages)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		return VM_FAULT_SIGBUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 	vmf->page = vdso_pagelist[vmf->pgoff];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	get_page(vmf->page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) static int vdso_mremap(const struct vm_special_mapping *sm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 		       struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 	unsigned long vdso_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 	vdso_pages = vdso64_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	if ((vdso_pages << PAGE_SHIFT) != vma->vm_end - vma->vm_start)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		return -EINVAL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 	if (WARN_ON_ONCE(current->mm != vma->vm_mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		return -EFAULT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 	current->mm->context.vdso_base = vma->vm_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) static const struct vm_special_mapping vdso_mapping = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	.name = "[vdso]",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	.fault = vdso_fault,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	.mremap = vdso_mremap,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) static int __init vdso_setup(char *str)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	bool enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 	if (!kstrtobool(str, &enabled))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		vdso_enabled = enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) 	return 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) __setup("vdso=", vdso_setup);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95)  * The vdso data page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) static union {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	struct vdso_data	data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 	u8			page[PAGE_SIZE];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) } vdso_data_store __page_aligned_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) struct vdso_data *vdso_data = (struct vdso_data *)&vdso_data_store.data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103)  * Allocate/free per cpu vdso data.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define SEGMENT_ORDER	2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) int vdso_alloc_per_cpu(struct lowcore *lowcore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 	unsigned long segment_table, page_table, page_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	struct vdso_per_cpu_data *vd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 	segment_table = __get_free_pages(GFP_KERNEL, SEGMENT_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 	page_table = get_zeroed_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	page_frame = get_zeroed_page(GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 	if (!segment_table || !page_table || !page_frame)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 		goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	arch_set_page_dat(virt_to_page(segment_table), SEGMENT_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	arch_set_page_dat(virt_to_page(page_table), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	/* Initialize per-cpu vdso data page */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 	vd = (struct vdso_per_cpu_data *) page_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	vd->cpu_nr = lowcore->cpu_nr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	vd->node_id = cpu_to_node(vd->cpu_nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	/* Set up page table for the vdso address space */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 	memset64((u64 *)segment_table, _SEGMENT_ENTRY_EMPTY, _CRST_ENTRIES);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	memset64((u64 *)page_table, _PAGE_INVALID, PTRS_PER_PTE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	*(unsigned long *) segment_table = _SEGMENT_ENTRY + page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	*(unsigned long *) page_table = _PAGE_PROTECT + page_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 	lowcore->vdso_asce = segment_table +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 		_ASCE_TABLE_LENGTH + _ASCE_USER_BITS + _ASCE_TYPE_SEGMENT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 	lowcore->vdso_per_cpu_data = page_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	free_page(page_frame);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 	free_page(page_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	free_pages(segment_table, SEGMENT_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 	return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) void vdso_free_per_cpu(struct lowcore *lowcore)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	unsigned long segment_table, page_table, page_frame;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	segment_table = lowcore->vdso_asce & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	page_table = *(unsigned long *) segment_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	page_frame = *(unsigned long *) page_table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	free_page(page_frame);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 	free_page(page_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 	free_pages(segment_table, SEGMENT_ORDER);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)  * This is called from binfmt_elf, we create the special vma for the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160)  * vDSO and insert it into the mm struct tree
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) int arch_setup_additional_pages(struct linux_binprm *bprm, int uses_interp)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	struct mm_struct *mm = current->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 	struct vm_area_struct *vma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	unsigned long vdso_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 	unsigned long vdso_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	if (!vdso_enabled)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	if (is_compat_task())
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 	vdso_pages = vdso64_pages;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 	 * vDSO has a problem and was disabled, just don't "enable" it for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	 * the process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 	if (vdso_pages == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	 * pick a base address for the vDSO in process space. We try to put
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	 * it at vdso_base which is the "natural" base for it, but we might
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	 * fail and end up putting it elsewhere.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	if (mmap_write_lock_killable(mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		return -EINTR;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	vdso_base = get_unmapped_area(NULL, 0, vdso_pages << PAGE_SHIFT, 0, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	if (IS_ERR_VALUE(vdso_base)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 		rc = vdso_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 		goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	 * our vma flags don't have VM_WRITE so by default, the process
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	 * isn't allowed to write those pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	 * gdb can break that with ptrace interface, and thus trigger COW
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 	 * on those pages but it's then your responsibility to never do that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 	 * on the "data" page of the vDSO or you'll stop getting kernel
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	 * updates and your nice userland gettimeofday will be totally dead.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	 * It's fine to use that for setting breakpoints in the vDSO code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) 	 * pages though.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) 	vma = _install_special_mapping(mm, vdso_base, vdso_pages << PAGE_SHIFT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 				       VM_READ|VM_EXEC|
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 				       VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 				       &vdso_mapping);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	if (IS_ERR(vma)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		rc = PTR_ERR(vma);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		goto out_up;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	current->mm->context.vdso_base = vdso_base;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 	rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) out_up:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	mmap_write_unlock(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) static int __init vdso_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	/* Calculate the size of the 64 bit vDSO */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	vdso64_pages = ((&vdso64_end - &vdso64_start
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 			 + PAGE_SIZE - 1) >> PAGE_SHIFT) + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	/* Make sure pages are in the correct state */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	vdso64_pagelist = kcalloc(vdso64_pages + 1, sizeof(struct page *),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 				  GFP_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 	BUG_ON(vdso64_pagelist == NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	for (i = 0; i < vdso64_pages - 1; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		struct page *pg = virt_to_page(vdso64_kbase + i*PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 		get_page(pg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		vdso64_pagelist[i] = pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	vdso64_pagelist[vdso64_pages - 1] = virt_to_page(vdso_data);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	vdso64_pagelist[vdso64_pages] = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	if (vdso_alloc_per_cpu(&S390_lowcore))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		BUG();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	get_page(virt_to_page(vdso_data));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) early_initcall(vdso_init);