Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) // SPDX-License-Identifier: GPL-2.0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  * Virtual Memory Map support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  * (C) 2007 sgi. Christoph Lameter.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * Virtual memory maps allow VM primitives pfn_to_page, page_to_pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * virt_to_page, page_address() to be implemented as a base offset
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  * calculation without memory access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * However, virtual mappings need a page table and TLBs. Many Linux
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * architectures already map their physical space using 1-1 mappings
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * via TLBs. For those arches the virtual memory map is essentially
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * for free if we use the same page size as the 1-1 mappings. In that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * case the overhead consists of a few additional pages that are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  * allocated to create a view of memory for vmemmap.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18)  * The architecture is expected to provide a vmemmap_populate() function
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19)  * to instantiate the mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #include <linux/mmzone.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) #include <linux/memremap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) #include <linux/slab.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) #include <linux/vmalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) #include <asm/dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34)  * Allocate a block of memory to be used to back the virtual memory map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35)  * or to back the page tables that are used to create the mapping.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36)  * Uses the main allocators if they are available, else bootmem.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) static void * __ref __earlyonly_bootmem_alloc(int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 				unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 				unsigned long align,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 				unsigned long goal)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 	return memblock_alloc_try_nid_raw(size, align, goal,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 					       MEMBLOCK_ALLOC_ACCESSIBLE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) void * __meminit vmemmap_alloc_block(unsigned long size, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 	/* If the main allocator is up use that, fallback to bootmem. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	if (slab_is_available()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 		gfp_t gfp_mask = GFP_KERNEL|__GFP_RETRY_MAYFAIL|__GFP_NOWARN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 		int order = get_order(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 		static bool warned;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) 		struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 		page = alloc_pages_node(node, gfp_mask, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 		if (page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 			return page_address(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		if (!warned) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 			warn_alloc(gfp_mask & ~__GFP_NOWARN, NULL,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 				   "vmemmap alloc failure: order:%u", order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) 			warned = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	} else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		return __earlyonly_bootmem_alloc(node, size, size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 				__pa(MAX_DMA_ADDRESS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) static void * __meminit altmap_alloc_block_buf(unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 					       struct vmem_altmap *altmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) /* need to make sure size is all the same during early stage */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) void * __meminit vmemmap_alloc_block_buf(unsigned long size, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 					 struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	void *ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	if (altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 		return altmap_alloc_block_buf(size, altmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 	ptr = sparse_buffer_alloc(size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 	if (!ptr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 		ptr = vmemmap_alloc_block(size, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	return ptr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) static unsigned long __meminit vmem_altmap_next_pfn(struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	return altmap->base_pfn + altmap->reserve + altmap->alloc
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 		+ altmap->align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) static unsigned long __meminit vmem_altmap_nr_free(struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	unsigned long allocated = altmap->alloc + altmap->align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	if (altmap->free > allocated)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		return altmap->free - allocated;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) static void * __meminit altmap_alloc_block_buf(unsigned long size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 					       struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 	unsigned long pfn, nr_pfns, nr_align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 	if (size & ~PAGE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 		pr_warn_once("%s: allocations must be multiple of PAGE_SIZE (%ld)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 				__func__, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 	pfn = vmem_altmap_next_pfn(altmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 	nr_pfns = size >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 	nr_align = 1UL << find_first_bit(&nr_pfns, BITS_PER_LONG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 	nr_align = ALIGN(pfn, nr_align) - pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	if (nr_pfns + nr_align > vmem_altmap_nr_free(altmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	altmap->alloc += nr_pfns;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) 	altmap->align += nr_align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 	pfn += nr_align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 	pr_debug("%s: pfn: %#lx alloc: %ld align: %ld nr: %#lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 			__func__, pfn, altmap->alloc, altmap->align, nr_pfns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	return __va(__pfn_to_phys(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) void __meminit vmemmap_verify(pte_t *pte, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 				unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 	unsigned long pfn = pte_pfn(*pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	int actual_node = early_pfn_to_nid(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	if (node_distance(actual_node, node) > LOCAL_DISTANCE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 		pr_warn("[%lx-%lx] potential offnode page_structs\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 			start, end - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) pte_t * __meminit vmemmap_pte_populate(pmd_t *pmd, unsigned long addr, int node,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 				       struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	pte_t *pte = pte_offset_kernel(pmd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	if (pte_none(*pte)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 		pte_t entry;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 		void *p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 		p = vmemmap_alloc_block_buf(PAGE_SIZE, node, altmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 		if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 			return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 		entry = pfn_pte(__pa(p) >> PAGE_SHIFT, PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		set_pte_at(&init_mm, addr, pte, entry);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 	return pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) static void * __meminit vmemmap_alloc_block_zero(unsigned long size, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	void *p = vmemmap_alloc_block(size, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 	if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 	memset(p, 0, size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) 	return p;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) pmd_t * __meminit vmemmap_pmd_populate(pud_t *pud, unsigned long addr, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	pmd_t *pmd = pmd_offset(pud, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	if (pmd_none(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 			return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		pmd_populate_kernel(&init_mm, pmd, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	return pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) pud_t * __meminit vmemmap_pud_populate(p4d_t *p4d, unsigned long addr, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	pud_t *pud = pud_offset(p4d, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	if (pud_none(*pud)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 			return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		pud_populate(&init_mm, pud, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	return pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) p4d_t * __meminit vmemmap_p4d_populate(pgd_t *pgd, unsigned long addr, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 	p4d_t *p4d = p4d_offset(pgd, addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	if (p4d_none(*p4d)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 		if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 			return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		p4d_populate(&init_mm, p4d, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	return p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) pgd_t * __meminit vmemmap_pgd_populate(unsigned long addr, int node)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	pgd_t *pgd = pgd_offset_k(addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 	if (pgd_none(*pgd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 		void *p = vmemmap_alloc_block_zero(PAGE_SIZE, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		if (!p)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 			return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 		pgd_populate(&init_mm, pgd, p);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 	return pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) int __meminit vmemmap_populate_basepages(unsigned long start, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 					 int node, struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	unsigned long addr = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 	pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	for (; addr < end; addr += PAGE_SIZE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		pgd = vmemmap_pgd_populate(addr, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 		if (!pgd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 		p4d = vmemmap_p4d_populate(pgd, addr, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 		if (!p4d)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 		pud = vmemmap_pud_populate(p4d, addr, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		if (!pud)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 		pmd = vmemmap_pmd_populate(pud, addr, node);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		if (!pmd)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 		pte = vmemmap_pte_populate(pmd, addr, node, altmap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 		if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 			return -ENOMEM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 		vmemmap_verify(pte, node, addr, addr + PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) struct page * __meminit __populate_section_memmap(unsigned long pfn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 		unsigned long nr_pages, int nid, struct vmem_altmap *altmap)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 	unsigned long start = (unsigned long) pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 	unsigned long end = start + nr_pages * sizeof(struct page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 	if (WARN_ON_ONCE(!IS_ALIGNED(pfn, PAGES_PER_SUBSECTION) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 		!IS_ALIGNED(nr_pages, PAGES_PER_SUBSECTION)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	if (vmemmap_populate(start, end, nid, altmap))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) 		return NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) 	return pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }