^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * License. See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 1998-2003 Hewlett-Packard Co
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * David Mosberger-Tang <davidm@hpl.hp.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Stephane Eranian <eranian@hpl.hp.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2000, Rohit Seth <rohit.seth@intel.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) * Copyright (C) 1999 VA Linux Systems
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright (C) 1999 Walt Drummond <drummond@valinux.com>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * Copyright (C) 2003 Silicon Graphics, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Routines used by ia64 machines with contiguous (or virtually contiguous)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * memory.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/efi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/nmi.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <asm/meminit.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <asm/mca.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #ifdef CONFIG_VIRTUAL_MEM_MAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) static unsigned long max_gap;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) /* physical address where the bootmem map is located */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) unsigned long bootmap_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) static void *cpu_data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) * per_cpu_init - setup per-cpu variables
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * Allocate and setup per-cpu data areas.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) void *per_cpu_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) static bool first_time = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) void *cpu0_data = __cpu0_per_cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) if (!first_time)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) goto skip;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) first_time = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * get_free_pages() cannot be used before cpu_init() done.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) * BSP allocates PERCPU_PAGE_SIZE bytes for all possible CPUs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) * to avoid that AP calls get_zeroed_page().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) for_each_possible_cpu(cpu) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) void *src = cpu == 0 ? cpu0_data : __phys_per_cpu_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) memcpy(cpu_data, src, __per_cpu_end - __per_cpu_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) __per_cpu_offset[cpu] = (char *)cpu_data - __per_cpu_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) per_cpu(local_per_cpu_offset, cpu) = __per_cpu_offset[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) * percpu area for cpu0 is moved from the __init area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) * which is setup by head.S and used till this point.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) * Update ar.k3. This move is ensures that percpu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) * area for cpu0 is on the correct node and its
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) * virtual address isn't insanely far from other
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) * percpu areas which is important for congruent
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) * percpu allocator.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (cpu == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) ia64_set_kr(IA64_KR_PER_CPU_DATA, __pa(cpu_data) -
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) (unsigned long)__per_cpu_start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) cpu_data += PERCPU_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) skip:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) return __per_cpu_start + __per_cpu_offset[smp_processor_id()];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) static inline void
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) alloc_per_cpu_data(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) size_t size = PERCPU_PAGE_SIZE * num_possible_cpus();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) cpu_data = memblock_alloc_from(size, PERCPU_PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) __pa(MAX_DMA_ADDRESS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) if (!cpu_data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) panic("%s: Failed to allocate %lu bytes align=%lx from=%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) __func__, size, PERCPU_PAGE_SIZE, __pa(MAX_DMA_ADDRESS));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) * setup_per_cpu_areas - setup percpu areas
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) * Arch code has already allocated and initialized percpu areas. All
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) * this function has to do is to teach the determined layout to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) * dynamic percpu allocator, which happens to be more complex than
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) * creating whole new ones using helpers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) setup_per_cpu_areas(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) struct pcpu_alloc_info *ai;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) struct pcpu_group_info *gi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) ssize_t static_size, reserved_size, dyn_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) ai = pcpu_alloc_alloc_info(1, num_possible_cpus());
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) if (!ai)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) panic("failed to allocate pcpu_alloc_info");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) gi = &ai->groups[0];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* units are assigned consecutively to possible cpus */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) gi->cpu_map[gi->nr_units++] = cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /* set parameters */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) static_size = __per_cpu_end - __per_cpu_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) reserved_size = PERCPU_MODULE_RESERVE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) dyn_size = PERCPU_PAGE_SIZE - static_size - reserved_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) if (dyn_size < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) panic("percpu area overflow static=%zd reserved=%zd\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) static_size, reserved_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) ai->static_size = static_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) ai->reserved_size = reserved_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) ai->dyn_size = dyn_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) ai->unit_size = PERCPU_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) ai->atom_size = PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) ai->alloc_size = PERCPU_PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) pcpu_setup_first_chunk(ai, __per_cpu_start + __per_cpu_offset[0]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) pcpu_free_alloc_info(ai);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define alloc_per_cpu_data() do { } while (0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * find_memory - setup memory map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * Walk the EFI memory map and find usable memory for the system, taking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * into account reserved areas.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) find_memory (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) reserve_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) /* first find highest page frame number */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) min_low_pfn = ~0UL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) max_low_pfn = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) efi_memmap_walk(find_max_min_low_pfn, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) max_pfn = max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) #ifdef CONFIG_VIRTUAL_MEM_MAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) efi_memmap_walk(filter_memory, register_active_ranges);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) memblock_add_node(0, PFN_PHYS(max_low_pfn), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) find_initrd();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) alloc_per_cpu_data();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) * Set up the page tables.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) void __init
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) paging_init (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) unsigned long max_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) unsigned long max_zone_pfns[MAX_NR_ZONES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #ifdef CONFIG_ZONE_DMA32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) max_dma = virt_to_phys((void *) MAX_DMA_ADDRESS) >> PAGE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) max_zone_pfns[ZONE_DMA32] = max_dma;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #ifdef CONFIG_VIRTUAL_MEM_MAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) efi_memmap_walk(find_largest_hole, (u64 *)&max_gap);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) if (max_gap < LARGE_GAP) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) vmem_map = (struct page *) 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) unsigned long map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) /* allocate virtual_mem_map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) map_size = PAGE_ALIGN(ALIGN(max_low_pfn, MAX_ORDER_NR_PAGES) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) sizeof(struct page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) VMALLOC_END -= map_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) vmem_map = (struct page *) VMALLOC_END;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) efi_memmap_walk(create_mem_map_page_table, NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * alloc_node_mem_map makes an adjustment for mem_map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) * which isn't compatible with vmem_map.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) NODE_DATA(0)->node_mem_map = vmem_map +
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) find_min_pfn_with_active_regions();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) printk("Virtual mem_map starts at 0x%p\n", mem_map);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #endif /* !CONFIG_VIRTUAL_MEM_MAP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) free_area_init(max_zone_pfns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) zero_page_memmap_ptr = virt_to_page(ia64_imva(empty_zero_page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }