^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * License. See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * Copyright (C) 1994 - 2000 Ralf Baechle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * Kevin D. Kissell, kevink@mips.com and Carsten Langgaard, carstenl@mips.com
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * Copyright (C) 2000 MIPS Technologies, Inc. All rights reserved.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) #include <linux/signal.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) #include <linux/sched.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) #include <linux/errno.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) #include <linux/string.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) #include <linux/ptrace.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) #include <linux/mman.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/proc_fs.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/pfn.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/hardirq.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/kcore.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/initrd.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <asm/bootinfo.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) #include <asm/cachectl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/cpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/dma.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/kmap_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/maar.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/sections.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <asm/fixmap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * We have up to 8 empty zeroed pages so we can map one of the right colour
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * when needed. This is necessary only on R4000 / R4400 SC and MC versions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * where we have to avoid VCED / VECI exceptions for good performance at
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) * any price. Since page is never written to after the initialization we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) * don't have to care about aliases on other CPUs.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) unsigned long empty_zero_page, zero_page_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) EXPORT_SYMBOL_GPL(empty_zero_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) EXPORT_SYMBOL(zero_page_mask);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * Not static inline because used by IP27 special magic initialization code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) void setup_zero_pages(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) unsigned int order, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) struct page *page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) if (cpu_has_vce)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) order = 3;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) order = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) empty_zero_page = __get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) if (!empty_zero_page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) panic("Oh boy, that early out of memory?");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) page = virt_to_page((void *)empty_zero_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) split_page(page, order);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) for (i = 0; i < (1 << order); i++, page++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) mark_page_reserved(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) zero_page_mask = ((PAGE_SIZE << order) - 1) & PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) static void *__kmap_pgprot(struct page *page, unsigned long addr, pgprot_t prot)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) enum fixed_addresses idx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) unsigned int old_mmid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) unsigned long vaddr, flags, entrylo;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) unsigned long old_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) pte_t pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) int tlbidx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) BUG_ON(Page_dcache_dirty(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) pagefault_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) idx = (addr >> PAGE_SHIFT) & (FIX_N_COLOURS - 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) idx += in_interrupt() ? FIX_N_COLOURS : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) vaddr = __fix_to_virt(FIX_CMAP_END - idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) pte = mk_pte(page, prot);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) #if defined(CONFIG_XPA)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) entrylo = pte_to_entrylo(pte.pte_high);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) #elif defined(CONFIG_PHYS_ADDR_T_64BIT) && defined(CONFIG_CPU_MIPS32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) entrylo = pte.pte_high;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) entrylo = pte_to_entrylo(pte_val(pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) old_ctx = read_c0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) write_c0_entryhi(vaddr & (PAGE_MASK << 1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) write_c0_entrylo0(entrylo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) write_c0_entrylo1(entrylo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) if (cpu_has_mmid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) old_mmid = read_c0_memorymapid();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) write_c0_memorymapid(MMID_KERNEL_WIRED);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #ifdef CONFIG_XPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) if (cpu_has_xpa) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) entrylo = (pte.pte_low & _PFNX_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) writex_c0_entrylo0(entrylo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) writex_c0_entrylo1(entrylo);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) tlbidx = num_wired_entries();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) write_c0_wired(tlbidx + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) write_c0_index(tlbidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) write_c0_entryhi(old_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) if (cpu_has_mmid)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) write_c0_memorymapid(old_mmid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) return (void*) vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) void *kmap_coherent(struct page *page, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) return __kmap_pgprot(page, addr, PAGE_KERNEL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) void *kmap_noncoherent(struct page *page, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) return __kmap_pgprot(page, addr, PAGE_KERNEL_NC);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) void kunmap_coherent(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) unsigned int wired;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) unsigned long flags, old_ctx;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) old_ctx = read_c0_entryhi();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) wired = num_wired_entries() - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) write_c0_wired(wired);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) write_c0_index(wired);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) write_c0_entryhi(UNIQUE_ENTRYHI(wired));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) write_c0_entrylo0(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) write_c0_entrylo1(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) mtc0_tlbw_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) tlb_write_indexed();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) tlbw_use_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) write_c0_entryhi(old_ctx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) pagefault_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) void copy_user_highpage(struct page *to, struct page *from,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) unsigned long vaddr, struct vm_area_struct *vma)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) void *vfrom, *vto;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) vto = kmap_atomic(to);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) if (cpu_has_dc_aliases &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) page_mapcount(from) && !Page_dcache_dirty(from)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) vfrom = kmap_coherent(from, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) copy_page(vto, vfrom);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) kunmap_coherent();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) vfrom = kmap_atomic(from);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) copy_page(vto, vfrom);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) kunmap_atomic(vfrom);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) if ((!cpu_has_ic_fills_f_dc) ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) pages_do_alias((unsigned long)vto, vaddr & PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) flush_data_cache_page((unsigned long)vto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) kunmap_atomic(vto);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) /* Make sure this page is cleared on other CPU's too before using it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) smp_wmb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) void copy_to_user_page(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) struct page *page, unsigned long vaddr, void *dst, const void *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) if (cpu_has_dc_aliases &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) page_mapcount(page) && !Page_dcache_dirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) void *vto = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) memcpy(vto, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) kunmap_coherent();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) memcpy(dst, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) if (cpu_has_dc_aliases)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) SetPageDcacheDirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) if (vma->vm_flags & VM_EXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) flush_cache_page(vma, vaddr, page_to_pfn(page));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) void copy_from_user_page(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) struct page *page, unsigned long vaddr, void *dst, const void *src,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) unsigned long len)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) if (cpu_has_dc_aliases &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) page_mapcount(page) && !Page_dcache_dirty(page)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) void *vfrom = kmap_coherent(page, vaddr) + (vaddr & ~PAGE_MASK);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) memcpy(dst, vfrom, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) kunmap_coherent();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) memcpy(dst, src, len);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) if (cpu_has_dc_aliases)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) SetPageDcacheDirty(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) EXPORT_SYMBOL_GPL(copy_from_user_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) void __init fixrange_init(unsigned long start, unsigned long end,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) pgd_t *pgd_base)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) int i, j, k;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) unsigned long vaddr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) vaddr = start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) i = pgd_index(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) j = pud_index(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) k = pmd_index(vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) pgd = pgd_base + i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) for ( ; (i < PTRS_PER_PGD) && (vaddr < end); pgd++, i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) pud = (pud_t *)pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) for ( ; (j < PTRS_PER_PUD) && (vaddr < end); pud++, j++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) pmd = (pmd_t *)pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) for (; (k < PTRS_PER_PMD) && (vaddr < end); pmd++, k++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) if (pmd_none(*pmd)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) pte = (pte_t *) memblock_alloc_low(PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) panic("%s: Failed to allocate %lu bytes align=%lx\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) __func__, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) set_pmd(pmd, __pmd((unsigned long)pte));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) BUG_ON(pte != pte_offset_kernel(pmd, 0));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) vaddr += PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) k = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) j = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) struct maar_walk_info {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) struct maar_config cfg[16];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) unsigned int num_cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) static int maar_res_walk(unsigned long start_pfn, unsigned long nr_pages,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) void *data)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) struct maar_walk_info *wi = data;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) struct maar_config *cfg = &wi->cfg[wi->num_cfg];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) unsigned int maar_align;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) /* MAAR registers hold physical addresses right shifted by 4 bits */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) maar_align = BIT(MIPS_MAAR_ADDR_SHIFT + 4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) /* Fill in the MAAR config entry */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) cfg->lower = ALIGN(PFN_PHYS(start_pfn), maar_align);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) cfg->upper = ALIGN_DOWN(PFN_PHYS(start_pfn + nr_pages), maar_align) - 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) cfg->attrs = MIPS_MAAR_S;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) /* Ensure we don't overflow the cfg array */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) if (!WARN_ON(wi->num_cfg >= ARRAY_SIZE(wi->cfg)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) wi->num_cfg++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) unsigned __weak platform_maar_init(unsigned num_pairs)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) unsigned int num_configured;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) struct maar_walk_info wi;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) wi.num_cfg = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) walk_system_ram_range(0, max_pfn, &wi, maar_res_walk);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) num_configured = maar_config(wi.cfg, wi.num_cfg, num_pairs);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) if (num_configured < wi.num_cfg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) pr_warn("Not enough MAAR pairs (%u) for all memory regions (%u)\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) num_pairs, wi.num_cfg);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) return num_configured;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) void maar_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) unsigned num_maars, used, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) phys_addr_t lower, upper, attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) static struct {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) struct maar_config cfgs[3];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) unsigned used;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) } recorded = { { { 0 } }, 0 };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) if (!cpu_has_maar)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) /* Detect the number of MAARs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) write_c0_maari(~0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) num_maars = read_c0_maari() + 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) /* MAARs should be in pairs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) WARN_ON(num_maars % 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) /* Set MAARs using values we recorded already */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) if (recorded.used) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) used = maar_config(recorded.cfgs, recorded.used, num_maars / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) BUG_ON(used != recorded.used);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) /* Configure the required MAARs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) used = platform_maar_init(num_maars / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /* Disable any further MAARs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) for (i = (used * 2); i < num_maars; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) write_c0_maari(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) write_c0_maar(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) if (recorded.used)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) pr_info("MAAR configuration:\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) for (i = 0; i < num_maars; i += 2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) write_c0_maari(i);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) upper = read_c0_maar();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #ifdef CONFIG_XPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) upper |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) write_c0_maari(i + 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) back_to_back_c0_hazard();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) lower = read_c0_maar();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #ifdef CONFIG_XPA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) lower |= (phys_addr_t)readx_c0_maar() << MIPS_MAARX_ADDR_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) attr = lower & upper;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) lower = (lower & MIPS_MAAR_ADDR) << 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) upper = ((upper & MIPS_MAAR_ADDR) << 4) | 0xffff;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) pr_info(" [%d]: ", i / 2);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) if ((attr & MIPS_MAAR_V) != MIPS_MAAR_V) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) pr_cont("disabled\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) pr_cont("%pa-%pa", &lower, &upper);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (attr & MIPS_MAAR_S)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) pr_cont(" speculate");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) pr_cont("\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) /* Record the setup for use on secondary CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) if (used <= ARRAY_SIZE(recorded.cfgs)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) recorded.cfgs[recorded.used].lower = lower;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) recorded.cfgs[recorded.used].upper = upper;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) recorded.cfgs[recorded.used].attrs = attr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) recorded.used++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) #ifndef CONFIG_NEED_MULTIPLE_NODES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399) void __init paging_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401) unsigned long max_zone_pfns[MAX_NR_ZONES];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) pagetable_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) kmap_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) #ifdef CONFIG_ZONE_DMA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409) max_zone_pfns[ZONE_DMA] = MAX_DMA_PFN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) #ifdef CONFIG_ZONE_DMA32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) max_zone_pfns[ZONE_DMA32] = MAX_DMA32_PFN;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414) max_zone_pfns[ZONE_NORMAL] = max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) max_zone_pfns[ZONE_HIGHMEM] = highend_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) if (cpu_has_dc_aliases && max_low_pfn != highend_pfn) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) printk(KERN_WARNING "This processor doesn't support highmem."
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) " %ldk highmem ignored\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) (highend_pfn - max_low_pfn) << (PAGE_SHIFT - 10));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) max_zone_pfns[ZONE_HIGHMEM] = max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) free_area_init(max_zone_pfns);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) static struct kcore_list kcore_kseg0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433) static inline void __init mem_init_free_highmem(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) unsigned long tmp;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) if (cpu_has_dc_aliases)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) for (tmp = highstart_pfn; tmp < highend_pfn; tmp++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) struct page *page = pfn_to_page(tmp);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444) if (!memblock_is_memory(PFN_PHYS(tmp)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) SetPageReserved(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) free_highmem_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) void __init mem_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455) * When _PFN_SHIFT is greater than PAGE_SHIFT we won't have enough PTE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) * bits to hold a full 32b physical address on MIPS32 systems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458) BUILD_BUG_ON(IS_ENABLED(CONFIG_32BIT) && (_PFN_SHIFT > PAGE_SHIFT));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) #ifdef CONFIG_HIGHMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461) #ifdef CONFIG_DISCONTIGMEM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) #error "CONFIG_HIGHMEM and CONFIG_DISCONTIGMEM dont work together yet"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464) max_mapnr = highend_pfn ? highend_pfn : max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) max_mapnr = max_low_pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468) high_memory = (void *) __va(max_low_pfn << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) maar_init();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471) memblock_free_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) setup_zero_pages(); /* Setup zeroed pages. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) mem_init_free_highmem();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474) mem_init_print_info(NULL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) #ifdef CONFIG_64BIT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) if ((unsigned long) &_text > (unsigned long) CKSEG0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) /* The -4 is a hack so that user tools don't have to handle
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479) the overflow. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) kclist_add(&kcore_kseg0, (void *) CKSEG0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) 0x80000000 - 4, KCORE_TEXT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) #endif /* !CONFIG_NEED_MULTIPLE_NODES */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) void free_init_pages(const char *what, unsigned long begin, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) unsigned long pfn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490) for (pfn = PFN_UP(begin); pfn < PFN_DOWN(end); pfn++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) struct page *page = pfn_to_page(pfn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) void *addr = phys_to_virt(PFN_PHYS(pfn));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) memset(addr, POISON_FREE_INITMEM, PAGE_SIZE);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) free_reserved_page(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) printk(KERN_INFO "Freeing %s: %ldk freed\n", what, (end - begin) >> 10);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) void (*free_init_pages_eva)(void *begin, void *end) = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) void __ref free_initmem(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504) prom_free_prom_memory();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) * Let the platform define a specific function to free the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) * init section since EVA may have used any possible mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508) * between virtual and physical addresses.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) if (free_init_pages_eva)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) free_init_pages_eva((void *)&__init_begin, (void *)&__init_end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) free_initmem_default(POISON_FREE_INITMEM);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516) #ifdef CONFIG_HAVE_SETUP_PER_CPU_AREA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) unsigned long __per_cpu_offset[NR_CPUS] __read_mostly;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) EXPORT_SYMBOL(__per_cpu_offset);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520) static int __init pcpu_cpu_distance(unsigned int from, unsigned int to)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) return node_distance(cpu_to_node(from), cpu_to_node(to));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525) static void * __init pcpu_fc_alloc(unsigned int cpu, size_t size,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) size_t align)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) return memblock_alloc_try_nid(size, align, __pa(MAX_DMA_ADDRESS),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) MEMBLOCK_ALLOC_ACCESSIBLE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) cpu_to_node(cpu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) static void __init pcpu_fc_free(void *ptr, size_t size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535) memblock_free_early(__pa(ptr), size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) void __init setup_per_cpu_areas(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) unsigned long delta;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) unsigned int cpu;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) int rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) * Always reserve area for module percpu variables. That's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) * what the legacy allocator did.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) rc = pcpu_embed_first_chunk(PERCPU_MODULE_RESERVE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) PERCPU_DYNAMIC_RESERVE, PAGE_SIZE,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) pcpu_cpu_distance,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551) pcpu_fc_alloc, pcpu_fc_free);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) if (rc < 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) panic("Failed to initialize percpu areas.");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) delta = (unsigned long)pcpu_base_addr - (unsigned long)__per_cpu_start;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) for_each_possible_cpu(cpu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) __per_cpu_offset[cpu] = delta + pcpu_unit_offsets[cpu];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) #ifndef CONFIG_MIPS_PGD_C0_CONTEXT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) unsigned long pgd_current[NR_CPUS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) * Align swapper_pg_dir in to 64K, allows its address to be loaded
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) * with a single LUI instruction in the TLB handlers. If we used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) * __aligned(64K), its size would get rounded up to the alignment
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) * size, and waste space. So we place it in its own section and align
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) * it in the linker script.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) pgd_t swapper_pg_dir[PTRS_PER_PGD] __section(".bss..swapper_pg_dir");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) #ifndef __PAGETABLE_PUD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) pud_t invalid_pud_table[PTRS_PER_PUD] __page_aligned_bss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) #ifndef __PAGETABLE_PMD_FOLDED
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) pmd_t invalid_pmd_table[PTRS_PER_PMD] __page_aligned_bss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578) EXPORT_SYMBOL_GPL(invalid_pmd_table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) pte_t invalid_pte_table[PTRS_PER_PTE] __page_aligned_bss;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) EXPORT_SYMBOL(invalid_pte_table);
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags