^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) // SPDX-License-Identifier: GPL-2.0-or-later
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) * This file contains the routines for TLB flushing.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4) * On machines where the MMU does not use a hash table to store virtual to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) * physical translations (ie, SW loaded TLBs or Book3E compilant processors,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * this does -not- include 603 however which shares the implementation with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * hash based processors)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * -- BenH
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * Copyright 2008,2009 Ben Herrenschmidt <benh@kernel.crashing.org>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * IBM Corp.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * Derived from arch/ppc/mm/init.c:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * Modifications by Paul Mackerras (PowerMac) (paulus@cs.anu.edu.au)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * and Cort Dougan (PReP) (cort@cs.nmt.edu)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * Copyright (C) 1996 Paul Mackerras
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * Derived from "arch/i386/mm/init.c"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) #include <linux/export.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) #include <linux/init.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) #include <linux/preempt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) #include <linux/spinlock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) #include <linux/memblock.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) #include <linux/of_fdt.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) #include <linux/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) #include <asm/code-patching.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) #include <asm/cputhreads.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) #include <asm/hugetlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) #include <asm/paca.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) #include <mm/mmu_decl.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) * This struct lists the sw-supported page sizes. The hardawre MMU may support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) * other sizes not listed here. The .ind field is only used on MMUs that have
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50) * indirect page table entries.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #if defined(CONFIG_PPC_BOOK3E_MMU) || defined(CONFIG_PPC_8xx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) [MMU_PAGE_4K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) .shift = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) .enc = BOOK3E_PAGESZ_4K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) [MMU_PAGE_2M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) .shift = 21,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) .enc = BOOK3E_PAGESZ_2M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) [MMU_PAGE_4M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) .shift = 22,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) .enc = BOOK3E_PAGESZ_4M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) [MMU_PAGE_16M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) .shift = 24,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) .enc = BOOK3E_PAGESZ_16M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) [MMU_PAGE_64M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) .shift = 26,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) .enc = BOOK3E_PAGESZ_64M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) [MMU_PAGE_256M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) .shift = 28,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) .enc = BOOK3E_PAGESZ_256M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) [MMU_PAGE_1G] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .shift = 30,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) .enc = BOOK3E_PAGESZ_1GB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84) #elif defined(CONFIG_PPC_8xx)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) [MMU_PAGE_4K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) .shift = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) [MMU_PAGE_16K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) .shift = 14,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) [MMU_PAGE_512K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) .shift = 19,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) [MMU_PAGE_8M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) .shift = 23,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) [MMU_PAGE_4K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .shift = 12,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) .ind = 20,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .enc = BOOK3E_PAGESZ_4K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) [MMU_PAGE_16K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) .shift = 14,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) .enc = BOOK3E_PAGESZ_16K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) [MMU_PAGE_64K] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .shift = 16,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) .ind = 28,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) .enc = BOOK3E_PAGESZ_64K,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) [MMU_PAGE_1M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) .shift = 20,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) .enc = BOOK3E_PAGESZ_1M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) [MMU_PAGE_16M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) .shift = 24,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) .ind = 36,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) .enc = BOOK3E_PAGESZ_16M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) [MMU_PAGE_256M] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) .shift = 28,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) .enc = BOOK3E_PAGESZ_256M,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) [MMU_PAGE_1G] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) .shift = 30,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) .enc = BOOK3E_PAGESZ_1GB,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) },
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #endif /* CONFIG_FSL_BOOKE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) static inline int mmu_get_tsize(int psize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) return mmu_psize_defs[psize].enc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) static inline int mmu_get_tsize(int psize)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) /* This isn't used on !Book3E for now */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #endif /* CONFIG_PPC_BOOK3E_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) /* The variables below are currently only used on 64-bit Book3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * though this will probably be made common with other nohash
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * implementations at some point
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) int mmu_linear_psize; /* Page size used for the linear mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) int mmu_pte_psize; /* Page size used for PTE pages */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) int mmu_vmemmap_psize; /* Page size used for the virtual mem map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) int book3e_htw_mode; /* HW tablewalk? Value is PPC_HTW_* */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) unsigned long linear_map_top; /* Top of linear mapping */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) * Number of bytes to add to SPRN_SPRG_TLB_EXFRAME on crit/mcheck/debug
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) * exceptions. This is used for bolted and e6500 TLB miss handlers which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) * do not modify this SPRG in the TLB miss code; for other TLB miss handlers,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) * this is set to zero.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) int extlb_level_exc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #endif /* CONFIG_PPC64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) /* next_tlbcam_idx is used to round-robin tlbcam entry assignment */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) DEFINE_PER_CPU(int, next_tlbcam_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) EXPORT_PER_CPU_SYMBOL(next_tlbcam_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) * Base TLB flushing operations:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) * - flush_tlb_mm(mm) flushes the specified mm context TLB's
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) * - flush_tlb_page(vma, vmaddr) flushes one page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) * - flush_tlb_range(vma, start, end) flushes a range of pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) * - flush_tlb_kernel_range(start, end) flushes kernel pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) * - local_* variants of page and mm only apply to the current
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) * processor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) * These are the base non-SMP variants of page and mm flushing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) void local_flush_tlb_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) unsigned int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) pid = mm->context.id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) if (pid != MMU_NO_CONTEXT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) _tlbil_pid(pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) EXPORT_SYMBOL(local_flush_tlb_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) int tsize, int ind)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) unsigned int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) pid = mm ? mm->context.id : 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) if (pid != MMU_NO_CONTEXT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) _tlbil_va(vmaddr, pid, tsize, ind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) __local_flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) mmu_get_tsize(mmu_virtual_psize), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) EXPORT_SYMBOL(local_flush_tlb_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) * And here are the SMP non-local implementations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) static DEFINE_RAW_SPINLOCK(tlbivax_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) struct tlb_flush_param {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) unsigned long addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) unsigned int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) unsigned int tsize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) unsigned int ind;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) static void do_flush_tlb_mm_ipi(void *param)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) struct tlb_flush_param *p = param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) _tlbil_pid(p ? p->pid : 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) static void do_flush_tlb_page_ipi(void *param)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) struct tlb_flush_param *p = param;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) _tlbil_va(p->addr, p->pid, p->tsize, p->ind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) /* Note on invalidations and PID:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) * We snapshot the PID with preempt disabled. At this point, it can still
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) * change either because:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) * - our context is being stolen (PID -> NO_CONTEXT) on another CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) * - we are invaliating some target that isn't currently running here
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) * and is concurrently acquiring a new PID on another CPU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) * - some other CPU is re-acquiring a lost PID for this mm
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) * etc...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) * However, this shouldn't be a problem as we only guarantee
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) * invalidation of TLB entries present prior to this call, so we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * don't care about the PID changing, and invalidating a stale PID
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * is generally harmless.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) void flush_tlb_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) unsigned int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) pid = mm->context.id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) if (unlikely(pid == MMU_NO_CONTEXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) goto no_context;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) if (!mm_is_core_local(mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) struct tlb_flush_param p = { .pid = pid };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) /* Ignores smp_processor_id() even if set. */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) smp_call_function_many(mm_cpumask(mm),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) do_flush_tlb_mm_ipi, &p, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) _tlbil_pid(pid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) no_context:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) EXPORT_SYMBOL(flush_tlb_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) int tsize, int ind)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) struct cpumask *cpu_mask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) unsigned int pid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) * This function as well as __local_flush_tlb_page() must only be called
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) * for user contexts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) if (WARN_ON(!mm))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) pid = mm->context.id;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) if (unlikely(pid == MMU_NO_CONTEXT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) cpu_mask = mm_cpumask(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) if (!mm_is_core_local(mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) /* If broadcast tlbivax is supported, use it */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) if (mmu_has_feature(MMU_FTR_USE_TLBIVAX_BCAST)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) int lock = mmu_has_feature(MMU_FTR_LOCK_BCAST_INVAL);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) if (lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) raw_spin_lock(&tlbivax_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) _tlbivax_bcast(vmaddr, pid, tsize, ind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) if (lock)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) raw_spin_unlock(&tlbivax_lock);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) goto bail;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) struct tlb_flush_param p = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) .pid = pid,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) .addr = vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) .tsize = tsize,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) .ind = ind,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* Ignores smp_processor_id() even if set in cpu_mask */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) smp_call_function_many(cpu_mask,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) do_flush_tlb_page_ipi, &p, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) _tlbil_va(vmaddr, pid, tsize, ind);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) bail:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) #ifdef CONFIG_HUGETLB_PAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) if (vma && is_vm_hugetlb_page(vma))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) flush_hugetlb_page(vma, vmaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) __flush_tlb_page(vma ? vma->vm_mm : NULL, vmaddr,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) mmu_get_tsize(mmu_virtual_psize), 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) EXPORT_SYMBOL(flush_tlb_page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #ifdef CONFIG_PPC_47x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) void __init early_init_mmu_47x(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) unsigned long root = of_get_flat_dt_root();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) if (of_get_flat_dt_prop(root, "cooperative-partition", NULL))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) mmu_clear_feature(MMU_FTR_USE_TLBIVAX_BCAST);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #endif /* CONFIG_SMP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #endif /* CONFIG_PPC_47x */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) * Flush kernel TLB entries in the given range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) void flush_tlb_kernel_range(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) preempt_disable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) smp_call_function(do_flush_tlb_mm_ipi, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) _tlbil_pid(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) preempt_enable();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) _tlbil_pid(0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) EXPORT_SYMBOL(flush_tlb_kernel_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) * Currently, for range flushing, we just do a full mm flush. This should
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) * be optimized based on a threshold on the size of the range, since
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) * some implementation can stack multiple tlbivax before a tlbsync but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) * for now, we keep it that way
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) void flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) if (end - start == PAGE_SIZE && !(start & ~PAGE_MASK))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) flush_tlb_page(vma, start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) flush_tlb_mm(vma->vm_mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) EXPORT_SYMBOL(flush_tlb_range);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) void tlb_flush(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) flush_tlb_mm(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 395) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 396) * Below are functions specific to the 64-bit variant of Book3E though that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 397) * may change in the future
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 398) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 399)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 400) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 401)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 402) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 403) * Handling of virtual linear page tables or indirect TLB entries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 404) * flushing when PTE pages are freed
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 405) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 406) void tlb_flush_pgtable(struct mmu_gather *tlb, unsigned long address)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 407) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 408) int tsize = mmu_psize_defs[mmu_pte_psize].enc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 409)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 410) if (book3e_htw_mode != PPC_HTW_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 411) unsigned long start = address & PMD_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 412) unsigned long end = address + PMD_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 413) unsigned long size = 1UL << mmu_psize_defs[mmu_pte_psize].shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 414)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 415) /* This isn't the most optimal, ideally we would factor out the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 416) * while preempt & CPU mask mucking around, or even the IPI but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 417) * it will do for now
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 418) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 419) while (start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 420) __flush_tlb_page(tlb->mm, start, tsize, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 421) start += size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 422) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 423) } else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 424) unsigned long rmask = 0xf000000000000000ul;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 425) unsigned long rid = (address & rmask) | 0x1000000000000000ul;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 426) unsigned long vpte = address & ~rmask;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 427)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 428) vpte = (vpte >> (PAGE_SHIFT - 3)) & ~0xffful;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 429) vpte |= rid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 430) __flush_tlb_page(tlb->mm, vpte, tsize, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 431) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 432) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 433)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 434) static void setup_page_sizes(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 435) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 436) unsigned int tlb0cfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 437) unsigned int tlb0ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 438) unsigned int eptcfg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 439) int i, psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 440)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 441) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 442) unsigned int mmucfg = mfspr(SPRN_MMUCFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 443) int fsl_mmu = mmu_has_feature(MMU_FTR_TYPE_FSL_E);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 444)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 445) if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V1) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 446) unsigned int tlb1cfg = mfspr(SPRN_TLB1CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 447) unsigned int min_pg, max_pg;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 448)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 449) min_pg = (tlb1cfg & TLBnCFG_MINSIZE) >> TLBnCFG_MINSIZE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 450) max_pg = (tlb1cfg & TLBnCFG_MAXSIZE) >> TLBnCFG_MAXSIZE_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 451)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 452) for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 453) struct mmu_psize_def *def;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 454) unsigned int shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 455)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 456) def = &mmu_psize_defs[psize];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 457) shift = def->shift;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 458)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 459) if (shift == 0 || shift & 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 460) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 461)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 462) /* adjust to be in terms of 4^shift Kb */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 463) shift = (shift - 10) >> 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 464)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 465) if ((shift >= min_pg) && (shift <= max_pg))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 466) def->flags |= MMU_PAGE_SIZE_DIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 467) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 468)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 469) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 470) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 471)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 472) if (fsl_mmu && (mmucfg & MMUCFG_MAVN) == MMUCFG_MAVN_V2) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 473) u32 tlb1cfg, tlb1ps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 474)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 475) tlb0cfg = mfspr(SPRN_TLB0CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 476) tlb1cfg = mfspr(SPRN_TLB1CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 477) tlb1ps = mfspr(SPRN_TLB1PS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 478) eptcfg = mfspr(SPRN_EPTCFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 479)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 480) if ((tlb1cfg & TLBnCFG_IND) && (tlb0cfg & TLBnCFG_PT))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 481) book3e_htw_mode = PPC_HTW_E6500;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 482)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 483) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 484) * We expect 4K subpage size and unrestricted indirect size.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 485) * The lack of a restriction on indirect size is a Freescale
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 486) * extension, indicated by PSn = 0 but SPSn != 0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 487) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 488) if (eptcfg != 2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 489) book3e_htw_mode = PPC_HTW_NONE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 490)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 491) for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 492) struct mmu_psize_def *def = &mmu_psize_defs[psize];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 493)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 494) if (!def->shift)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 495) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 496)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 497) if (tlb1ps & (1U << (def->shift - 10))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 498) def->flags |= MMU_PAGE_SIZE_DIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 499)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 500) if (book3e_htw_mode && psize == MMU_PAGE_2M)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 501) def->flags |= MMU_PAGE_SIZE_INDIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 502) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 503) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 504)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 505) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 506) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 507) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 508)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 509) tlb0cfg = mfspr(SPRN_TLB0CFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 510) tlb0ps = mfspr(SPRN_TLB0PS);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 511) eptcfg = mfspr(SPRN_EPTCFG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 512)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 513) /* Look for supported direct sizes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 514) for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 515) struct mmu_psize_def *def = &mmu_psize_defs[psize];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 516)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 517) if (tlb0ps & (1U << (def->shift - 10)))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 518) def->flags |= MMU_PAGE_SIZE_DIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 519) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 520)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 521) /* Indirect page sizes supported ? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 522) if ((tlb0cfg & TLBnCFG_IND) == 0 ||
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 523) (tlb0cfg & TLBnCFG_PT) == 0)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 524) goto out;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 525)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 526) book3e_htw_mode = PPC_HTW_IBM;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 527)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 528) /* Now, we only deal with one IND page size for each
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 529) * direct size. Hopefully all implementations today are
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 530) * unambiguous, but we might want to be careful in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 531) * future.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 532) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 533) for (i = 0; i < 3; i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 534) unsigned int ps, sps;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 535)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 536) sps = eptcfg & 0x1f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 537) eptcfg >>= 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 538) ps = eptcfg & 0x1f;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 539) eptcfg >>= 5;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 540) if (!ps || !sps)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 541) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 542) for (psize = 0; psize < MMU_PAGE_COUNT; psize++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 543) struct mmu_psize_def *def = &mmu_psize_defs[psize];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 544)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 545) if (ps == (def->shift - 10))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 546) def->flags |= MMU_PAGE_SIZE_INDIRECT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 547) if (sps == (def->shift - 10))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 548) def->ind = ps + 10;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 549) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 550) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 551)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 552) out:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 553) /* Cleanup array and print summary */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 554) pr_info("MMU: Supported page sizes\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 555) for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 556) struct mmu_psize_def *def = &mmu_psize_defs[psize];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 557) const char *__page_type_names[] = {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 558) "unsupported",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 559) "direct",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 560) "indirect",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 561) "direct & indirect"
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 562) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 563) if (def->flags == 0) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 564) def->shift = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 565) continue;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 566) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 567) pr_info(" %8ld KB as %s\n", 1ul << (def->shift - 10),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 568) __page_type_names[def->flags & 0x3]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 569) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 570) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 571)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 572) static void setup_mmu_htw(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 573) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 574) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 575) * If we want to use HW tablewalk, enable it by patching the TLB miss
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 576) * handlers to branch to the one dedicated to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 577) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 578)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 579) switch (book3e_htw_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 580) case PPC_HTW_IBM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 581) patch_exception(0x1c0, exc_data_tlb_miss_htw_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 582) patch_exception(0x1e0, exc_instruction_tlb_miss_htw_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 583) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 584) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 585) case PPC_HTW_E6500:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 586) extlb_level_exc = EX_TLB_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 587) patch_exception(0x1c0, exc_data_tlb_miss_e6500_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 588) patch_exception(0x1e0, exc_instruction_tlb_miss_e6500_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 589) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 590) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 591) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 592) pr_info("MMU: Book3E HW tablewalk %s\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 593) book3e_htw_mode != PPC_HTW_NONE ? "enabled" : "not supported");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 594) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 595)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 596) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 597) * Early initialization of the MMU TLB code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 598) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 599) static void early_init_this_mmu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 600) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 601) unsigned int mas4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 602)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 603) /* Set MAS4 based on page table setting */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 604)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 605) mas4 = 0x4 << MAS4_WIMGED_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 606) switch (book3e_htw_mode) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 607) case PPC_HTW_E6500:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 608) mas4 |= MAS4_INDD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 609) mas4 |= BOOK3E_PAGESZ_2M << MAS4_TSIZED_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 610) mas4 |= MAS4_TLBSELD(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 611) mmu_pte_psize = MMU_PAGE_2M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 612) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 613)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 614) case PPC_HTW_IBM:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 615) mas4 |= MAS4_INDD;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 616) mas4 |= BOOK3E_PAGESZ_1M << MAS4_TSIZED_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 617) mmu_pte_psize = MMU_PAGE_1M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 618) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 619)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 620) case PPC_HTW_NONE:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 621) mas4 |= BOOK3E_PAGESZ_4K << MAS4_TSIZED_SHIFT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 622) mmu_pte_psize = mmu_virtual_psize;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 623) break;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 624) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 625) mtspr(SPRN_MAS4, mas4);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 626)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 627) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 628) if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 629) unsigned int num_cams;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 630) bool map = true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 631)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 632) /* use a quarter of the TLBCAM for bolted linear map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 633) num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 634)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 635) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 636) * Only do the mapping once per core, or else the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 637) * transient mapping would cause problems.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 638) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 639) #ifdef CONFIG_SMP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 640) if (hweight32(get_tensr()) > 1)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 641) map = false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 642) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 643)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 644) if (map)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 645) linear_map_top = map_mem_in_cams(linear_map_top,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 646) num_cams, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 647) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 648) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 649)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 650) /* A sync won't hurt us after mucking around with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 651) * the MMU configuration
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 652) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 653) mb();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 654) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 655)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 656) static void __init early_init_mmu_global(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 657) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 658) /* XXX This will have to be decided at runtime, but right
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 659) * now our boot and TLB miss code hard wires it. Ideally
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 660) * we should find out a suitable page size and patch the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 661) * TLB miss code (either that or use the PACA to store
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 662) * the value we want)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 663) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 664) mmu_linear_psize = MMU_PAGE_1G;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 665)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 666) /* XXX This should be decided at runtime based on supported
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 667) * page sizes in the TLB, but for now let's assume 16M is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 668) * always there and a good fit (which it probably is)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 669) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 670) * Freescale booke only supports 4K pages in TLB0, so use that.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 671) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 672) if (mmu_has_feature(MMU_FTR_TYPE_FSL_E))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 673) mmu_vmemmap_psize = MMU_PAGE_4K;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 674) else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 675) mmu_vmemmap_psize = MMU_PAGE_16M;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 676)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 677) /* XXX This code only checks for TLB 0 capabilities and doesn't
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 678) * check what page size combos are supported by the HW. It
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 679) * also doesn't handle the case where a separate array holds
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 680) * the IND entries from the array loaded by the PT.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 681) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 682) /* Look for supported page sizes */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 683) setup_page_sizes();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 684)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 685) /* Look for HW tablewalk support */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 686) setup_mmu_htw();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 687)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 688) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 689) if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 690) if (book3e_htw_mode == PPC_HTW_NONE) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 691) extlb_level_exc = EX_TLB_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 692) patch_exception(0x1c0, exc_data_tlb_miss_bolted_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 693) patch_exception(0x1e0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 694) exc_instruction_tlb_miss_bolted_book3e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 695) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 696) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 697) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 698)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 699) /* Set the global containing the top of the linear mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 700) * for use by the TLB miss code
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 701) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 702) linear_map_top = memblock_end_of_DRAM();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 703)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 704) ioremap_bot = IOREMAP_BASE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 705) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 706)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 707) static void __init early_mmu_set_memory_limit(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 708) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 709) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 710) if (mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 711) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 712) * Limit memory so we dont have linear faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 713) * Unlike memblock_set_current_limit, which limits
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 714) * memory available during early boot, this permanently
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 715) * reduces the memory available to Linux. We need to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 716) * do this because highmem is not supported on 64-bit.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 717) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 718) memblock_enforce_memory_limit(linear_map_top);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 719) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 720) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 721)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 722) memblock_set_current_limit(linear_map_top);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 723) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 724)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 725) /* boot cpu only */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 726) void __init early_init_mmu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 727) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 728) early_init_mmu_global();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 729) early_init_this_mmu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 730) early_mmu_set_memory_limit();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 731) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 732)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 733) void early_init_mmu_secondary(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 734) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 735) early_init_this_mmu();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 736) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 737)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 738) void setup_initial_memory_limit(phys_addr_t first_memblock_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 739) phys_addr_t first_memblock_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 740) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 741) /* On non-FSL Embedded 64-bit, we adjust the RMA size to match
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 742) * the bolted TLB entry. We know for now that only 1G
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 743) * entries are supported though that may eventually
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 744) * change.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 745) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 746) * on FSL Embedded 64-bit, usually all RAM is bolted, but with
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 747) * unusual memory sizes it's possible for some RAM to not be mapped
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 748) * (such RAM is not used at all by Linux, since we don't support
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 749) * highmem on 64-bit). We limit ppc64_rma_size to what would be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 750) * mappable if this memblock is the only one. Additional memblocks
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 751) * can only increase, not decrease, the amount that ends up getting
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 752) * mapped. We still limit max to 1G even if we'll eventually map
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 753) * more. This is due to what the early init code is set up to do.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 754) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 755) * We crop it to the size of the first MEMBLOCK to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 756) * avoid going over total available memory just in case...
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 757) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 758) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 759) if (early_mmu_has_feature(MMU_FTR_TYPE_FSL_E)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 760) unsigned long linear_sz;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 761) unsigned int num_cams;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 762)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 763) /* use a quarter of the TLBCAM for bolted linear map */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 764) num_cams = (mfspr(SPRN_TLB1CFG) & TLBnCFG_N_ENTRY) / 4;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 765)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 766) linear_sz = map_mem_in_cams(first_memblock_size, num_cams,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 767) true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 768)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 769) ppc64_rma_size = min_t(u64, linear_sz, 0x40000000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 770) } else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 771) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 772) ppc64_rma_size = min_t(u64, first_memblock_size, 0x40000000);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 773)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 774) /* Finally limit subsequent allocations */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 775) memblock_set_current_limit(first_memblock_base + ppc64_rma_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 776) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 777) #else /* ! CONFIG_PPC64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 778) void __init early_init_mmu(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 779) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 780) #ifdef CONFIG_PPC_47x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 781) early_init_mmu_47x();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 782) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 783)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 784) #ifdef CONFIG_PPC_MM_SLICES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 785) mm_ctx_set_slb_addr_limit(&init_mm.context, SLB_ADDR_LIMIT_DEFAULT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 786) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 787) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 788) #endif /* CONFIG_PPC64 */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags