Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #ifndef _ASM_POWERPC_MMU_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #define _ASM_POWERPC_MMU_H_
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #ifdef __KERNEL__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <asm/asm-const.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  * MMU features bit definitions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  * MMU families
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #define MMU_FTR_HPTE_TABLE		ASM_CONST(0x00000001)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #define MMU_FTR_TYPE_8xx		ASM_CONST(0x00000002)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #define MMU_FTR_TYPE_40x		ASM_CONST(0x00000004)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #define MMU_FTR_TYPE_44x		ASM_CONST(0x00000008)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #define MMU_FTR_TYPE_FSL_E		ASM_CONST(0x00000010)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) #define MMU_FTR_TYPE_47x		ASM_CONST(0x00000020)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) /* Radix page table supported and enabled */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) #define MMU_FTR_TYPE_RADIX		ASM_CONST(0x00000040)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28)  * Individual features below.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32)  * Support for KUEP feature.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) #define MMU_FTR_KUEP			ASM_CONST(0x00000400)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37)  * Support for memory protection keys.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) #define MMU_FTR_PKEY			ASM_CONST(0x00000800)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) /* Guest Translation Shootdown Enable */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) #define MMU_FTR_GTSE			ASM_CONST(0x00001000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45)  * Support for 68 bit VA space. We added that from ISA 2.05
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) #define MMU_FTR_68_BIT_VA		ASM_CONST(0x00002000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49)  * Kernel read only support.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50)  * We added the ppp value 0b110 in ISA 2.04.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) #define MMU_FTR_KERNEL_RO		ASM_CONST(0x00004000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55)  * We need to clear top 16bits of va (from the remaining 64 bits )in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56)  * tlbie* instructions
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) #define MMU_FTR_TLBIE_CROP_VA		ASM_CONST(0x00008000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) /* Enable use of high BAT registers */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) #define MMU_FTR_USE_HIGH_BATS		ASM_CONST(0x00010000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) /* Enable >32-bit physical addresses on 32-bit processor, only used
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64)  * by CONFIG_PPC_BOOK3S_32 currently as BookE supports that from day 1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) #define MMU_FTR_BIG_PHYS		ASM_CONST(0x00020000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) /* Enable use of broadcast TLB invalidations. We don't always set it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69)  * on processors that support it due to other constraints with the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70)  * use of such invalidations
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #define MMU_FTR_USE_TLBIVAX_BCAST	ASM_CONST(0x00040000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) /* Enable use of tlbilx invalidate instructions.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) #define MMU_FTR_USE_TLBILX		ASM_CONST(0x00080000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) /* This indicates that the processor cannot handle multiple outstanding
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79)  * broadcast tlbivax or tlbsync. This makes the code use a spinlock
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80)  * around such invalidate forms.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) #define MMU_FTR_LOCK_BCAST_INVAL	ASM_CONST(0x00100000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) /* This indicates that the processor doesn't handle way selection
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85)  * properly and needs SW to track and update the LRU state.  This
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86)  * is specific to an errata on e300c2/c3/c4 class parts
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) #define MMU_FTR_NEED_DTLB_SW_LRU	ASM_CONST(0x00200000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) /* Enable use of TLB reservation.  Processor should support tlbsrx.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91)  * instruction and MAS0[WQ].
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) #define MMU_FTR_USE_TLBRSRV		ASM_CONST(0x00800000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) /* Use paired MAS registers (MAS7||MAS3, etc.)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) #define MMU_FTR_USE_PAIRED_MAS		ASM_CONST(0x01000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) /* Doesn't support the B bit (1T segment) in SLBIE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) #define MMU_FTR_NO_SLBIE_B		ASM_CONST(0x02000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) /* Support 16M large pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) #define MMU_FTR_16M_PAGE		ASM_CONST(0x04000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) /* Supports TLBIEL variant
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) #define MMU_FTR_TLBIEL			ASM_CONST(0x08000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) /* Supports tlbies w/o locking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define MMU_FTR_LOCKLESS_TLBIE		ASM_CONST(0x10000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) /* Large pages can be marked CI
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) #define MMU_FTR_CI_LARGE_PAGE		ASM_CONST(0x20000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) /* 1T segments available
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) #define MMU_FTR_1T_SEGMENT		ASM_CONST(0x40000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  * Supports KUAP (key 0 controlling userspace addresses) on radix
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) #define MMU_FTR_RADIX_KUAP		ASM_CONST(0x80000000)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) /* MMU feature bit sets for various CPUs */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) #define MMU_FTRS_DEFAULT_HPTE_ARCH_V2	\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	MMU_FTR_HPTE_TABLE | MMU_FTR_PPCAS_ARCH_V2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define MMU_FTRS_POWER		MMU_FTRS_DEFAULT_HPTE_ARCH_V2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) #define MMU_FTRS_PPC970		MMU_FTRS_POWER | MMU_FTR_TLBIE_CROP_VA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) #define MMU_FTRS_POWER5		MMU_FTRS_POWER | MMU_FTR_LOCKLESS_TLBIE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) #define MMU_FTRS_POWER6		MMU_FTRS_POWER5 | MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) #define MMU_FTRS_POWER7		MMU_FTRS_POWER6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) #define MMU_FTRS_POWER8		MMU_FTRS_POWER6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) #define MMU_FTRS_POWER9		MMU_FTRS_POWER6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) #define MMU_FTRS_POWER10	MMU_FTRS_POWER6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) #define MMU_FTRS_CELL		MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 				MMU_FTR_CI_LARGE_PAGE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) #define MMU_FTRS_PA6T		MMU_FTRS_DEFAULT_HPTE_ARCH_V2 | \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 				MMU_FTR_CI_LARGE_PAGE | MMU_FTR_NO_SLBIE_B
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) #include <linux/bug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) #include <asm/cputable.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) #include <asm/page.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) typedef pte_t *pgtable_t;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) #ifdef CONFIG_PPC_FSL_BOOK3E
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) #include <asm/percpu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) DECLARE_PER_CPU(int, next_tlbcam_idx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	MMU_FTRS_POSSIBLE =
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) #ifdef CONFIG_PPC_BOOK3S
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 		MMU_FTR_HPTE_TABLE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) #ifdef CONFIG_PPC_8xx
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 		MMU_FTR_TYPE_8xx |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) #ifdef CONFIG_40x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 		MMU_FTR_TYPE_40x |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) #ifdef CONFIG_44x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 		MMU_FTR_TYPE_44x |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) #if defined(CONFIG_E200) || defined(CONFIG_E500)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 		MMU_FTR_TYPE_FSL_E | MMU_FTR_BIG_PHYS | MMU_FTR_USE_TLBILX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) #ifdef CONFIG_PPC_47x
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 		MMU_FTR_TYPE_47x | MMU_FTR_USE_TLBIVAX_BCAST | MMU_FTR_LOCK_BCAST_INVAL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) #ifdef CONFIG_PPC_BOOK3S_32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 		MMU_FTR_USE_HIGH_BATS | MMU_FTR_NEED_DTLB_SW_LRU |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) #ifdef CONFIG_PPC_BOOK3E_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 		MMU_FTR_USE_TLBRSRV | MMU_FTR_USE_PAIRED_MAS |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 		MMU_FTR_NO_SLBIE_B | MMU_FTR_16M_PAGE | MMU_FTR_TLBIEL |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		MMU_FTR_LOCKLESS_TLBIE | MMU_FTR_CI_LARGE_PAGE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		MMU_FTR_1T_SEGMENT | MMU_FTR_TLBIE_CROP_VA |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		MMU_FTR_KERNEL_RO | MMU_FTR_68_BIT_VA |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) #ifdef CONFIG_PPC_RADIX_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 		MMU_FTR_TYPE_RADIX |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 		MMU_FTR_GTSE |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) #ifdef CONFIG_PPC_KUAP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 		MMU_FTR_RADIX_KUAP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) #endif /* CONFIG_PPC_KUAP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) #endif /* CONFIG_PPC_RADIX_MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) #ifdef CONFIG_PPC_MEM_KEYS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 	MMU_FTR_PKEY |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) #ifdef CONFIG_PPC_KUEP
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	MMU_FTR_KUEP |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) #endif /* CONFIG_PPC_KUAP */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) static inline bool early_mmu_has_feature(unsigned long feature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 	return !!(MMU_FTRS_POSSIBLE & cur_cpu_spec->mmu_features & feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECKS
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) #include <linux/jump_label.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) #define NUM_MMU_FTR_KEYS	32
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) extern struct static_key_true mmu_feature_keys[NUM_MMU_FTR_KEYS];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) extern void mmu_feature_keys_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) static __always_inline bool mmu_has_feature(unsigned long feature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) #ifndef __clang__ /* clang can't cope with this */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 	BUILD_BUG_ON(!__builtin_constant_p(feature));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) #ifdef CONFIG_JUMP_LABEL_FEATURE_CHECK_DEBUG
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	if (!static_key_initialized) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		printk("Warning! mmu_has_feature() used prior to jump label init!\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 		dump_stack();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 		return early_mmu_has_feature(feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	if (!(MMU_FTRS_POSSIBLE & feature))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 	i = __builtin_ctzl(feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 	return static_branch_likely(&mmu_feature_keys[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) static inline void mmu_clear_feature(unsigned long feature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 	i = __builtin_ctzl(feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 	cur_cpu_spec->mmu_features &= ~feature;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 	static_branch_disable(&mmu_feature_keys[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) static inline void mmu_feature_keys_init(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) static inline bool mmu_has_feature(unsigned long feature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 	return early_mmu_has_feature(feature);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) static inline void mmu_clear_feature(unsigned long feature)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 	cur_cpu_spec->mmu_features &= ~feature;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) #endif /* CONFIG_JUMP_LABEL */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) extern unsigned int __start___mmu_ftr_fixup, __stop___mmu_ftr_fixup;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) #ifdef CONFIG_PPC64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) /* This is our real memory area size on ppc64 server, on embedded, we
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271)  * make it match the size our of bolted TLB area
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) extern u64 ppc64_rma_size;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) /* Cleanup function used by kexec */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) extern void mmu_cleanup_all(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) extern void radix__mmu_cleanup_all(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) /* Functions for creating and updating partition table on POWER9 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) extern void mmu_partition_table_init(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) extern void mmu_partition_table_set_entry(unsigned int lpid, unsigned long dw0,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) 					  unsigned long dw1, bool flush);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) #endif /* CONFIG_PPC64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) struct mm_struct;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) #ifdef CONFIG_DEBUG_VM
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) extern void assert_pte_locked(struct mm_struct *mm, unsigned long addr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) #else /* CONFIG_DEBUG_VM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) static inline void assert_pte_locked(struct mm_struct *mm, unsigned long addr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) #endif /* !CONFIG_DEBUG_VM */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) #ifdef CONFIG_PPC_RADIX_MMU
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) static inline bool radix_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) 	return mmu_has_feature(MMU_FTR_TYPE_RADIX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) static inline bool early_radix_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	return early_mmu_has_feature(MMU_FTR_TYPE_RADIX);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) static inline bool radix_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) static inline bool early_radix_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) #ifdef CONFIG_STRICT_KERNEL_RWX
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) static inline bool strict_kernel_rwx_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 	return rodata_enabled;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) static inline bool strict_kernel_rwx_enabled(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #endif /* !__ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) /* The kernel use the constants below to index in the page sizes array.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)  * The use of fixed constants for this purpose is better for performances
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331)  * of the low level hash refill handlers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333)  * A non supported page size has a "shift" field set to 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335)  * Any new page size being implemented can get a new entry in here. Whether
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336)  * the kernel will use it or not is a different matter though. The actual page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337)  * size used by hugetlbfs is not defined here and may be made variable
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339)  * Note: This array ended up being a false good idea as it's growing to the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340)  * point where I wonder if we should replace it with something different,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341)  * to think about, feedback welcome. --BenH.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) /* These are #defines as they have to be used in assembly */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) #define MMU_PAGE_4K	0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) #define MMU_PAGE_16K	1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) #define MMU_PAGE_64K	2
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) #define MMU_PAGE_64K_AP	3	/* "Admixed pages" (hash64 only) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) #define MMU_PAGE_256K	4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) #define MMU_PAGE_512K	5
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) #define MMU_PAGE_1M	6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) #define MMU_PAGE_2M	7
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) #define MMU_PAGE_4M	8
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) #define MMU_PAGE_8M	9
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) #define MMU_PAGE_16M	10
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356) #define MMU_PAGE_64M	11
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) #define MMU_PAGE_256M	12
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) #define MMU_PAGE_1G	13
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) #define MMU_PAGE_16G	14
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) #define MMU_PAGE_64G	15
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363)  * N.B. we need to change the type of hpte_page_sizes if this gets to be > 16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364)  * Also we need to change he type of mm_context.low/high_slices_psize.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) #define MMU_PAGE_COUNT	16
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) #ifdef CONFIG_PPC_BOOK3S_64
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) #include <asm/book3s/64/mmu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) #else /* CONFIG_PPC_BOOK3S_64 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) /* MMU initialization */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) extern void early_init_mmu(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) extern void early_init_mmu_secondary(void);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) extern void setup_initial_memory_limit(phys_addr_t first_memblock_base,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) 				       phys_addr_t first_memblock_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) static inline void mmu_early_init_devtree(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) static inline void pkey_early_init_devtree(void) {}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 381) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 382) extern void *abatron_pteptrs[2];
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 383) #endif /* __ASSEMBLY__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 384) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 385) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 386) #if defined(CONFIG_PPC_BOOK3S_32)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 387) /* 32-bit classic hash table MMU */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 388) #include <asm/book3s/32/mmu-hash.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 389) #elif defined(CONFIG_PPC_MMU_NOHASH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 390) #include <asm/nohash/mmu.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 391) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 392) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 393) #endif /* __KERNEL__ */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 394) #endif /* _ASM_POWERPC_MMU_H_ */