^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 1) /* SPDX-License-Identifier: GPL-2.0 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 2) #ifndef _SPARC64_TSB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 3) #define _SPARC64_TSB_H
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 4)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 5) /* The sparc64 TSB is similar to the powerpc hashtables. It's a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 6) * power-of-2 sized table of TAG/PTE pairs. The cpu precomputes
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 7) * pointers into this table for 8K and 64K page sizes, and also a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 8) * comparison TAG based upon the virtual address and context which
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 9) * faults.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 10) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 11) * TLB miss trap handler software does the actual lookup via something
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 12) * of the form:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 13) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 14) * ldxa [%g0] ASI_{D,I}MMU_TSB_8KB_PTR, %g1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 15) * ldxa [%g0] ASI_{D,I}MMU, %g6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 16) * sllx %g6, 22, %g6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 17) * srlx %g6, 22, %g6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 18) * ldda [%g1] ASI_NUCLEUS_QUAD_LDD, %g4
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 19) * cmp %g4, %g6
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 20) * bne,pn %xcc, tsb_miss_{d,i}tlb
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 21) * mov FAULT_CODE_{D,I}TLB, %g3
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 22) * stxa %g5, [%g0] ASI_{D,I}TLB_DATA_IN
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 23) * retry
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 24) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 25) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 26) * Each 16-byte slot of the TSB is the 8-byte tag and then the 8-byte
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 27) * PTE. The TAG is of the same layout as the TLB TAG TARGET mmu
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 28) * register which is:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 29) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 30) * -------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 31) * | - | CONTEXT | - | VADDR bits 63:22 |
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 32) * -------------------------------------------------
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 33) * 63 61 60 48 47 42 41 0
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 34) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 35) * But actually, since we use per-mm TSB's, we zero out the CONTEXT
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 36) * field.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 37) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 38) * Like the powerpc hashtables we need to use locking in order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 39) * synchronize while we update the entries. PTE updates need locking
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 40) * as well.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 41) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 42) * We need to carefully choose a lock bits for the TSB entry. We
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 43) * choose to use bit 47 in the tag. Also, since we never map anything
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 44) * at page zero in context zero, we use zero as an invalid tag entry.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 45) * When the lock bit is set, this forces a tag comparison failure.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 46) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 47)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 48) #define TSB_TAG_LOCK_BIT 47
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 49) #define TSB_TAG_LOCK_HIGH (1 << (TSB_TAG_LOCK_BIT - 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 50)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 51) #define TSB_TAG_INVALID_BIT 46
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 52) #define TSB_TAG_INVALID_HIGH (1 << (TSB_TAG_INVALID_BIT - 32))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 53)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 54) /* Some cpus support physical address quad loads. We want to use
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 55) * those if possible so we don't need to hard-lock the TSB mapping
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 56) * into the TLB. We encode some instruction patching in order to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 57) * support this.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 58) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 59) * The kernel TSB is locked into the TLB by virtue of being in the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 60) * kernel image, so we don't play these games for swapper_tsb access.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 61) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 62) #ifndef __ASSEMBLY__
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 63) struct tsb_ldquad_phys_patch_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 64) unsigned int addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 65) unsigned int sun4u_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 66) unsigned int sun4v_insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 67) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 68) extern struct tsb_ldquad_phys_patch_entry __tsb_ldquad_phys_patch,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 69) __tsb_ldquad_phys_patch_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 70)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 71) struct tsb_phys_patch_entry {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 72) unsigned int addr;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 73) unsigned int insn;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 74) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 75) extern struct tsb_phys_patch_entry __tsb_phys_patch, __tsb_phys_patch_end;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 76) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 77) #define TSB_LOAD_QUAD(TSB, REG) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 78) 661: ldda [TSB] ASI_NUCLEUS_QUAD_LDD, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 79) .section .tsb_ldquad_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 80) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 81) ldda [TSB] ASI_QUAD_LDD_PHYS, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 82) ldda [TSB] ASI_QUAD_LDD_PHYS_4V, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 83) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 84)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 85) #define TSB_LOAD_TAG_HIGH(TSB, REG) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 86) 661: lduwa [TSB] ASI_N, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 87) .section .tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 88) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 89) lduwa [TSB] ASI_PHYS_USE_EC, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 90) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 91)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 92) #define TSB_LOAD_TAG(TSB, REG) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 93) 661: ldxa [TSB] ASI_N, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 94) .section .tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 95) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 96) ldxa [TSB] ASI_PHYS_USE_EC, REG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 97) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 98)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 99) #define TSB_CAS_TAG_HIGH(TSB, REG1, REG2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 661: casa [TSB] ASI_N, REG1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) .section .tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) casa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #define TSB_CAS_TAG(TSB, REG1, REG2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 661: casxa [TSB] ASI_N, REG1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) .section .tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) casxa [TSB] ASI_PHYS_USE_EC, REG1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) #define TSB_STORE(ADDR, VAL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 661: stxa VAL, [ADDR] ASI_N; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) .section .tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) stxa VAL, [ADDR] ASI_PHYS_USE_EC; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) .previous
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) #define TSB_LOCK_TAG(TSB, REG1, REG2) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 99: TSB_LOAD_TAG_HIGH(TSB, REG1); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) sethi %hi(TSB_TAG_LOCK_HIGH), REG2;\
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) andcc REG1, REG2, %g0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) bne,pn %icc, 99b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) TSB_CAS_TAG_HIGH(TSB, REG1, REG2); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) cmp REG1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) bne,pn %icc, 99b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) #define TSB_WRITE(TSB, TTE, TAG) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) add TSB, 0x8, TSB; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) TSB_STORE(TSB, TTE); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) sub TSB, 0x8, TSB; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) TSB_STORE(TSB, TAG);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) /* Do a kernel page table walk. Leaves valid PTE value in
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) * REG1. Jumps to FAIL_LABEL on early page table walk
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) * termination. VADDR will not be clobbered, but REG2 will.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) * There are two masks we must apply to propagate bits from
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) * the virtual address into the PTE physical address field
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) * when dealing with huge pages. This is because the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) * table boundaries do not match the huge page size(s) the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) * hardware supports.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) * In these cases we propagate the bits that are below the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) * page table level where we saw the huge page mapping, but
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) * are still within the relevant physical bits for the huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) * page size in question. So for PMD mappings (which fall on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) * bit 23, for 8MB per PMD) we must propagate bit 22 for a
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) * 4MB huge page. For huge PUDs (which fall on bit 33, for
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) * 8GB per PUD), we have to accommodate 256MB and 2GB huge
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) * pages. So for those we propagate bits 32 to 28.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) #define KERN_PGTABLE_WALK(VADDR, REG1, REG2, FAIL_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) sethi %hi(swapper_pg_dir), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) or REG1, %lo(swapper_pg_dir), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) ldx [REG1 + REG2], REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) sethi %uhi(_PAGE_PUD_HUGE), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) sllx REG2, 32, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) andcc REG1, REG2, %g0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) sethi %hi(0xf8000000), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) bne,pt %xcc, 697f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) sllx REG2, 1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) sethi %uhi(_PAGE_PMD_HUGE), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) sllx REG2, 32, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) andcc REG1, REG2, %g0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) be,pn %xcc, 698f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) sethi %hi(0x400000), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 697: brgez,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) andn REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) and VADDR, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) ba,pt %xcc, 699f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) or REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 698: sllx VADDR, 64 - PMD_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) brgez,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) 699:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) /* PUD has been loaded into REG1, interpret the value, seeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) * if it is a HUGE PUD or a normal one. If it is not valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) * then jump to FAIL_LABEL. If it is a HUGE PUD, and it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) * translates to a valid PTE, branch to PTE_LABEL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) * We have to propagate bits [32:22] from the virtual address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) * to resolve at 4M granularity.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) #define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 700: ba 700f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) .section .pud_huge_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) .word 700b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) .previous; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) sethi %uhi(_PAGE_PUD_HUGE), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) sllx REG2, 32, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) andcc REG1, REG2, %g0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) be,pt %xcc, 700f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) sethi %hi(0xffe00000), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) sllx REG2, 1, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) brgez,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) andn REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) and VADDR, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) brlz,pt REG1, PTE_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) or REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 700:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) #define USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) nop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) /* PMD has been loaded into REG1, interpret the value, seeing
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) * if it is a HUGE PMD or a normal one. If it is not valid
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) * then jump to FAIL_LABEL. If it is a HUGE PMD, and it
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) * translates to a valid PTE, branch to PTE_LABEL.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) * We have to propagate the 4MB bit of the virtual address
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) * because we are fabricating 8MB pages using 4MB hw pages.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) #if defined(CONFIG_HUGETLB_PAGE) || defined(CONFIG_TRANSPARENT_HUGEPAGE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) sethi %uhi(_PAGE_PMD_HUGE), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) sllx REG2, 32, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) andcc REG1, REG2, %g0; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) be,pt %xcc, 700f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) sethi %hi(4 * 1024 * 1024), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) brgez,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) andn REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) and VADDR, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) brlz,pt REG1, PTE_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) or REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 700:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) #define USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, PTE_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) nop;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) /* Do a user page table walk in MMU globals. Leaves final,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) * valid, PTE value in REG1. Jumps to FAIL_LABEL on early
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) * page table walk termination or if the PTE is not valid.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) * Physical base of page tables is in PHYS_PGD which will not
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) * be modified.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) * VADDR will not be clobbered, but REG1 and REG2 will.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) #define USER_PGTABLE_WALK_TL1(VADDR, PHYS_PGD, REG1, REG2, FAIL_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) sllx VADDR, 64 - (PGDIR_SHIFT + PGDIR_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) ldxa [PHYS_PGD + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) sllx VADDR, 64 - (PUD_SHIFT + PUD_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) USER_PGTABLE_CHECK_PUD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) brz,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) sllx VADDR, 64 - (PMD_SHIFT + PMD_BITS), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) ldxa [REG1 + REG2] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) USER_PGTABLE_CHECK_PMD_HUGE(VADDR, REG1, REG2, FAIL_LABEL, 800f) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) sllx VADDR, 64 - PMD_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) srlx REG2, 64 - PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) andn REG2, 0x7, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291) add REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292) ldxa [REG1] ASI_PHYS_USE_EC, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293) brgez,pn REG1, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295) 800:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297) /* Lookup a OBP mapping on VADDR in the prom_trans[] table at TL>0.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298) * If no entry is found, FAIL_LABEL will be branched to. On success
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) * the resulting PTE value will be left in REG1. VADDR is preserved
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) * by this routine.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) #define OBP_TRANS_LOOKUP(VADDR, REG1, REG2, REG3, FAIL_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) sethi %hi(prom_trans), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) or REG1, %lo(prom_trans), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 97: ldx [REG1 + 0x00], REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) brz,pn REG2, FAIL_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) nop; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) ldx [REG1 + 0x08], REG3; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) add REG2, REG3, REG3; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) cmp REG2, VADDR; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) bgu,pt %xcc, 98f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) cmp VADDR, REG3; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) bgeu,pt %xcc, 98f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) ldx [REG1 + 0x10], REG3; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) sub VADDR, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) ba,pt %xcc, 99f; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) add REG3, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 98: ba,pt %xcc, 97b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) add REG1, (3 * 8), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 99:
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) /* We use a 32K TSB for the whole kernel, this allows to
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) * handle about 16MB of modules and vmalloc mappings without
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) * incurring many hash conflicts.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) #define KERNEL_TSB_SIZE_BYTES (32 * 1024)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) #define KERNEL_TSB_NENTRIES \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) (KERNEL_TSB_SIZE_BYTES / 16)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) #define KERNEL_TSB4M_NENTRIES 4096
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) /* Do a kernel TSB lookup at tl>0 on VADDR+TAG, branch to OK_LABEL
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) * on TSB hit. REG1, REG2, REG3, and REG4 are used as temporaries
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) * and the found TTE will be left in REG1. REG3 and REG4 must
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) * be an even/odd pair of registers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 335) *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 336) * VADDR and TAG will be preserved and not clobbered by this macro.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 337) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 338) #define KERN_TSB_LOOKUP_TL1(VADDR, TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 339) 661: sethi %uhi(swapper_tsb), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 340) sethi %hi(swapper_tsb), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 341) or REG1, %ulo(swapper_tsb), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 342) or REG2, %lo(swapper_tsb), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 343) .section .swapper_tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 344) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 345) .previous; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 346) sllx REG1, 32, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 347) or REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 348) srlx VADDR, PAGE_SHIFT, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 349) and REG2, (KERNEL_TSB_NENTRIES - 1), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 350) sllx REG2, 4, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 351) add REG1, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 352) TSB_LOAD_QUAD(REG2, REG3); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 353) cmp REG3, TAG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 354) be,a,pt %xcc, OK_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 355) mov REG4, REG1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 356)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 357) #ifndef CONFIG_DEBUG_PAGEALLOC
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 358) /* This version uses a trick, the TAG is already (VADDR >> 22) so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 359) * we can make use of that for the index computation.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 360) */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 361) #define KERN_TSB4M_LOOKUP_TL1(TAG, REG1, REG2, REG3, REG4, OK_LABEL) \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 362) 661: sethi %uhi(swapper_4m_tsb), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 363) sethi %hi(swapper_4m_tsb), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 364) or REG1, %ulo(swapper_4m_tsb), REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 365) or REG2, %lo(swapper_4m_tsb), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 366) .section .swapper_4m_tsb_phys_patch, "ax"; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 367) .word 661b; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 368) .previous; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 369) sllx REG1, 32, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 370) or REG1, REG2, REG1; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 371) and TAG, (KERNEL_TSB4M_NENTRIES - 1), REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 372) sllx REG2, 4, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 373) add REG1, REG2, REG2; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 374) TSB_LOAD_QUAD(REG2, REG3); \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 375) cmp REG3, TAG; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 376) be,a,pt %xcc, OK_LABEL; \
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 377) mov REG4, REG1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 378) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 379)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 380) #endif /* !(_SPARC64_TSB_H) */
Orange Pi5 kernel
Deprecated Linux kernel 5.10.110 for OrangePi 5/5B/5+ boards
3 Commits
0 Branches
0 Tags
| Donate