Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2)  * arch/xtensa/mm/tlb.c
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4)  * Logic that manipulates the Xtensa MMU.  Derived from MIPS.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6)  * This file is subject to the terms and conditions of the GNU General Public
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7)  * License.  See the file "COPYING" in the main directory of this archive
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8)  * for more details.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10)  * Copyright (C) 2001 - 2003 Tensilica Inc.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12)  * Joe Taylor
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13)  * Chris Zankel	<chris@zankel.net>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14)  * Marc Gauthier
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) #include <linux/mm.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) #include <asm/processor.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) #include <asm/mmu_context.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) #include <asm/tlbflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) #include <asm/cacheflush.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) static inline void __flush_itlb_all (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	int w, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 	for (w = 0; w < ITLB_ARF_WAYS; w++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 		for (i = 0; i < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 			int e = w + (i << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 			invalidate_itlb_entry_no_isync(e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	asm volatile ("isync\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) static inline void __flush_dtlb_all (void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	int w, i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	for (w = 0; w < DTLB_ARF_WAYS; w++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) 		for (i = 0; i < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); i++) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 			int e = w + (i << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) 			invalidate_dtlb_entry_no_isync(e);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 	asm volatile ("isync\n");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) void local_flush_tlb_all(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) 	__flush_itlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 	__flush_dtlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) /* If mm is current, we simply assign the current task a new ASID, thus,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58)  * invalidating all previous tlb entries. If mm is someone else's user mapping,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59)  * wie invalidate the context, thus, when that user mapping is swapped in,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60)  * a new context will be assigned to it.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) void local_flush_tlb_mm(struct mm_struct *mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) 	if (mm == current->active_mm) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 		unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 		local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 		mm->context.asid[cpu] = NO_CONTEXT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 		activate_context(mm, cpu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) 		local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) 		mm->context.asid[cpu] = NO_CONTEXT;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 		mm->context.cpu = -1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) #define _ITLB_ENTRIES (ITLB_ARF_WAYS << XCHAL_ITLB_ARF_ENTRIES_LOG2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) #define _DTLB_ENTRIES (DTLB_ARF_WAYS << XCHAL_DTLB_ARF_ENTRIES_LOG2)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) #if _ITLB_ENTRIES > _DTLB_ENTRIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) # define _TLB_ENTRIES _ITLB_ENTRIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) #else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) # define _TLB_ENTRIES _DTLB_ENTRIES
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) void local_flush_tlb_range(struct vm_area_struct *vma,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 		unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) 	struct mm_struct *mm = vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 	if (mm->context.asid[cpu] == NO_CONTEXT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	pr_debug("[tlbrange<%02lx,%08lx,%08lx>]\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 		 (unsigned long)mm->context.asid[cpu], start, end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 	if (end-start + (PAGE_SIZE-1) <= _TLB_ENTRIES << PAGE_SHIFT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 		int oldpid = get_rasid_register();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 		set_rasid_register(ASID_INSERT(mm->context.asid[cpu]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) 		start &= PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 		if (vma->vm_flags & VM_EXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) 			while(start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109) 				invalidate_itlb_mapping(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110) 				invalidate_dtlb_mapping(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111) 				start += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113) 		else
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114) 			while(start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115) 				invalidate_dtlb_mapping(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116) 				start += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119) 		set_rasid_register(oldpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121) 		local_flush_tlb_mm(mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123) 	local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126) void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128) 	int cpu = smp_processor_id();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129) 	struct mm_struct* mm = vma->vm_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131) 	int oldpid;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133) 	if (mm->context.asid[cpu] == NO_CONTEXT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134) 		return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 	local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) 	oldpid = get_rasid_register();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	set_rasid_register(ASID_INSERT(mm->context.asid[cpu]));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 	if (vma->vm_flags & VM_EXEC)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) 		invalidate_itlb_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) 	invalidate_dtlb_mapping(page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	set_rasid_register(oldpid);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) 	if (end > start && start >= TASK_SIZE && end <= PAGE_OFFSET &&
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 	    end - start < _TLB_ENTRIES << PAGE_SHIFT) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) 		start &= PAGE_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) 		while (start < end) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 			invalidate_itlb_mapping(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) 			invalidate_dtlb_mapping(start);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 			start += PAGE_SIZE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) 	} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 		local_flush_tlb_all();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) #ifdef CONFIG_DEBUG_TLB_SANITY
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) static unsigned get_pte_for_vaddr(unsigned vaddr)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) 	struct task_struct *task = get_current();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	struct mm_struct *mm = task->mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) 	pgd_t *pgd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 	p4d_t *p4d;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) 	pud_t *pud;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 	pmd_t *pmd;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) 	pte_t *pte;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177) 	if (!mm)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) 		mm = task->active_mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) 	pgd = pgd_offset(mm, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	if (pgd_none_or_clear_bad(pgd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 	p4d = p4d_offset(pgd, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 	if (p4d_none_or_clear_bad(p4d))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 	pud = pud_offset(p4d, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 	if (pud_none_or_clear_bad(pud))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) 	pmd = pmd_offset(pud, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 	if (pmd_none_or_clear_bad(pmd))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) 	pte = pte_offset_map(pmd, vaddr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	if (!pte)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 		return 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) 	return pte_val(*pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) enum {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	TLB_SUSPICIOUS	= 1,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 	TLB_INSANE	= 2,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) };
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) static void tlb_insane(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	BUG_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) static void tlb_suspicious(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	WARN_ON(1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213)  * Check that TLB entries with kernel ASID (1) have kernel VMA (>= TASK_SIZE),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214)  * and TLB entries with user ASID (>=4) have VMA < TASK_SIZE.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216)  * Check that valid TLB entries either have the same PA as the PTE, or PTE is
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217)  * marked as non-present. Non-present PTE and the page with non-zero refcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218)  * and zero mapcount is normal for batched TLB flush operation. Zero refcount
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219)  * means that the page was freed prematurely. Non-zero mapcount is unusual,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220)  * but does not necessary means an error, thus marked as suspicious.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) static int check_tlb_entry(unsigned w, unsigned e, bool dtlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) 	unsigned tlbidx = w | (e << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 	unsigned r0 = dtlb ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) 		read_dtlb_virtual(tlbidx) : read_itlb_virtual(tlbidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) 	unsigned r1 = dtlb ?
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 		read_dtlb_translation(tlbidx) : read_itlb_translation(tlbidx);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) 	unsigned vpn = (r0 & PAGE_MASK) | (e << PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 	unsigned pte = get_pte_for_vaddr(vpn);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) 	unsigned mm_asid = (get_rasid_register() >> 8) & ASID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 	unsigned tlb_asid = r0 & ASID_MASK;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) 	bool kernel = tlb_asid == 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) 	int rc = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) 	if (tlb_asid > 0 && ((vpn < TASK_SIZE) == kernel)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 		pr_err("%cTLB: way: %u, entry: %u, VPN %08x in %s PTE\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) 				dtlb ? 'D' : 'I', w, e, vpn,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) 				kernel ? "kernel" : "user");
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 		rc |= TLB_INSANE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) 	if (tlb_asid == mm_asid) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) 		if ((pte ^ r1) & PAGE_MASK) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 			pr_err("%cTLB: way: %u, entry: %u, mapping: %08x->%08x, PTE: %08x\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) 					dtlb ? 'D' : 'I', w, e, r0, r1, pte);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) 			if (pte == 0 || !pte_present(__pte(pte))) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 				struct page *p = pfn_to_page(r1 >> PAGE_SHIFT);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 				pr_err("page refcount: %d, mapcount: %d\n",
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) 						page_count(p),
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 						page_mapcount(p));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) 				if (!page_count(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253) 					rc |= TLB_INSANE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254) 				else if (page_mapcount(p))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255) 					rc |= TLB_SUSPICIOUS;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256) 			} else {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257) 				rc |= TLB_INSANE;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258) 			}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261) 	return rc;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void check_tlb_sanity(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) 	unsigned long flags;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	unsigned w, e;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 	int bug = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	local_irq_save(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 	for (w = 0; w < DTLB_ARF_WAYS; ++w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) 		for (e = 0; e < (1 << XCHAL_DTLB_ARF_ENTRIES_LOG2); ++e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 			bug |= check_tlb_entry(w, e, true);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	for (w = 0; w < ITLB_ARF_WAYS; ++w)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 		for (e = 0; e < (1 << XCHAL_ITLB_ARF_ENTRIES_LOG2); ++e)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 			bug |= check_tlb_entry(w, e, false);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	if (bug & TLB_INSANE)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 		tlb_insane();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) 	if (bug & TLB_SUSPICIOUS)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 		tlb_suspicious();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	local_irq_restore(flags);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #endif /* CONFIG_DEBUG_TLB_SANITY */