Orange Pi5 kernel

^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   1) #include <linux/gfp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   2) #include <linux/highmem.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   3) #include <linux/kernel.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   4) #include <linux/mmdebug.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   5) #include <linux/mm_types.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   6) #include <linux/pagemap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   7) #include <linux/rcupdate.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   8) #include <linux/smp.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300   9) #include <linux/swap.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  10) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  11) #include <asm/pgalloc.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  12) #include <asm/tlb.h>
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  13) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  14) #ifndef CONFIG_MMU_GATHER_NO_GATHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  15) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  16) static bool tlb_next_batch(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  17) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  18) 	struct mmu_gather_batch *batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  19) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  20) 	batch = tlb->active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  21) 	if (batch->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  22) 		tlb->active = batch->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  23) 		return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  24) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  25) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  26) 	if (tlb->batch_count == MAX_GATHER_BATCH_COUNT)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  27) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  28) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  29) 	batch = (void *)__get_free_pages(GFP_NOWAIT | __GFP_NOWARN, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  30) 	if (!batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  31) 		return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  32) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  33) 	tlb->batch_count++;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  34) 	batch->next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  35) 	batch->nr   = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  36) 	batch->max  = MAX_GATHER_BATCH;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  37) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  38) 	tlb->active->next = batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  39) 	tlb->active = batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  40) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  41) 	return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  42) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  43) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  44) static void tlb_batch_pages_flush(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  45) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  46) 	struct mmu_gather_batch *batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  47) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  48) 	for (batch = &tlb->local; batch && batch->nr; batch = batch->next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  49) 		free_pages_and_swap_cache(batch->pages, batch->nr);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  50) 		batch->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  51) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  52) 	tlb->active = &tlb->local;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  53) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  54) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  55) static void tlb_batch_list_free(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  56) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  57) 	struct mmu_gather_batch *batch, *next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  58) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  59) 	for (batch = tlb->local.next; batch; batch = next) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  60) 		next = batch->next;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  61) 		free_pages((unsigned long)batch, 0);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  62) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  63) 	tlb->local.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  64) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  65) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  66) bool __tlb_remove_page_size(struct mmu_gather *tlb, struct page *page, int page_size)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  67) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  68) 	struct mmu_gather_batch *batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  69) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  70) 	VM_BUG_ON(!tlb->end);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  71) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  72) #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  73) 	VM_WARN_ON(tlb->page_size != page_size);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  74) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  75) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  76) 	batch = tlb->active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  77) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  78) 	 * Add the page and check if we are full. If so
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  79) 	 * force a flush.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  80) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  81) 	batch->pages[batch->nr++] = page;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  82) 	if (batch->nr == batch->max) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  83) 		if (!tlb_next_batch(tlb))
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  84) 			return true;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  85) 		batch = tlb->active;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  86) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  87) 	VM_BUG_ON_PAGE(batch->nr > batch->max, page);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  88) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  89) 	return false;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  90) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  91) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  92) #endif /* MMU_GATHER_NO_GATHER */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  93) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  94) #ifdef CONFIG_MMU_GATHER_TABLE_FREE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  95) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  96) static void __tlb_remove_table_free(struct mmu_table_batch *batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  97) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  98) 	int i;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300  99) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 100) 	for (i = 0; i < batch->nr; i++)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 101) 		__tlb_remove_table(batch->tables[i]);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 102) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 103) 	free_page((unsigned long)batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 104) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 105) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 106) #ifdef CONFIG_MMU_GATHER_RCU_TABLE_FREE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 107) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 108) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 109)  * Semi RCU freeing of the page directories.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 110)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 111)  * This is needed by some architectures to implement software pagetable walkers.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 112)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 113)  * gup_fast() and other software pagetable walkers do a lockless page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 114)  * walk and therefore needs some synchronization with the freeing of the page
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 115)  * directories. The chosen means to accomplish that is by disabling IRQs over
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 116)  * the walk.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 117)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 118)  * Architectures that use IPIs to flush TLBs will then automagically DTRT,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 119)  * since we unlink the page, flush TLBs, free the page. Since the disabling of
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 120)  * IRQs delays the completion of the TLB flush we can never observe an already
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 121)  * freed page.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 122)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 123)  * Architectures that do not have this (PPC) need to delay the freeing by some
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 124)  * other means, this is that means.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 125)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 126)  * What we do is batch the freed directory pages (tables) and RCU free them.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 127)  * We use the sched RCU variant, as that guarantees that IRQ/preempt disabling
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 128)  * holds off grace periods.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 129)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 130)  * However, in order to batch these pages we need to allocate storage, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 131)  * allocation is deep inside the MM code and can thus easily fail on memory
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 132)  * pressure. To guarantee progress we fall back to single table freeing, see
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 133)  * the implementation of tlb_remove_table_one().
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 134)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 135)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 136) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 137) static void tlb_remove_table_smp_sync(void *arg)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 138) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 139) 	/* Simply deliver the interrupt */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 140) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 141) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 142) static void tlb_remove_table_sync_one(void)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 143) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 144) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 145) 	 * This isn't an RCU grace period and hence the page-tables cannot be
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 146) 	 * assumed to be actually RCU-freed.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 147) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 148) 	 * It is however sufficient for software page-table walkers that rely on
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 149) 	 * IRQ disabling.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 150) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 151) 	smp_call_function(tlb_remove_table_smp_sync, NULL, 1);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 152) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 153) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 154) static void tlb_remove_table_rcu(struct rcu_head *head)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 155) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 156) 	__tlb_remove_table_free(container_of(head, struct mmu_table_batch, rcu));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 157) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 158) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 159) static void tlb_remove_table_free(struct mmu_table_batch *batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 160) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 161) 	call_rcu(&batch->rcu, tlb_remove_table_rcu);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 162) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 163) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 164) #else /* !CONFIG_MMU_GATHER_RCU_TABLE_FREE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 165) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 166) static void tlb_remove_table_sync_one(void) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 167) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 168) static void tlb_remove_table_free(struct mmu_table_batch *batch)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 169) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 170) 	__tlb_remove_table_free(batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 171) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 172) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 173) #endif /* CONFIG_MMU_GATHER_RCU_TABLE_FREE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 174) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 175) /*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 176)  * If we want tlb_remove_table() to imply TLB invalidates.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 177)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 178) static inline void tlb_table_invalidate(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 179) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 180) 	if (tlb_needs_table_invalidate()) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 181) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 182) 		 * Invalidate page-table caches used by hardware walkers. Then
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 183) 		 * we still need to RCU-sched wait while freeing the pages
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 184) 		 * because software walkers can still be in-flight.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 185) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 186) 		tlb_flush_mmu_tlbonly(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 187) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 188) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 189) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 190) static void tlb_remove_table_one(void *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 191) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 192) 	tlb_remove_table_sync_one();
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 193) 	__tlb_remove_table(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 194) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 195) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 196) static void tlb_table_flush(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 197) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 198) 	struct mmu_table_batch **batch = &tlb->batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 199) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 200) 	if (*batch) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 201) 		tlb_table_invalidate(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 202) 		tlb_remove_table_free(*batch);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 203) 		*batch = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 204) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 205) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 206) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 207) void tlb_remove_table(struct mmu_gather *tlb, void *table)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 208) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 209) 	struct mmu_table_batch **batch = &tlb->batch;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 210) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 211) 	if (*batch == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 212) 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 213) 		if (*batch == NULL) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 214) 			tlb_table_invalidate(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 215) 			tlb_remove_table_one(table);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 216) 			return;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 217) 		}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 218) 		(*batch)->nr = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 219) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 220) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 221) 	(*batch)->tables[(*batch)->nr++] = table;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 222) 	if ((*batch)->nr == MAX_TABLE_BATCH)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 223) 		tlb_table_flush(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 224) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 225) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 226) static inline void tlb_table_init(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 227) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 228) 	tlb->batch = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 229) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 230) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 231) #else /* !CONFIG_MMU_GATHER_TABLE_FREE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 232) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 233) static inline void tlb_table_flush(struct mmu_gather *tlb) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 234) static inline void tlb_table_init(struct mmu_gather *tlb) { }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 235) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 236) #endif /* CONFIG_MMU_GATHER_TABLE_FREE */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 237) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 238) static void tlb_flush_mmu_free(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 239) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 240) 	tlb_table_flush(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 241) #ifndef CONFIG_MMU_GATHER_NO_GATHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 242) 	tlb_batch_pages_flush(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 243) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 244) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 245) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 246) void tlb_flush_mmu(struct mmu_gather *tlb)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 247) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 248) 	tlb_flush_mmu_tlbonly(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 249) 	tlb_flush_mmu_free(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 250) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 251) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 252) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 253)  * tlb_gather_mmu - initialize an mmu_gather structure for page-table tear-down
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 254)  * @tlb: the mmu_gather structure to initialize
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 255)  * @mm: the mm_struct of the target address space
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 256)  * @start: start of the region that will be removed from the page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 257)  * @end: end of the region that will be removed from the page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 258)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 259)  * Called to initialize an (on-stack) mmu_gather structure for page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 260)  * tear-down from @mm. The @start and @end are set to 0 and -1
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 261)  * respectively when @mm is without users and we're going to destroy
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 262)  * the full address space (exit/execve).
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 263)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 264) void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 265) 			unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 266) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 267) 	tlb->mm = mm;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 268) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 269) 	/* Is it from 0 to ~0? */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 270) 	tlb->fullmm     = !(start | (end+1));
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 271) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 272) #ifndef CONFIG_MMU_GATHER_NO_GATHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 273) 	tlb->need_flush_all = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 274) 	tlb->local.next = NULL;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 275) 	tlb->local.nr   = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 276) 	tlb->local.max  = ARRAY_SIZE(tlb->__pages);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 277) 	tlb->active     = &tlb->local;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 278) 	tlb->batch_count = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 279) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 280) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 281) 	tlb_table_init(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 282) #ifdef CONFIG_MMU_GATHER_PAGE_SIZE
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 283) 	tlb->page_size = 0;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 284) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 285) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 286) 	__tlb_reset_range(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 287) 	inc_tlb_flush_pending(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 288) }
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 289) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 290) /**
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 291)  * tlb_finish_mmu - finish an mmu_gather structure
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 292)  * @tlb: the mmu_gather structure to finish
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 293)  * @start: start of the region that will be removed from the page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 294)  * @end: end of the region that will be removed from the page-table
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 295)  *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 296)  * Called at the end of the shootdown operation to free up any resources that
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 297)  * were required.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 298)  */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 299) void tlb_finish_mmu(struct mmu_gather *tlb,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 300) 		unsigned long start, unsigned long end)
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 301) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 302) 	/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 303) 	 * If there are parallel threads are doing PTE changes on same range
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 304) 	 * under non-exclusive lock (e.g., mmap_lock read-side) but defer TLB
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 305) 	 * flush by batching, one thread may end up seeing inconsistent PTEs
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 306) 	 * and result in having stale TLB entries.  So flush TLB forcefully
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 307) 	 * if we detect parallel PTE batching threads.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 308) 	 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 309) 	 * However, some syscalls, e.g. munmap(), may free page tables, this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 310) 	 * needs force flush everything in the given range. Otherwise this
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 311) 	 * may result in having stale TLB entries for some architectures,
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 312) 	 * e.g. aarch64, that could specify flush what level TLB.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 313) 	 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 314) 	if (mm_tlb_flush_nested(tlb->mm)) {
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 315) 		/*
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 316) 		 * The aarch64 yields better performance with fullmm by
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 317) 		 * avoiding multiple CPUs spamming TLBI messages at the
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 318) 		 * same time.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 319) 		 *
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 320) 		 * On x86 non-fullmm doesn't yield significant difference
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 321) 		 * against fullmm.
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 322) 		 */
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 323) 		tlb->fullmm = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 324) 		__tlb_reset_range(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 325) 		tlb->freed_tables = 1;
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 326) 	}
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 327) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 328) 	tlb_flush_mmu(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 329) 
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 330) #ifndef CONFIG_MMU_GATHER_NO_GATHER
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 331) 	tlb_batch_list_free(tlb);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 332) #endif
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 333) 	dec_tlb_flush_pending(tlb->mm);
^8f3ce5b39 (kx 2023-10-28 12:00:06 +0300 334) }